稳定AI原生壳异步终态测试
等待 Agent 终态和执行通道释放后再断言持久化副作用 使用一次性测试注入覆盖 CI root 环境下的对话写失败 补充异步 Runtime 与 CI 写失败的共享踩坑记录
This commit is contained in:
@@ -5488,6 +5488,8 @@ fn append_local_conversation_message_for_session_internal_at(
|
||||
return Err("finalization conversation 审计身份或角色无效".to_string());
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
take_local_conversation_append_failure_injection(root, role)?;
|
||||
let record = PersistedLocalConversationMessageRecord {
|
||||
schema_version: LOCAL_CONVERSATION_SCHEMA_VERSION.to_string(),
|
||||
role: role.to_string(),
|
||||
@@ -5591,6 +5593,21 @@ fn append_local_conversation_message_for_session_internal_at(
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn take_local_conversation_append_failure_injection(root: &Path, role: &str) -> Result<(), String> {
|
||||
let path = root.join(".agent/runtime/test-fail-next-conversation-append");
|
||||
match fs::read_to_string(&path) {
|
||||
Ok(expected_role) if expected_role.trim() == role => {
|
||||
fs::remove_file(&path)
|
||||
.map_err(|error| format!("清理对话写入测试失败注入标记失败:{error}"))?;
|
||||
Err(format!("测试注入 {role} 对话写入失败"))
|
||||
}
|
||||
Ok(_) => Ok(()),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(format!("读取对话写入测试失败注入标记失败:{error}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn append_local_conversation_message_for_session_at(
|
||||
root: &Path,
|
||||
agent_id: Option<&str>,
|
||||
|
||||
@@ -182,7 +182,14 @@ async fn agent_goal_edit_pause_resume_keeps_one_session_and_run_until_completion
|
||||
.send(final_tool_plan_response("持久 Goal 已在同一 run 完成。"))
|
||||
.expect("complete resumed Goal");
|
||||
|
||||
let completed = wait_for_agent_runtime_idle(&root, "code-prototype");
|
||||
let completed = wait_for_agent_runtime_terminal_and_lane_release(
|
||||
&root,
|
||||
"code-prototype",
|
||||
run_id,
|
||||
"idle",
|
||||
"completed",
|
||||
)
|
||||
.state;
|
||||
assert_eq!(completed.phase, "completed");
|
||||
assert_eq!(completed.run_id, run_id);
|
||||
let goal = read_game_creator_agent_goal_at(&root, "code-prototype", &session_id)
|
||||
@@ -1315,6 +1322,42 @@ fn wait_for_agent_runtime_idle(root: &Path, agent_id: &str) -> AgentRuntimeState
|
||||
runtime
|
||||
}
|
||||
|
||||
fn wait_for_agent_runtime_terminal_and_lane_release(
|
||||
root: &Path,
|
||||
agent_id: &str,
|
||||
run_id: &str,
|
||||
status: &str,
|
||||
phase: &str,
|
||||
) -> AgentRuntimeResult {
|
||||
let mut result = read_game_creator_agent_runtime_at(root, agent_id)
|
||||
.expect("read runtime while waiting for terminal lane release");
|
||||
for _ in 0..250 {
|
||||
let matches_terminal = result.state.run_id == run_id
|
||||
&& result.state.status == status
|
||||
&& result.state.phase == phase;
|
||||
if matches_terminal
|
||||
&& game_creator_agent_runtime_task_lock_is_available(root, agent_id)
|
||||
.expect("probe runtime lane while waiting for terminal release")
|
||||
{
|
||||
let terminal = read_game_creator_agent_runtime_at(root, agent_id)
|
||||
.expect("reread runtime after terminal lane release");
|
||||
if terminal.state.run_id == run_id
|
||||
&& terminal.state.status == status
|
||||
&& terminal.state.phase == phase
|
||||
{
|
||||
return terminal;
|
||||
}
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
result = read_game_creator_agent_runtime_at(root, agent_id)
|
||||
.expect("read runtime while waiting for terminal lane release");
|
||||
}
|
||||
panic!(
|
||||
"runtime did not reach {status}/{phase} for run {run_id} before the Agent lane released; last run={} status={} phase={}",
|
||||
result.state.run_id, result.state.status, result.state.phase
|
||||
);
|
||||
}
|
||||
|
||||
fn wait_for_agent_runtime_phase(root: &Path, agent_id: &str, phase: &str) -> AgentRuntimeState {
|
||||
let mut runtime = read_game_creator_agent_runtime_at(root, agent_id)
|
||||
.expect("read runtime while waiting for phase")
|
||||
@@ -17947,18 +17990,14 @@ async fn background_agent_runtime_marks_unconverged_loop_budget_exhausted() {
|
||||
.expect("planning request");
|
||||
assert!(request.contains(&format!("第 {iteration} 轮")));
|
||||
}
|
||||
assert!(receiver.recv_timeout(Duration::from_millis(200)).is_err());
|
||||
|
||||
let mut result =
|
||||
read_game_creator_agent_runtime_at(&root, "design-director").expect("read budget runtime");
|
||||
for _ in 0..50 {
|
||||
if result.state.status == "failed" {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
result = read_game_creator_agent_runtime_at(&root, "design-director")
|
||||
.expect("read budget runtime");
|
||||
}
|
||||
let result = wait_for_agent_runtime_terminal_and_lane_release(
|
||||
&root,
|
||||
"design-director",
|
||||
"design-budget-exhausted-run",
|
||||
"failed",
|
||||
"budget-exhausted",
|
||||
);
|
||||
assert!(receiver.try_recv().is_err());
|
||||
assert_eq!(result.state.status, "failed");
|
||||
assert_eq!(result.state.phase, "budget-exhausted");
|
||||
assert!(result
|
||||
@@ -24233,18 +24272,11 @@ fn background_task_does_not_execute_when_user_message_cannot_persist() {
|
||||
init_local_game_project_at(&root, "project-1", "后台任务对话一致性测试").expect("project init");
|
||||
let session_id = resolve_agent_conversation_session_id_at(&root, "design-director", None, true)
|
||||
.expect("resolve agent session");
|
||||
let (conversation_path, _, _) =
|
||||
conversation_file_path_for_session(&root, Some("design-director"), Some(&session_id))
|
||||
.expect("resolve conversation path");
|
||||
fs::create_dir_all(conversation_path.parent().expect("conversation parent"))
|
||||
.expect("create conversation parent");
|
||||
fs::write(&conversation_path, "").expect("create empty conversation file");
|
||||
let mut conversation_permissions = fs::metadata(&conversation_path)
|
||||
.expect("read conversation permissions")
|
||||
.permissions();
|
||||
conversation_permissions.set_readonly(true);
|
||||
fs::set_permissions(&conversation_path, conversation_permissions)
|
||||
.expect("make conversation read only");
|
||||
fs::write(
|
||||
root.join(".agent/runtime/test-fail-next-background-conversation"),
|
||||
b"fail once\n",
|
||||
)
|
||||
.expect("arm one-shot background conversation failure");
|
||||
let runtime_lock = try_acquire_game_creator_agent_runtime_task_lock(&root, "design-director")
|
||||
.expect("acquire runtime lock")
|
||||
.expect("runtime lock available");
|
||||
@@ -24268,14 +24300,18 @@ fn background_task_does_not_execute_when_user_message_cannot_persist() {
|
||||
.expect("failed queued task exists");
|
||||
assert_eq!(task.status, "failed");
|
||||
assert_eq!(task.phase, "conversation-write-failed");
|
||||
assert!(!root
|
||||
.join(".agent/runtime/test-fail-next-background-conversation")
|
||||
.exists());
|
||||
let conversation =
|
||||
read_local_conversation_for_session_at(&root, Some("design-director"), Some(&session_id))
|
||||
.expect("read conversation after injected user write failure");
|
||||
assert!(conversation
|
||||
.messages
|
||||
.iter()
|
||||
.all(|message| message.content != "这条任务必须先持久化对话"));
|
||||
|
||||
drop(runtime_lock);
|
||||
let mut conversation_permissions = fs::metadata(&conversation_path)
|
||||
.expect("read final conversation permissions")
|
||||
.permissions();
|
||||
conversation_permissions.set_readonly(false);
|
||||
fs::set_permissions(&conversation_path, conversation_permissions)
|
||||
.expect("restore conversation permissions");
|
||||
fs::remove_dir_all(root).ok();
|
||||
}
|
||||
|
||||
@@ -24317,15 +24353,9 @@ async fn background_task_recovers_when_assistant_message_cannot_persist() {
|
||||
let session_id =
|
||||
resolve_agent_conversation_session_id_at(&root, "design-director", None, false)
|
||||
.expect("resolve agent session");
|
||||
let (conversation_path, _, _) =
|
||||
conversation_file_path_for_session(&root, Some("design-director"), Some(&session_id))
|
||||
.expect("resolve conversation path");
|
||||
let mut conversation_permissions = fs::metadata(&conversation_path)
|
||||
.expect("read conversation permissions")
|
||||
.permissions();
|
||||
conversation_permissions.set_readonly(true);
|
||||
fs::set_permissions(&conversation_path, conversation_permissions)
|
||||
.expect("make conversation read only");
|
||||
let conversation_failure_path = root.join(".agent/runtime/test-fail-next-conversation-append");
|
||||
fs::write(&conversation_failure_path, b"assistant\n")
|
||||
.expect("arm one-shot assistant conversation failure");
|
||||
release_sender
|
||||
.send(())
|
||||
.expect("release background planning response");
|
||||
@@ -24341,16 +24371,10 @@ async fn background_task_recovers_when_assistant_message_cannot_persist() {
|
||||
.expect("read assistant persistence runtime");
|
||||
}
|
||||
|
||||
let mut conversation_permissions = fs::metadata(&conversation_path)
|
||||
.expect("read final conversation permissions")
|
||||
.permissions();
|
||||
conversation_permissions.set_readonly(false);
|
||||
fs::set_permissions(&conversation_path, conversation_permissions)
|
||||
.expect("restore conversation permissions");
|
||||
|
||||
assert_eq!(result.state.status, "running");
|
||||
assert_eq!(result.state.phase, "finalizing");
|
||||
assert!(result.state.error.is_some());
|
||||
assert!(!conversation_failure_path.exists());
|
||||
assert!(result.recent_tasks.iter().any(|task| {
|
||||
task.run_id == "assistant-conversation-write-failure-run"
|
||||
&& task.status == "running"
|
||||
@@ -24413,7 +24437,14 @@ async fn background_task_recovers_when_assistant_message_cannot_persist() {
|
||||
assert!(request_receiver
|
||||
.recv_timeout(Duration::from_millis(250))
|
||||
.is_err());
|
||||
let completed = wait_for_agent_runtime_idle(&root, "design-director");
|
||||
let completed = wait_for_agent_runtime_terminal_and_lane_release(
|
||||
&root,
|
||||
"design-director",
|
||||
"assistant-conversation-write-failure-run",
|
||||
"idle",
|
||||
"completed",
|
||||
)
|
||||
.state;
|
||||
assert_eq!(completed.phase, "completed");
|
||||
assert_eq!(completed.run_id, "assistant-conversation-write-failure-run");
|
||||
assert_eq!(
|
||||
@@ -31263,6 +31294,7 @@ async fn background_finalization_replans_same_run_after_cross_agent_revision_dri
|
||||
assert!(current_final_reply_request.contains(run_id));
|
||||
|
||||
let runtime = wait_for_agent_runtime_idle(&root, "design-director");
|
||||
wait_for_provider_handoff_terminal_cleanup(&root, "design-director", run_id);
|
||||
assert_eq!(runtime.status, "idle");
|
||||
assert_eq!(runtime.phase, "completed");
|
||||
assert_eq!(runtime.run_id, run_id);
|
||||
@@ -62694,19 +62726,14 @@ fn project_supervisor_corrupt_child_evidence_enters_parent_reconciliation() {
|
||||
.expect("delivery remains present");
|
||||
assert_eq!(persisted.status, StaticDelegateDeliveryStatus::Dispatched);
|
||||
assert_eq!(persisted.structured_result, None);
|
||||
let mut parent_runtime = None;
|
||||
for _ in 0..100 {
|
||||
let current =
|
||||
read_game_creator_agent_runtime_at(&root, GAME_CREATOR_PROJECT_SUPERVISOR_AGENT_ID)
|
||||
.expect("read reconciled Supervisor runtime")
|
||||
.state;
|
||||
if current.phase == "needs-reconciliation" {
|
||||
parent_runtime = Some(current);
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
let parent_runtime = parent_runtime.expect("busy parent lane eventually reconciles");
|
||||
let parent_runtime = wait_for_agent_runtime_terminal_and_lane_release(
|
||||
&root,
|
||||
GAME_CREATOR_PROJECT_SUPERVISOR_AGENT_ID,
|
||||
parent_run_id,
|
||||
"failed",
|
||||
"needs-reconciliation",
|
||||
)
|
||||
.state;
|
||||
assert_eq!(parent_runtime.run_id, parent_run_id);
|
||||
assert_eq!(parent_runtime.status, "failed");
|
||||
assert_eq!(parent_runtime.phase, "needs-reconciliation");
|
||||
|
||||
@@ -32,12 +32,20 @@
|
||||
|
||||
## 异步 Runtime 测试不能把 child idle 当成终态结果已发布
|
||||
|
||||
- 现象:isolated child 已显示 idle,单次 all-join reconcile 却偶发返回空列表,完整 Rust suite 里出现低概率失败,单独重跑通常通过。
|
||||
- 原因:child Runtime 释放执行 lane 与持久化终态 result、发布 join readiness 不是同一个原子观测点;测试只等待 idle,会在终态 result 发布前抢先 reconcile。
|
||||
- 处理:产品协议仍以 durable terminal result 和 join readiness 为准。测试在有界时限内重复调用幂等 reconcile,直到取得唯一 join 或超时;不得靠固定长 sleep,也不能因为第一次为空就把协议改成吞掉未完成 child。
|
||||
- 验证:`isolated_agents_with_same_template_run_independently_and_join_once` 最多执行 100 次、每次间隔 20ms 的 reconcile,并继续断言只有一个 all-join 和一次父唤醒。
|
||||
- 现象:isolated child 已显示 idle,单次 all-join reconcile 却偶发返回空列表;或者 Runtime 已显示 completed / failed,Goal、conversation、Agent DB 审计和 per-Agent lock 仍未完成,完整 Rust suite 里出现低概率失败,单独重跑通常通过。
|
||||
- 原因:Runtime state、Goal sidecar、终态 result、conversation、审计记录、handoff 清理和执行 lane 释放不是同一个原子观测点;测试只等待 idle / failed 会在同一后台 drain 的 durable 收尾前抢先断言。
|
||||
- 处理:产品协议仍以 durable terminal result 和 join readiness 为准。测试在有界时限内等待业务目标终态;需要断言同一 drain 的后续副作用时,同时以 per-Agent runtime task lock 释放为 fence,命中后重新读取投影。join 场景继续重复调用幂等 reconcile,直到取得唯一 join 或超时;不得靠固定长 sleep,也不能因为第一次为空就把协议改成吞掉未完成 child。
|
||||
- 验证:`isolated_agents_with_same_template_run_independently_and_join_once` 最多执行 100 次、每次间隔 20ms 的 reconcile,并继续断言只有一个 all-join 和一次父唤醒;Goal、loop-budget、finalization 与 Supervisor reconciliation 测试必须在目标 status / phase 与 Agent lane 同时收束后再读取最终副作用。
|
||||
- 关联:`apps/ai-game-creator-shell/src-tauri/src/tests.rs`、`apps/ai-game-creator-shell/src-tauri/src/agent.rs`。
|
||||
|
||||
## CI root 环境不能用文件只读权限注入写失败
|
||||
|
||||
- 现象:本地测试把 conversation 文件设为 readonly 后能稳定得到写入失败,Gitea Actions 中同一断言却发现写入成功并继续执行任务。
|
||||
- 原因:隔离 job 内测试进程可能以 root 运行;root 不受普通 owner write bit 的同等限制,`set_readonly(true)` 不是跨 runner 身份的确定性故障注入。
|
||||
- 处理:需要覆盖写失败恢复时使用仅在 `cfg(test)` 生效、一次性消费并限定写入阶段的 marker;生产路径仍走真实持久化函数。测试同时断言 marker 已消费、失败前数据未落盘和恢复后 exactly-once,不依赖 chmod、固定 sleep 或 runner 用户身份。
|
||||
- 验证:在普通本地用户和 root 容器中分别运行用户消息、assistant 最终回复持久化失败测试,均应进入相同 durable phase 并通过恢复断言。
|
||||
- 关联:`apps/ai-game-creator-shell/src-tauri/src/project.rs`、`apps/ai-game-creator-shell/src-tauri/src/agent.rs`、`apps/ai-game-creator-shell/src-tauri/src/tests.rs`。
|
||||
|
||||
## PTY 测试不能假设输入回显与后续输出必然分行
|
||||
|
||||
- 现象:PTY 环境隔离用例偶发得到 `你好BRIDGE_ENV:`,而不是独立的 `你好` 与 `BRIDGE_ENV:` 两行;真实私有环境变量并未泄漏,但整行相等断言失败。
|
||||
|
||||
Reference in New Issue
Block a user