修复 Linux 沙箱命令退出回收的偶发竞态
Project CI / AI game creator shell Rust crates (pull_request) Successful in 1m33s
Project CI / AI game creator shell Rust smoke (pull_request) Successful in 2m7s
Project CI / Backend tests (pull_request) Successful in 3m49s
Project CI / Frontend tests (pull_request) Successful in 1m53s
Project CI / Native shell tests (pull_request) Successful in 5m51s
Project CI / AI game creator shell Rust lane 2/2 (pull_request) Successful in 8m57s
Project CI / Repository checks (pull_request) Successful in 1m55s
Project CI / AI game creator shell Rust lane 1/2 (pull_request) Successful in 9m19s
Project CI / AI game creator shell web tests (pull_request) Successful in 1m24s
Project CI / AI game creator shell Rust crates (push) Successful in 1m30s
Project CI / AI game creator shell Rust smoke (push) Successful in 2m9s
Project CI / Backend tests (push) Successful in 4m49s
Project CI / Frontend tests (push) Successful in 2m9s
Project CI / Native shell tests (push) Successful in 6m54s
Project CI / AI game creator shell Rust lane 2/2 (push) Successful in 9m56s
Project CI / AI game creator shell Rust lane 1/2 (push) Successful in 10m14s
Project CI / Repository checks (push) Successful in 2m2s
Project CI / AI game creator shell web tests (push) Successful in 1m30s

统一正常退出、取消和超时的有界进程组退出确认,保留归属校验
提前记录启动身份,修补终态协议错误的清理及输出任务回收
补充确定性竞态回归并同步运行时规范和排障记忆
保持现有 CI 并行、分片和重试策略不变
This commit was merged in pull request #517.
This commit is contained in:
2026-09-24 16:45:07 +00:00
parent 4912aa4df0
commit 6ed1fd26ed
3 changed files with 161 additions and 55 deletions
@@ -15,6 +15,8 @@ const PROJECT_COMMAND_MAX_ARGUMENT_BYTES: usize = 8 * 1024;
const PROJECT_COMMAND_MIN_TIMEOUT_SECONDS: u64 = 1;
const PROJECT_COMMAND_MAX_TIMEOUT_SECONDS: u64 = 300;
const PROJECT_COMMAND_OUTPUT_MAX_BYTES: usize = 24 * 1024;
#[cfg(target_os = "linux")]
const PROJECT_COMMAND_CLEANUP_TIMEOUT: Duration = Duration::from_secs(5);
const PROJECT_COMMAND_FINGERPRINT_MAX_ENTRIES: usize = 20_000;
const PROJECT_COMMAND_FINGERPRINT_MAX_FILES: usize = 10_000;
const PROJECT_COMMAND_FINGERPRINT_MAX_BYTES: u64 = 512 * 1024 * 1024;
@@ -222,14 +224,30 @@ impl ProjectCommandTree {
}
#[cfg(not(windows))]
{
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
let requested = self.request_owned_group_termination();
let _ = child.start_kill();
let waited = tokio::time::timeout(Duration::from_secs(5), child.wait()).await;
requested?;
let waited = tokio::time::timeout_at(deadline, child.wait()).await;
waited
.map_err(|_| "等待受控命令主进程退出超时")?
.map_err(|_| "受控命令主进程退出未确认")?;
Ok("已请求终止受控进程组并回收主进程,完整子树状态未证明".into())
#[cfg(target_os = "linux")]
{
let Self::Group { pid, .. } = self;
// leader 可能与取消同时退出;只要组已停止便无需补发信号。
if let Err(error) = wait_linux_project_command_group_exit(*pid, deadline).await {
return Err(match requested {
Ok(_) => error,
Err(request_error) => format!("{request_error};{error}"),
});
}
return Ok("主进程已回收,受控进程组已无活成员;完整子树状态未证明".into());
}
#[cfg(not(target_os = "linux"))]
{
requested?;
Ok("已请求终止受控进程组并回收主进程,完整子树状态未证明".into())
}
}
}
@@ -244,17 +262,39 @@ impl ProjectCommandTree {
#[cfg(target_os = "linux")]
{
let Self::Group { pid, .. } = self;
// 容器 PID 1 可能不回收 bwrap 的孤儿僵尸;它们不再执行,也无法被信号终止。
// 仅在确认没有存活成员时免除清理,存活成员仍须通过 leader 身份核对。
if !linux_project_command_group_has_live_members(*pid)? {
return Ok(());
}
// wait 已回收 leader,不能再用旧 PID 授权发信号。
// namespace 后代可能仍在退出,容器 PID 1 也可能保留孤儿僵尸。
return wait_linux_project_command_group_exit(
*pid,
tokio::time::Instant::now() + PROJECT_COMMAND_CLEANUP_TIMEOUT,
)
.await;
}
#[cfg(not(target_os = "linux"))]
self.request_owned_group_termination().map(|_| ())
}
}
}
#[cfg(target_os = "linux")]
async fn wait_linux_project_command_group_exit(
group: u32,
deadline: tokio::time::Instant,
) -> Result<(), String> {
while linux_project_command_group_has_live_members(group)? {
if tokio::time::Instant::now() >= deadline {
return Err(
"受控进程组仍有存活成员,退出未确认;不得向身份未确认的进程组补发信号".into(),
);
}
tokio::time::sleep_until(
deadline.min(tokio::time::Instant::now() + Duration::from_millis(20)),
)
.await;
}
Ok(())
}
#[cfg(target_os = "linux")]
fn linux_project_command_group_has_live_members(group: u32) -> Result<bool, String> {
let inspect = || -> std::io::Result<bool> {
@@ -1842,6 +1882,17 @@ where
),
));
}
// 目标放行后可能立即退出,必须趁 ready gate 仍持有 launcher 时记录归属。
let tree = match ProjectCommandTree::attach(&child) {
Ok(tree) => tree,
Err(error) => {
let termination = terminate_project_command_process_group(&mut child).await;
return Err(ProjectCommandError::new(
ProjectCommandErrorStage::Preflight,
project_command_launch_error_with_termination(error, termination),
));
}
};
if let Err(error) = durable_commit() {
let termination = terminate_project_command_process_group(&mut child).await;
return Err(ProjectCommandError::new(
@@ -1860,12 +1911,7 @@ where
// cancelled future must not erase the launch-unknown decision window.
let exec = gate.wait_target_exec(Duration::from_secs(3));
match exec {
Ok(TargetExecState::Established) => {
let tree = ProjectCommandTree::attach(&child).map_err(|error| {
ProjectCommandError::new(ProjectCommandErrorStage::LaunchUnknown, error)
})?;
Ok(EstablishedProjectCommand { tree, child, gate })
}
Ok(TargetExecState::Established) => Ok(EstablishedProjectCommand { tree, child, gate }),
Ok(TargetExecState::Failed { errno }) => {
let termination = terminate_project_command_process_group_after_commit(&mut child);
Err(ProjectCommandError::new(
@@ -1977,7 +2023,11 @@ async fn terminate_project_command_process_group(
.ok_or_else(|| "请求终止受控进程组失败:子进程缺少 pid".to_string())?;
let group_result = request_unix_project_command_process_group_termination(process_id);
let child_kill_error = child.start_kill().err();
let wait_result = child.wait().await;
let deadline = tokio::time::Instant::now() + PROJECT_COMMAND_CLEANUP_TIMEOUT;
let wait_result = tokio::time::timeout_at(deadline, child.wait())
.await
.map_err(|_| "等待受控命令主进程退出超时".to_string())?
.map_err(|error| error.to_string());
if let Err(error) = &group_result {
let fallback = match (&child_kill_error, &wait_result) {
(_, Ok(_)) => "主进程已回收,但无法确认其余组内进程".to_string(),
@@ -1989,6 +2039,7 @@ async fn terminate_project_command_process_group(
return Err(format!("{error};{fallback}"));
}
wait_result.map_err(|error| format!("请求终止受控进程组后等待主进程失败:{error}"))?;
wait_linux_project_command_group_exit(process_id, deadline).await?;
Ok(format!(
"{}并完成主进程回收",
group_result.expect("group termination result checked")
@@ -2220,7 +2271,7 @@ where
let (exit_code, timed_out, termination_summary) = match wait {
ProjectCommandWait::Exited(Ok(status)) => {
#[cfg(target_os = "linux")]
let _terminal = wait_established_project_command_terminal(gate).await?;
let terminal = wait_established_project_command_terminal(gate).await;
if let Err(error) = tree.after_main_exit(&mut child).await {
stdout_task.abort();
stderr_task.abort();
@@ -2229,6 +2280,12 @@ where
format!("command.exec 主进程退出后进程树未确认回收,需要人工核对:{error}"),
));
}
#[cfg(target_os = "linux")]
if let Err(error) = terminal {
stdout_task.abort();
stderr_task.abort();
return Err(error);
}
(status.code(), false, None)
}
ProjectCommandWait::Exited(Err(error)) => {
@@ -2541,45 +2598,83 @@ mod tests {
return;
}
assert_eq!(unsafe { libc::prctl(libc::PR_SET_CHILD_SUBREAPER, 1) }, 0);
let mut command = tokio::process::Command::new("/bin/sh");
command
.args(["-c", "sleep 60 & echo $!; read release"])
.stdin(Stdio::piped())
.stdout(Stdio::piped());
command.as_std_mut().process_group(0);
let mut child = command.spawn().unwrap();
let tree = ProjectCommandTree::attach(&child).unwrap();
let mut output = tokio::io::BufReader::new(child.stdout.take().unwrap());
let mut line = String::new();
tokio::io::AsyncBufReadExt::read_line(&mut output, &mut line)
.await
.unwrap();
let descendant: i32 = line.trim().parse().unwrap();
drop(child.stdin.take());
child.wait().await.unwrap();
let live_result = tree.after_main_exit(&mut child).await;
assert_eq!(unsafe { libc::kill(descendant, libc::SIGKILL) }, 0);
let mut info = unsafe { std::mem::zeroed::<libc::siginfo_t>() };
assert_eq!(
unsafe {
libc::waitid(
libc::P_PID,
descendant as u32,
&mut info,
libc::WEXITED | libc::WNOWAIT,
)
},
0
);
let zombie_result = tree.after_main_exit(&mut child).await;
assert_eq!(
unsafe { libc::waitpid(descendant, std::ptr::null_mut(), 0) },
descendant
);
let error = live_result.expect_err("存活成员缺少 leader 身份时必须拒绝清理");
assert!(error.contains("leader 身份未确认"), "{error}");
zombie_result.expect("已回收 leader 的进程组只剩僵尸时不应要求人工核对");
tree.after_main_exit(&mut child).await.unwrap();
struct DescendantGuard(i32);
impl Drop for DescendantGuard {
fn drop(&mut self) {
unsafe {
libc::kill(self.0, libc::SIGKILL);
libc::waitpid(self.0, std::ptr::null_mut(), 0);
}
}
}
// 正常退出与取消/超时共用的 terminate 都覆盖 leader 已回收的窗口。
for terminate in [false, true] {
let mut command = tokio::process::Command::new("/bin/sh");
command
.args(["-c", "sleep 60 & echo $!; read release"])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.kill_on_drop(true);
command.as_std_mut().process_group(0);
let mut child = command.spawn().unwrap();
let tree = ProjectCommandTree::attach(&child).unwrap();
let mut output = tokio::io::BufReader::new(child.stdout.take().unwrap());
let mut line = String::new();
tokio::io::AsyncBufReadExt::read_line(&mut output, &mut line)
.await
.unwrap();
let descendant: i32 = line.trim().parse().unwrap();
let descendant_guard = DescendantGuard(descendant);
drop(child.stdin.take());
child.wait().await.unwrap();
let ProjectCommandTree::Group { pid, .. } = &tree;
let error = tree
.request_owned_group_termination()
.expect_err("存活成员缺少 leader 身份时必须拒绝发送信号");
assert!(error.contains("leader 身份未确认"), "{error}");
let error = wait_linux_project_command_group_exit(*pid, tokio::time::Instant::now())
.await
.expect_err("持续存活成员必须在预算用尽时失败");
assert!(error.contains("仍有存活成员"), "{error}");
{
let cleanup = async {
if terminate {
tree.terminate(&mut child).await.map(|_| ())
} else {
tree.after_main_exit(&mut child).await
}
};
tokio::pin!(cleanup);
// 先 poll 生产清理,确认它确实等待,才让后代进入僵尸态。
tokio::select! {
biased;
result = &mut cleanup => panic!("后代仍存活时提前结束清理:{result:?}"),
_ = tokio::task::yield_now() => {}
}
assert_eq!(unsafe { libc::kill(descendant, libc::SIGKILL) }, 0);
let mut info = unsafe { std::mem::zeroed::<libc::siginfo_t>() };
assert_eq!(
unsafe {
libc::waitid(
libc::P_PID,
descendant as u32,
&mut info,
libc::WEXITED | libc::WNOWAIT,
)
},
0
);
cleanup.await.expect("leader 消失后,组成员停止应完成清理");
}
tree.after_main_exit(&mut child)
.await
.expect("僵尸不应阻止完成");
wait_linux_project_command_group_exit(*pid, tokio::time::Instant::now())
.await
.expect("无活成员时不消耗等待预算");
drop(descendant_guard);
tree.after_main_exit(&mut child).await.unwrap();
}
}
#[test]
@@ -3426,6 +3521,9 @@ raise SystemExit(code)'
.expect("run timeout test");
assert!(timed_out.timed_out);
assert_eq!(timed_out.status, "failed");
#[cfg(target_os = "linux")]
assert!(timed_out.output.contains("主进程已回收"));
#[cfg(not(target_os = "linux"))]
assert!(timed_out.output.contains("请求终止受控进程组"));
assert!(timed_out.output.contains("不等同完整 OS sandbox"));