修复 Linux 沙箱命令退出回收的偶发竞态
Project CI / AI game creator shell Rust crates (pull_request) Successful in 1m33s
Project CI / AI game creator shell Rust smoke (pull_request) Successful in 2m7s
Project CI / Backend tests (pull_request) Successful in 3m49s
Project CI / Frontend tests (pull_request) Successful in 1m53s
Project CI / Native shell tests (pull_request) Successful in 5m51s
Project CI / AI game creator shell Rust lane 2/2 (pull_request) Successful in 8m57s
Project CI / Repository checks (pull_request) Successful in 1m55s
Project CI / AI game creator shell Rust lane 1/2 (pull_request) Successful in 9m19s
Project CI / AI game creator shell web tests (pull_request) Successful in 1m24s
Project CI / AI game creator shell Rust crates (push) Successful in 1m30s
Project CI / AI game creator shell Rust smoke (push) Successful in 2m9s
Project CI / Backend tests (push) Successful in 4m49s
Project CI / Frontend tests (push) Successful in 2m9s
Project CI / Native shell tests (push) Successful in 6m54s
Project CI / AI game creator shell Rust lane 2/2 (push) Successful in 9m56s
Project CI / AI game creator shell Rust lane 1/2 (push) Successful in 10m14s
Project CI / Repository checks (push) Successful in 2m2s
Project CI / AI game creator shell web tests (push) Successful in 1m30s
Project CI / AI game creator shell Rust crates (pull_request) Successful in 1m33s
Project CI / AI game creator shell Rust smoke (pull_request) Successful in 2m7s
Project CI / Backend tests (pull_request) Successful in 3m49s
Project CI / Frontend tests (pull_request) Successful in 1m53s
Project CI / Native shell tests (pull_request) Successful in 5m51s
Project CI / AI game creator shell Rust lane 2/2 (pull_request) Successful in 8m57s
Project CI / Repository checks (pull_request) Successful in 1m55s
Project CI / AI game creator shell Rust lane 1/2 (pull_request) Successful in 9m19s
Project CI / AI game creator shell web tests (pull_request) Successful in 1m24s
Project CI / AI game creator shell Rust crates (push) Successful in 1m30s
Project CI / AI game creator shell Rust smoke (push) Successful in 2m9s
Project CI / Backend tests (push) Successful in 4m49s
Project CI / Frontend tests (push) Successful in 2m9s
Project CI / Native shell tests (push) Successful in 6m54s
Project CI / AI game creator shell Rust lane 2/2 (push) Successful in 9m56s
Project CI / AI game creator shell Rust lane 1/2 (push) Successful in 10m14s
Project CI / Repository checks (push) Successful in 2m2s
Project CI / AI game creator shell web tests (push) Successful in 1m30s
统一正常退出、取消和超时的有界进程组退出确认,保留归属校验 提前记录启动身份,修补终态协议错误的清理及输出任务回收 补充确定性竞态回归并同步运行时规范和排障记忆 保持现有 CI 并行、分片和重试策略不变
This commit was merged in pull request #517.
This commit is contained in:
@@ -15,6 +15,8 @@ const PROJECT_COMMAND_MAX_ARGUMENT_BYTES: usize = 8 * 1024;
|
||||
const PROJECT_COMMAND_MIN_TIMEOUT_SECONDS: u64 = 1;
|
||||
const PROJECT_COMMAND_MAX_TIMEOUT_SECONDS: u64 = 300;
|
||||
const PROJECT_COMMAND_OUTPUT_MAX_BYTES: usize = 24 * 1024;
|
||||
#[cfg(target_os = "linux")]
|
||||
const PROJECT_COMMAND_CLEANUP_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const PROJECT_COMMAND_FINGERPRINT_MAX_ENTRIES: usize = 20_000;
|
||||
const PROJECT_COMMAND_FINGERPRINT_MAX_FILES: usize = 10_000;
|
||||
const PROJECT_COMMAND_FINGERPRINT_MAX_BYTES: u64 = 512 * 1024 * 1024;
|
||||
@@ -222,14 +224,30 @@ impl ProjectCommandTree {
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
|
||||
let requested = self.request_owned_group_termination();
|
||||
let _ = child.start_kill();
|
||||
let waited = tokio::time::timeout(Duration::from_secs(5), child.wait()).await;
|
||||
requested?;
|
||||
let waited = tokio::time::timeout_at(deadline, child.wait()).await;
|
||||
waited
|
||||
.map_err(|_| "等待受控命令主进程退出超时")?
|
||||
.map_err(|_| "受控命令主进程退出未确认")?;
|
||||
Ok("已请求终止受控进程组并回收主进程,完整子树状态未证明".into())
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let Self::Group { pid, .. } = self;
|
||||
// leader 可能与取消同时退出;只要组已停止便无需补发信号。
|
||||
if let Err(error) = wait_linux_project_command_group_exit(*pid, deadline).await {
|
||||
return Err(match requested {
|
||||
Ok(_) => error,
|
||||
Err(request_error) => format!("{request_error};{error}"),
|
||||
});
|
||||
}
|
||||
return Ok("主进程已回收,受控进程组已无活成员;完整子树状态未证明".into());
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
requested?;
|
||||
Ok("已请求终止受控进程组并回收主进程,完整子树状态未证明".into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -244,17 +262,39 @@ impl ProjectCommandTree {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let Self::Group { pid, .. } = self;
|
||||
// 容器 PID 1 可能不回收 bwrap 的孤儿僵尸;它们不再执行,也无法被信号终止。
|
||||
// 仅在确认没有存活成员时免除清理,存活成员仍须通过 leader 身份核对。
|
||||
if !linux_project_command_group_has_live_members(*pid)? {
|
||||
return Ok(());
|
||||
}
|
||||
// wait 已回收 leader,不能再用旧 PID 授权发信号。
|
||||
// namespace 后代可能仍在退出,容器 PID 1 也可能保留孤儿僵尸。
|
||||
return wait_linux_project_command_group_exit(
|
||||
*pid,
|
||||
tokio::time::Instant::now() + PROJECT_COMMAND_CLEANUP_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
self.request_owned_group_termination().map(|_| ())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
async fn wait_linux_project_command_group_exit(
|
||||
group: u32,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<(), String> {
|
||||
while linux_project_command_group_has_live_members(group)? {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(
|
||||
"受控进程组仍有存活成员,退出未确认;不得向身份未确认的进程组补发信号".into(),
|
||||
);
|
||||
}
|
||||
tokio::time::sleep_until(
|
||||
deadline.min(tokio::time::Instant::now() + Duration::from_millis(20)),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn linux_project_command_group_has_live_members(group: u32) -> Result<bool, String> {
|
||||
let inspect = || -> std::io::Result<bool> {
|
||||
@@ -1842,6 +1882,17 @@ where
|
||||
),
|
||||
));
|
||||
}
|
||||
// 目标放行后可能立即退出,必须趁 ready gate 仍持有 launcher 时记录归属。
|
||||
let tree = match ProjectCommandTree::attach(&child) {
|
||||
Ok(tree) => tree,
|
||||
Err(error) => {
|
||||
let termination = terminate_project_command_process_group(&mut child).await;
|
||||
return Err(ProjectCommandError::new(
|
||||
ProjectCommandErrorStage::Preflight,
|
||||
project_command_launch_error_with_termination(error, termination),
|
||||
));
|
||||
}
|
||||
};
|
||||
if let Err(error) = durable_commit() {
|
||||
let termination = terminate_project_command_process_group(&mut child).await;
|
||||
return Err(ProjectCommandError::new(
|
||||
@@ -1860,12 +1911,7 @@ where
|
||||
// cancelled future must not erase the launch-unknown decision window.
|
||||
let exec = gate.wait_target_exec(Duration::from_secs(3));
|
||||
match exec {
|
||||
Ok(TargetExecState::Established) => {
|
||||
let tree = ProjectCommandTree::attach(&child).map_err(|error| {
|
||||
ProjectCommandError::new(ProjectCommandErrorStage::LaunchUnknown, error)
|
||||
})?;
|
||||
Ok(EstablishedProjectCommand { tree, child, gate })
|
||||
}
|
||||
Ok(TargetExecState::Established) => Ok(EstablishedProjectCommand { tree, child, gate }),
|
||||
Ok(TargetExecState::Failed { errno }) => {
|
||||
let termination = terminate_project_command_process_group_after_commit(&mut child);
|
||||
Err(ProjectCommandError::new(
|
||||
@@ -1977,7 +2023,11 @@ async fn terminate_project_command_process_group(
|
||||
.ok_or_else(|| "请求终止受控进程组失败:子进程缺少 pid".to_string())?;
|
||||
let group_result = request_unix_project_command_process_group_termination(process_id);
|
||||
let child_kill_error = child.start_kill().err();
|
||||
let wait_result = child.wait().await;
|
||||
let deadline = tokio::time::Instant::now() + PROJECT_COMMAND_CLEANUP_TIMEOUT;
|
||||
let wait_result = tokio::time::timeout_at(deadline, child.wait())
|
||||
.await
|
||||
.map_err(|_| "等待受控命令主进程退出超时".to_string())?
|
||||
.map_err(|error| error.to_string());
|
||||
if let Err(error) = &group_result {
|
||||
let fallback = match (&child_kill_error, &wait_result) {
|
||||
(_, Ok(_)) => "主进程已回收,但无法确认其余组内进程".to_string(),
|
||||
@@ -1989,6 +2039,7 @@ async fn terminate_project_command_process_group(
|
||||
return Err(format!("{error};{fallback}"));
|
||||
}
|
||||
wait_result.map_err(|error| format!("请求终止受控进程组后等待主进程失败:{error}"))?;
|
||||
wait_linux_project_command_group_exit(process_id, deadline).await?;
|
||||
Ok(format!(
|
||||
"{}并完成主进程回收",
|
||||
group_result.expect("group termination result checked")
|
||||
@@ -2220,7 +2271,7 @@ where
|
||||
let (exit_code, timed_out, termination_summary) = match wait {
|
||||
ProjectCommandWait::Exited(Ok(status)) => {
|
||||
#[cfg(target_os = "linux")]
|
||||
let _terminal = wait_established_project_command_terminal(gate).await?;
|
||||
let terminal = wait_established_project_command_terminal(gate).await;
|
||||
if let Err(error) = tree.after_main_exit(&mut child).await {
|
||||
stdout_task.abort();
|
||||
stderr_task.abort();
|
||||
@@ -2229,6 +2280,12 @@ where
|
||||
format!("command.exec 主进程退出后进程树未确认回收,需要人工核对:{error}"),
|
||||
));
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Err(error) = terminal {
|
||||
stdout_task.abort();
|
||||
stderr_task.abort();
|
||||
return Err(error);
|
||||
}
|
||||
(status.code(), false, None)
|
||||
}
|
||||
ProjectCommandWait::Exited(Err(error)) => {
|
||||
@@ -2541,45 +2598,83 @@ mod tests {
|
||||
return;
|
||||
}
|
||||
assert_eq!(unsafe { libc::prctl(libc::PR_SET_CHILD_SUBREAPER, 1) }, 0);
|
||||
let mut command = tokio::process::Command::new("/bin/sh");
|
||||
command
|
||||
.args(["-c", "sleep 60 & echo $!; read release"])
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped());
|
||||
command.as_std_mut().process_group(0);
|
||||
let mut child = command.spawn().unwrap();
|
||||
let tree = ProjectCommandTree::attach(&child).unwrap();
|
||||
let mut output = tokio::io::BufReader::new(child.stdout.take().unwrap());
|
||||
let mut line = String::new();
|
||||
tokio::io::AsyncBufReadExt::read_line(&mut output, &mut line)
|
||||
.await
|
||||
.unwrap();
|
||||
let descendant: i32 = line.trim().parse().unwrap();
|
||||
drop(child.stdin.take());
|
||||
child.wait().await.unwrap();
|
||||
let live_result = tree.after_main_exit(&mut child).await;
|
||||
assert_eq!(unsafe { libc::kill(descendant, libc::SIGKILL) }, 0);
|
||||
let mut info = unsafe { std::mem::zeroed::<libc::siginfo_t>() };
|
||||
assert_eq!(
|
||||
unsafe {
|
||||
libc::waitid(
|
||||
libc::P_PID,
|
||||
descendant as u32,
|
||||
&mut info,
|
||||
libc::WEXITED | libc::WNOWAIT,
|
||||
)
|
||||
},
|
||||
0
|
||||
);
|
||||
let zombie_result = tree.after_main_exit(&mut child).await;
|
||||
assert_eq!(
|
||||
unsafe { libc::waitpid(descendant, std::ptr::null_mut(), 0) },
|
||||
descendant
|
||||
);
|
||||
let error = live_result.expect_err("存活成员缺少 leader 身份时必须拒绝清理");
|
||||
assert!(error.contains("leader 身份未确认"), "{error}");
|
||||
zombie_result.expect("已回收 leader 的进程组只剩僵尸时不应要求人工核对");
|
||||
tree.after_main_exit(&mut child).await.unwrap();
|
||||
struct DescendantGuard(i32);
|
||||
impl Drop for DescendantGuard {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
libc::kill(self.0, libc::SIGKILL);
|
||||
libc::waitpid(self.0, std::ptr::null_mut(), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 正常退出与取消/超时共用的 terminate 都覆盖 leader 已回收的窗口。
|
||||
for terminate in [false, true] {
|
||||
let mut command = tokio::process::Command::new("/bin/sh");
|
||||
command
|
||||
.args(["-c", "sleep 60 & echo $!; read release"])
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
command.as_std_mut().process_group(0);
|
||||
let mut child = command.spawn().unwrap();
|
||||
let tree = ProjectCommandTree::attach(&child).unwrap();
|
||||
let mut output = tokio::io::BufReader::new(child.stdout.take().unwrap());
|
||||
let mut line = String::new();
|
||||
tokio::io::AsyncBufReadExt::read_line(&mut output, &mut line)
|
||||
.await
|
||||
.unwrap();
|
||||
let descendant: i32 = line.trim().parse().unwrap();
|
||||
let descendant_guard = DescendantGuard(descendant);
|
||||
drop(child.stdin.take());
|
||||
child.wait().await.unwrap();
|
||||
let ProjectCommandTree::Group { pid, .. } = &tree;
|
||||
let error = tree
|
||||
.request_owned_group_termination()
|
||||
.expect_err("存活成员缺少 leader 身份时必须拒绝发送信号");
|
||||
assert!(error.contains("leader 身份未确认"), "{error}");
|
||||
let error = wait_linux_project_command_group_exit(*pid, tokio::time::Instant::now())
|
||||
.await
|
||||
.expect_err("持续存活成员必须在预算用尽时失败");
|
||||
assert!(error.contains("仍有存活成员"), "{error}");
|
||||
{
|
||||
let cleanup = async {
|
||||
if terminate {
|
||||
tree.terminate(&mut child).await.map(|_| ())
|
||||
} else {
|
||||
tree.after_main_exit(&mut child).await
|
||||
}
|
||||
};
|
||||
tokio::pin!(cleanup);
|
||||
// 先 poll 生产清理,确认它确实等待,才让后代进入僵尸态。
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut cleanup => panic!("后代仍存活时提前结束清理:{result:?}"),
|
||||
_ = tokio::task::yield_now() => {}
|
||||
}
|
||||
assert_eq!(unsafe { libc::kill(descendant, libc::SIGKILL) }, 0);
|
||||
let mut info = unsafe { std::mem::zeroed::<libc::siginfo_t>() };
|
||||
assert_eq!(
|
||||
unsafe {
|
||||
libc::waitid(
|
||||
libc::P_PID,
|
||||
descendant as u32,
|
||||
&mut info,
|
||||
libc::WEXITED | libc::WNOWAIT,
|
||||
)
|
||||
},
|
||||
0
|
||||
);
|
||||
cleanup.await.expect("leader 消失后,组成员停止应完成清理");
|
||||
}
|
||||
tree.after_main_exit(&mut child)
|
||||
.await
|
||||
.expect("僵尸不应阻止完成");
|
||||
wait_linux_project_command_group_exit(*pid, tokio::time::Instant::now())
|
||||
.await
|
||||
.expect("无活成员时不消耗等待预算");
|
||||
drop(descendant_guard);
|
||||
tree.after_main_exit(&mut child).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3426,6 +3521,9 @@ raise SystemExit(code)'
|
||||
.expect("run timeout test");
|
||||
assert!(timed_out.timed_out);
|
||||
assert_eq!(timed_out.status, "failed");
|
||||
#[cfg(target_os = "linux")]
|
||||
assert!(timed_out.output.contains("主进程已回收"));
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
assert!(timed_out.output.contains("请求终止受控进程组"));
|
||||
assert!(timed_out.output.contains("不等同完整 OS sandbox"));
|
||||
|
||||
|
||||
Reference in New Issue
Block a user