闸门补 leader 失效接管,避免卡死目标永久失败关闭
Project CI / AI game creator shell Rust crates (pull_request) Successful in 1m25s
Project CI / Backend tests (pull_request) Failing after 13s
Project CI / AI game creator shell Rust smoke (pull_request) Successful in 2m6s
Project CI / Frontend tests (pull_request) Successful in 2m10s
Project CI / Repository checks (pull_request) Failing after 13s
Project CI / AI game creator shell web tests (pull_request) Successful in 1m28s
Project CI / Native shell tests (pull_request) Successful in 5m38s
Project CI / AI game creator shell Rust lane 2/2 (pull_request) Successful in 9m9s
Project CI / AI game creator shell Rust lane 1/2 (pull_request) Successful in 9m38s
Project CI / AI game creator shell Rust crates (pull_request) Successful in 1m25s
Project CI / Backend tests (pull_request) Failing after 13s
Project CI / AI game creator shell Rust smoke (pull_request) Successful in 2m6s
Project CI / Frontend tests (pull_request) Successful in 2m10s
Project CI / Repository checks (pull_request) Failing after 13s
Project CI / AI game creator shell web tests (pull_request) Successful in 1m28s
Project CI / Native shell tests (pull_request) Successful in 5m38s
Project CI / AI game creator shell Rust lane 2/2 (pull_request) Successful in 9m9s
Project CI / AI game creator shell Rust lane 1/2 (pull_request) Successful in 9m38s
- acl_repair_gate:策略新增 leader_deadline(默认 5 分钟),超过后新调用接管仍是 running 的 key;Entry 记 started_at + leader_id,被接管后旧 leader 迟到的结果按令牌丢弃,不覆盖接管者写下的结果 - acl_repair_gate:complete()/Drop 改为按令牌就地更新(不再无条件 insert),panic 兜底与接管语义保持一致 - acl_repair_gate:running 状态的 entry 不再写 recorded_at(那时还没有结果),冷却基准只在真正落库时记录 - tests/acl_repair_gate:新增 stale_leader_is_taken_over_and_its_late_result_is_discarded;临时关掉接管分支即红(逆向确认:Reused(Failed(...)) 而非 Executed(Repaired)) - docs:decision-log 记 leader 失效接管决策,pitfalls 把「已知残余边界」改成已兜底说明,验证清单补该用例
This commit is contained in:
@@ -6,6 +6,7 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::hash::Hash;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Condvar, LazyLock, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -31,6 +32,10 @@ pub(crate) struct AclRepairPolicy {
|
||||
pub(crate) denial_cooldown: Duration,
|
||||
pub(crate) failure_cooldown: Duration,
|
||||
pub(crate) wait_timeout: Duration,
|
||||
/// leader 超过这个时长仍未落库即视为卡死,允许新调用接管该 key。
|
||||
/// UAC 弹窗最多被系统挂约两分钟,所以这个上限取得比它宽得多;没有它,
|
||||
/// 一次挂死的 `Start-Process -Wait` 会让这个目标在进程重启前一直失败关闭。
|
||||
pub(crate) leader_deadline: Duration,
|
||||
}
|
||||
|
||||
impl AclRepairPolicy {
|
||||
@@ -53,11 +58,16 @@ struct Entry {
|
||||
running: bool,
|
||||
outcome: Option<AclRepairOutcome>,
|
||||
recorded_at: Option<Instant>,
|
||||
/// leader 起跑时刻,用于判定该 leader 是否已经卡死。
|
||||
started_at: Instant,
|
||||
/// 当前 leader 的令牌:被接管后旧 leader 迟到的结果不得覆盖新 leader 的结果。
|
||||
leader_id: u64,
|
||||
}
|
||||
|
||||
pub(crate) struct AclRepairGate<K> {
|
||||
entries: Mutex<HashMap<K, Entry>>,
|
||||
settled: Condvar,
|
||||
next_leader_id: AtomicU64,
|
||||
}
|
||||
|
||||
impl<K: Clone + Eq + Hash> AclRepairGate<K> {
|
||||
@@ -65,6 +75,7 @@ impl<K: Clone + Eq + Hash> AclRepairGate<K> {
|
||||
Self {
|
||||
entries: Mutex::new(HashMap::new()),
|
||||
settled: Condvar::new(),
|
||||
next_leader_id: AtomicU64::new(1),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,6 +96,11 @@ impl<K: Clone + Eq + Hash> AclRepairGate<K> {
|
||||
loop {
|
||||
match entries.get(&key) {
|
||||
Some(entry) if entry.running => {
|
||||
// 卡死的 leader(例如 `Start-Process -Wait` 真挂住)不能永久占住这个 key:
|
||||
// 超过 leader_deadline 就由新调用接管,否则该目标在进程重启前只会一直失败关闭。
|
||||
if now.saturating_duration_since(entry.started_at) >= policy.leader_deadline {
|
||||
break;
|
||||
}
|
||||
let remaining = wait_deadline.saturating_duration_since(Instant::now());
|
||||
if remaining.is_zero() {
|
||||
return AclRepairGateResult::WaitTimedOut;
|
||||
@@ -112,12 +128,16 @@ impl<K: Clone + Eq + Hash> AclRepairGate<K> {
|
||||
}
|
||||
|
||||
prune(&mut entries, now, policy);
|
||||
let leader_id = self.next_leader_id.fetch_add(1, Ordering::Relaxed);
|
||||
entries.insert(
|
||||
key.clone(),
|
||||
Entry {
|
||||
running: true,
|
||||
outcome: None,
|
||||
recorded_at: Some(now),
|
||||
// 结果尚未落库:冷却基准只在真正记录结果时才写。
|
||||
recorded_at: None,
|
||||
started_at: now,
|
||||
leader_id,
|
||||
},
|
||||
);
|
||||
drop(entries);
|
||||
@@ -125,6 +145,7 @@ impl<K: Clone + Eq + Hash> AclRepairGate<K> {
|
||||
let guard = LeaderGuard {
|
||||
gate: self,
|
||||
key: key.clone(),
|
||||
leader_id,
|
||||
armed: true,
|
||||
};
|
||||
let outcome = execute();
|
||||
@@ -161,6 +182,7 @@ where
|
||||
struct LeaderGuard<'a, K: Clone + Eq + Hash> {
|
||||
gate: &'a AclRepairGate<K>,
|
||||
key: K,
|
||||
leader_id: u64,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
@@ -168,17 +190,18 @@ impl<K: Clone + Eq + Hash> LeaderGuard<'_, K> {
|
||||
fn complete(mut self, outcome: AclRepairOutcome) -> AclRepairGateResult {
|
||||
self.armed = false;
|
||||
let mut entries = lock(&self.gate.entries);
|
||||
entries.insert(
|
||||
self.key.clone(),
|
||||
Entry {
|
||||
running: false,
|
||||
outcome: Some(outcome.clone()),
|
||||
// 只在仍是当前 leader 时落库:leader 卡死被接管后,迟到的结果必须丢弃,
|
||||
// 否则会把接管者已经写下的结果覆盖回去。
|
||||
if let Some(entry) = entries.get_mut(&self.key) {
|
||||
if entry.leader_id == self.leader_id {
|
||||
entry.running = false;
|
||||
entry.outcome = Some(outcome.clone());
|
||||
// 冷却从「结果落库」时刻算起,而不是 leader 起跑时刻:UAC 弹窗可能被挂着
|
||||
// 几十秒到两分钟,用起跑时刻会让 120s 拒绝冷却在用户应答前就过期,
|
||||
// 紧接着的自动重查会立刻再弹一次。
|
||||
recorded_at: Some(Instant::now()),
|
||||
},
|
||||
);
|
||||
entry.recorded_at = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
drop(entries);
|
||||
self.gate.settled.notify_all();
|
||||
AclRepairGateResult::Executed(outcome)
|
||||
@@ -192,16 +215,15 @@ impl<K: Clone + Eq + Hash> Drop for LeaderGuard<'_, K> {
|
||||
return;
|
||||
}
|
||||
let mut entries = lock(&self.gate.entries);
|
||||
entries.insert(
|
||||
self.key.clone(),
|
||||
Entry {
|
||||
running: false,
|
||||
outcome: Some(AclRepairOutcome::Failed(
|
||||
if let Some(entry) = entries.get_mut(&self.key) {
|
||||
if entry.leader_id == self.leader_id {
|
||||
entry.running = false;
|
||||
entry.outcome = Some(AclRepairOutcome::Failed(
|
||||
"AGC ACL 提权修复执行线程异常退出".to_string(),
|
||||
)),
|
||||
recorded_at: Some(Instant::now()),
|
||||
},
|
||||
);
|
||||
));
|
||||
entry.recorded_at = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
drop(entries);
|
||||
self.gate.settled.notify_all();
|
||||
}
|
||||
@@ -237,6 +259,8 @@ pub(crate) const ACL_REPAIR_POLICY: AclRepairPolicy = AclRepairPolicy {
|
||||
denial_cooldown: Duration::from_secs(120),
|
||||
failure_cooldown: Duration::from_secs(15),
|
||||
wait_timeout: Duration::from_secs(60),
|
||||
// 系统对无人应答的 UAC 弹窗约 2 分钟超时,取 5 分钟只兜「真挂死」这一种情况。
|
||||
leader_deadline: Duration::from_secs(300),
|
||||
};
|
||||
|
||||
/// 用户主动操作(打开/新建项目、重命名刷新)后调用:解除「被拒绝」记忆。
|
||||
|
||||
@@ -12,6 +12,7 @@ fn test_policy() -> AclRepairPolicy {
|
||||
denial_cooldown: Duration::from_secs(300),
|
||||
failure_cooldown: Duration::from_secs(15),
|
||||
wait_timeout: Duration::from_secs(5),
|
||||
leader_deadline: Duration::from_secs(300),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,6 +246,50 @@ fn repair_gate_key_merges_path_spelling_variants_of_one_target() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_leader_is_taken_over_and_its_late_result_is_discarded() {
|
||||
// 真机场景:`Start-Process -Wait` 挂死时,follower 等到 60s 只会失败关闭,
|
||||
// 而这个 key 会被永久占住(clear_denials 也不清理 running)——只能重启客户端。
|
||||
// 超过 leader_deadline 必须允许接管,且旧 leader 迟到的结果不得覆盖接管者。
|
||||
let gate = Arc::new(AclRepairGate::new());
|
||||
let key = test_key("c:\\stale-leader");
|
||||
let policy = AclRepairPolicy {
|
||||
leader_deadline: Duration::from_millis(150),
|
||||
..test_policy()
|
||||
};
|
||||
let started_at = Instant::now();
|
||||
let slow = {
|
||||
let gate = Arc::clone(&gate);
|
||||
let key = key.clone();
|
||||
std::thread::spawn(move || {
|
||||
gate.run(key, started_at, &policy, || {
|
||||
std::thread::sleep(Duration::from_millis(400));
|
||||
AclRepairOutcome::Failed("卡死的 leader 迟到落库".to_string())
|
||||
})
|
||||
})
|
||||
};
|
||||
|
||||
std::thread::sleep(Duration::from_millis(250));
|
||||
let taken_over = gate.run(key.clone(), Instant::now(), &policy, || {
|
||||
AclRepairOutcome::Repaired
|
||||
});
|
||||
assert_eq!(
|
||||
taken_over,
|
||||
AclRepairGateResult::Executed(AclRepairOutcome::Repaired)
|
||||
);
|
||||
|
||||
assert!(matches!(
|
||||
slow.join().expect("leader 线程不得 panic"),
|
||||
AclRepairGateResult::Executed(AclRepairOutcome::Failed(_))
|
||||
));
|
||||
assert_eq!(
|
||||
gate.run(key, Instant::now(), &policy, || {
|
||||
panic!("冷却内必须复用接管者的结果,不得再执行")
|
||||
}),
|
||||
AclRepairGateResult::Reused(AclRepairOutcome::Repaired)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clearing_denials_allows_an_explicit_user_retry() {
|
||||
let gate = AclRepairGate::new();
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
- 决策:新增进程级闸门 `acl_repair_gate`,key = `(规范化 repair target, scope)`。并发调用只允许一次真实提权,其余等待并复用**同一结果**;结果在冷却窗口内直接复用(成功 30s / 失败 15s / 用户取消 120s),等待窗口 60s 超时按失败关闭。leader 异常退出由 RAII 兜底记为失败并唤醒全部等待者,避免等待者被永久挂住。
|
||||
- 决策补充(key 归一化):key 的路径半边经 `windows_acl_repair_gate_key` 归一化——去掉 `\\?\` / `\\?\UNC\` 前缀并统一小写。最近项目列表里同一项目实测同时存在 `\\?\C:\...` 与 `C:\...` 两种写法(客户端 localStorage 实测),不归一化就是两个 key,同一个目录仍会弹两次 UAC。这里刻意只做前缀与大小写归一而不 `canonicalize`:待修复目标恰恰是「读不动的目录」,解析不可靠。
|
||||
- 决策补充(冷却基准):冷却从**结果落库**时刻算起,不是 leader 起跑时刻。UAC 弹窗会被挂着几十秒到两分钟,用起跑时刻会让 120s 拒绝冷却在用户应答前就过期,前端 15s/45s/120s 的整表重查紧跟着再弹一次。
|
||||
- 决策补充(leader 失效接管):`leader_deadline`(默认 5 分钟)之后,新调用可以接管仍是 `running` 的 key;每个 leader 带令牌,被接管后旧 leader 迟到的结果直接丢弃,不会覆盖接管者的结果。真机上无人应答的 UAC 约 2 分钟自然超时,所以这个上限只兜「提权子进程真挂死」——否则该目标会永久按失败关闭(`clear_denials` 不清理 running,只能重启客户端)。
|
||||
- 错误类型化:用户取消 UAC 的错误统一带稳定标记 `AGC_ACL_ELEVATION_DENIED`,前端据此判定「不可自动重试」,不再依赖中文文案匹配。
|
||||
- 用户主动操作(打开/新建项目、文件选择器选择目录、重命名刷新)会调用 `clear_game_creator_acl_elevation_denials` 清除拒绝记忆,保证显式重试仍能再次请求提权。前端唯一入口是 `features/app-shell/aclElevation.ts` 的 `clearAclElevationDenials()`:最近项目 hook(`rememberRecentWorkspace` / `refreshRecentWorkspace`)与打开/新建链路(`useHomeProjectCreation.openProject`,覆盖行内打开与 picker)共用它;漏挂入口会让用户「点了打开立即失败、也不问授权」。
|
||||
- 未做:给提权子进程加有界等待(`Start-Process -Wait` 目前无超时)。理由:中断挂起的 UAC 流程比等待更糟,single-flight 已把并发弹窗收成一个,follower 的等待由 60s 窗口兜底。
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
- **原因**:`windows_acl_repair_target`(`src-tauri/src/config.rs`)对 Managed 作用域返回「第一个读取被拒的祖先」——同一祖先下的多个项目解析到**同一个** repair target;而唯一的去重是单次调用内的局部 `attempted_targets`,跨调用、跨线程都没有记忆。启动页一次并发检查 ≤8 个最近项目,就会并发启动同样多次 `powershell -Verb RunAs`。
|
||||
- **处理**:进程级 single-flight(key = `(规范化 repair target, scope)`)+ 结果冷却(成功 30s / 失败 15s / 用户取消 120s)+ 等待窗口 60s 超时按失败关闭;leader 异常退出由 RAII 兜底唤醒等待者。用户取消带稳定标记 `AGC_ACL_ELEVATION_DENIED`,前端据此不自动重试;用户主动操作会清除拒绝记忆。
|
||||
- **不要踩的坑**:① 闸门 key 必须归一化 `\\?\` / `\\?\UNC\` 前缀——最近项目列表里同一项目实测同时存在 `\\?\C:\...` 与 `C:\...` 两种写法,按原始字符串做 key 会让同一个目录弹两次 UAC(`windows_acl_repair_gate_key`);② 冷却必须从**结果落库**时刻算起,用 leader 起跑时刻会让 120s 拒绝冷却在 UAC 被挂着两分钟时提前过期,紧接着的自动重查立刻再弹一次;③ 复现「多个项目共用同一 target」时,DENY 要写在祖先的**父目录**上靠继承落入祖先——`icacls` 直接加在容器自身实测只影响子项(容器自身 `GetFileAttributes` 仍成功),target 会退化成每个项目自己,repro 不出并发弹窗;④ 夹具路径必须落在 `game_creator_private_path_allows_auto_elevation` 放行范围内(runtime config dir / `.config/genarrative` / 打包 AppData / 带 `.agent/manifest.json` 的项目根),因为提权子进程会按 **repair target** 再校验一次 `scope.allows_path`,否则失败关闭。
|
||||
- **验证**:`src-tauri/src/tests/acl_repair_gate.rs`(并发只执行一次、冷却复用、拒绝冷却、清除后可重试、follower 超时、leader panic 唤醒等待者、冷却基准、路径写法归一)。真机复现(无需提权交互即可计数):在 Managed 放行范围内建 8 个带 `.agent/manifest.json` 的假项目 → 对共同祖先的**父目录** `icacls <父目录> /deny *<sid>:(OI)(CI)(RX)` → 挂载启动页,同时数 `powershell.exe` 里命令行带 `RunAs` 的进程数(`Start-Process -Wait` 会让它一直存活到用户应答)与 `consent.exe` 峰值:修复前 8 个并发请求,修复后 1 个;把同一目录的 `\\?\C:\...` 与 `C:\...` 两种写法一起塞进最近项目,还能验证 key 归一化是否生效(修复前 2 个、修复后 1 个)。
|
||||
- **已知残余边界**:闸门只有 follower 的有界等待(60s),没有 leader 失效接管——若提权子进程真的挂死(`Start-Process -Wait` 无超时),该 key 会一直 `running`,之后所有同目标调用都按 60s 超时失败,`clear_game_creator_acl_elevation_denials` 也不清理 running,只能重启客户端恢复。需要更激进策略时再单独讨论(记 `started_at` + 硬上限接管)。
|
||||
- **验证**:`src-tauri/src/tests/acl_repair_gate.rs`(并发只执行一次、冷却复用、拒绝冷却、清除后可重试、follower 超时、leader panic 唤醒等待者、冷却基准、路径写法归一、leader 卡死接管与迟到结果丢弃)。真机复现(无需提权交互即可计数):在 Managed 放行范围内建 8 个带 `.agent/manifest.json` 的假项目 → 对共同祖先的**父目录** `icacls <父目录> /deny *<sid>:(OI)(CI)(RX)` → 挂载启动页,同时数 `powershell.exe` 里命令行带 `RunAs` 的进程数(`Start-Process -Wait` 会让它一直存活到用户应答)与 `consent.exe` 峰值:修复前 8 个并发请求,修复后 1 个;把同一目录的 `\\?\C:\...` 与 `C:\...` 两种写法一起塞进最近项目,还能验证 key 归一化是否生效(修复前 2 个、修复后 1 个)。
|
||||
- **leader 卡死的兜底**:闸门只有 follower 的有界等待(60s),若提权子进程真的挂死(`Start-Process -Wait` 无超时),`leader_deadline`(5 分钟)之前该 key 一直被占住,之后新调用会接管并按新 leader 执行;被接管后旧 leader 迟到的结果按令牌丢弃,不会覆盖接管者。`clear_game_creator_acl_elevation_denials` 只清「被拒绝」记忆,不清理 running。
|
||||
- **关联**:`src-tauri/src/acl_repair_gate.rs`、`src-tauri/src/config.rs`、issue #498。
|
||||
|
||||
## 2026-09-24 对话过程卡的读秒退回 1 秒一跳:刷新粒度必须与显示精度同格
|
||||
|
||||
Reference in New Issue
Block a user