修复工具调用坏行读取截断后续记录(P2)

- direct_tool_calls.rs:read_tool_call_lines 由 BufReader::lines() + let Ok(line) = line else { break }
  改成按行读字节(read_until)+ 逐行 from_utf8 解码:非法 UTF-8 行只跳过该行并继续读后续行,
  不再把这一行之后的所有记录一起丢掉;只有读 I/O 错误才停止读取
- 契约「单行损坏跳过该行继续」在"损坏行缺换行、与下一行黏成一行"时也成立:丢掉的只是黏连的那一行
- 新增单测:tool_call_read_skips_invalid_utf8_line(合法行 + 非法字节行 + 合法行 → 读回 2 条;
  以及损坏行缺换行的黏连形态)
This commit is contained in:
2026-09-15 18:21:12 +08:00
parent bc89bf45c3
commit 885f2f10f3
@@ -444,13 +444,25 @@ fn read_tool_call_lines(path: &Path) -> Vec<DirectToolCall> {
let Ok(file) = File::open(path) else {
return Vec::new();
};
let mut reader = BufReader::new(file);
let mut buffer = Vec::new();
let mut calls = Vec::new();
for line in BufReader::new(file).lines() {
let Ok(line) = line else {
break;
};
if let Some(call) = tool_call_from_line(&line) {
calls.push(call);
loop {
buffer.clear();
match reader.read_until(b'\n', &mut buffer) {
Ok(0) => break,
// 单行解码失败(非法 UTF-8)只跳过这一行,继续读后面的行;
// 契约要求「单行损坏跳过该行继续」,不能把后续记录一起丢掉。
Ok(_) => match std::str::from_utf8(&buffer) {
Ok(line) => {
if let Some(call) = tool_call_from_line(line) {
calls.push(call);
}
}
Err(_) => continue,
},
// 读 I/O 错误:无法再定位下一行边界,停止读取(已读到的照常返回)。
Err(_) => break,
}
}
calls
@@ -1164,4 +1176,49 @@ mod tests {
);
}
/// 判据:单行损坏(含非法 UTF-8 字节)只跳过损坏行,后续合法记录必须继续读回。
#[test]
fn tool_call_read_skips_invalid_utf8_line() {
let root = init_tool_call_project("tool-call-invalid-utf8");
let path = tool_calls_path(root.path());
std::fs::create_dir_all(path.parent().expect("parent")).expect("create dir");
// 形态一(契约原文):合法行 + 非法字节行 + 合法行 → 读回 2 条。
let mut bytes = Vec::new();
bytes.extend_from_slice(tool_call_row("item-a", 1000, 1000).as_bytes());
bytes.push(b'\n');
bytes.extend_from_slice(&[0xff, 0xfe, b'\n']);
bytes.extend_from_slice(tool_call_row("item-b", 2000, 2000).as_bytes());
bytes.push(b'\n');
std::fs::write(&path, &bytes).expect("write invalid utf8 fixture");
let calls = read_direct_tool_calls_at(root.path()).expect("read with invalid utf8");
assert_eq!(
calls.len(),
2,
"非法 UTF-8 行只跳过该行,后面的合法记录必须读回"
);
assert_eq!(calls[0].id, "item-a");
assert_eq!(calls[1].id, "item-b");
// 形态二:损坏行缺换行(写入被截断),与紧随其后的记录黏成一行。
// 此时被丢掉的只有黏连的那一行,其后的合法记录必须继续读回。
let mut bytes = Vec::new();
bytes.extend_from_slice(tool_call_row("item-a", 1000, 1000).as_bytes());
bytes.push(b'\n');
bytes.push(0xff);
bytes.extend_from_slice(tool_call_row("item-b", 2000, 2000).as_bytes());
bytes.push(b'\n');
bytes.extend_from_slice(tool_call_row("item-c", 3000, 3000).as_bytes());
bytes.push(b'\n');
std::fs::write(&path, &bytes).expect("write truncated utf8 fixture");
let calls = read_direct_tool_calls_at(root.path()).expect("read with truncated line");
assert_eq!(
calls.len(),
2,
"损坏行缺换行时只丢黏连的那一行,其后的合法记录必须继续读回"
);
assert_eq!(calls[0].id, "item-a");
assert_eq!(calls[1].id, "item-c");
}
}