修复工具调用坏行读取截断后续记录(P2)
- direct_tool_calls.rs:read_tool_call_lines 由 BufReader::lines() + let Ok(line) = line else { break }
改成按行读字节(read_until)+ 逐行 from_utf8 解码:非法 UTF-8 行只跳过该行并继续读后续行,
不再把这一行之后的所有记录一起丢掉;只有读 I/O 错误才停止读取
- 契约「单行损坏跳过该行继续」在"损坏行缺换行、与下一行黏成一行"时也成立:丢掉的只是黏连的那一行
- 新增单测:tool_call_read_skips_invalid_utf8_line(合法行 + 非法字节行 + 合法行 → 读回 2 条;
以及损坏行缺换行的黏连形态)
This commit is contained in:
@@ -444,13 +444,25 @@ fn read_tool_call_lines(path: &Path) -> Vec<DirectToolCall> {
|
||||
let Ok(file) = File::open(path) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut reader = BufReader::new(file);
|
||||
let mut buffer = Vec::new();
|
||||
let mut calls = Vec::new();
|
||||
for line in BufReader::new(file).lines() {
|
||||
let Ok(line) = line else {
|
||||
break;
|
||||
};
|
||||
if let Some(call) = tool_call_from_line(&line) {
|
||||
calls.push(call);
|
||||
loop {
|
||||
buffer.clear();
|
||||
match reader.read_until(b'\n', &mut buffer) {
|
||||
Ok(0) => break,
|
||||
// 单行解码失败(非法 UTF-8)只跳过这一行,继续读后面的行;
|
||||
// 契约要求「单行损坏跳过该行继续」,不能把后续记录一起丢掉。
|
||||
Ok(_) => match std::str::from_utf8(&buffer) {
|
||||
Ok(line) => {
|
||||
if let Some(call) = tool_call_from_line(line) {
|
||||
calls.push(call);
|
||||
}
|
||||
}
|
||||
Err(_) => continue,
|
||||
},
|
||||
// 读 I/O 错误:无法再定位下一行边界,停止读取(已读到的照常返回)。
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
calls
|
||||
@@ -1164,4 +1176,49 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// 判据:单行损坏(含非法 UTF-8 字节)只跳过损坏行,后续合法记录必须继续读回。
|
||||
#[test]
|
||||
fn tool_call_read_skips_invalid_utf8_line() {
|
||||
let root = init_tool_call_project("tool-call-invalid-utf8");
|
||||
let path = tool_calls_path(root.path());
|
||||
std::fs::create_dir_all(path.parent().expect("parent")).expect("create dir");
|
||||
|
||||
// 形态一(契约原文):合法行 + 非法字节行 + 合法行 → 读回 2 条。
|
||||
let mut bytes = Vec::new();
|
||||
bytes.extend_from_slice(tool_call_row("item-a", 1000, 1000).as_bytes());
|
||||
bytes.push(b'\n');
|
||||
bytes.extend_from_slice(&[0xff, 0xfe, b'\n']);
|
||||
bytes.extend_from_slice(tool_call_row("item-b", 2000, 2000).as_bytes());
|
||||
bytes.push(b'\n');
|
||||
std::fs::write(&path, &bytes).expect("write invalid utf8 fixture");
|
||||
let calls = read_direct_tool_calls_at(root.path()).expect("read with invalid utf8");
|
||||
assert_eq!(
|
||||
calls.len(),
|
||||
2,
|
||||
"非法 UTF-8 行只跳过该行,后面的合法记录必须读回"
|
||||
);
|
||||
assert_eq!(calls[0].id, "item-a");
|
||||
assert_eq!(calls[1].id, "item-b");
|
||||
|
||||
// 形态二:损坏行缺换行(写入被截断),与紧随其后的记录黏成一行。
|
||||
// 此时被丢掉的只有黏连的那一行,其后的合法记录必须继续读回。
|
||||
let mut bytes = Vec::new();
|
||||
bytes.extend_from_slice(tool_call_row("item-a", 1000, 1000).as_bytes());
|
||||
bytes.push(b'\n');
|
||||
bytes.push(0xff);
|
||||
bytes.extend_from_slice(tool_call_row("item-b", 2000, 2000).as_bytes());
|
||||
bytes.push(b'\n');
|
||||
bytes.extend_from_slice(tool_call_row("item-c", 3000, 3000).as_bytes());
|
||||
bytes.push(b'\n');
|
||||
std::fs::write(&path, &bytes).expect("write truncated utf8 fixture");
|
||||
let calls = read_direct_tool_calls_at(root.path()).expect("read with truncated line");
|
||||
assert_eq!(
|
||||
calls.len(),
|
||||
2,
|
||||
"损坏行缺换行时只丢黏连的那一行,其后的合法记录必须继续读回"
|
||||
);
|
||||
assert_eq!(calls[0].id, "item-a");
|
||||
assert_eq!(calls[1].id, "item-c");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user