DirectProject 读侧白名单兼容格式切换前的旧行,存量项目不再全部读不动
- direct_project_history.rs:两处行信封判定收敛到 direct_project_history_item_from_parsed_line,response_item 与白名单化的 legacy 行走同一条判定路径 - 新增 direct_project_legacy_row:只认 schemaVersion=game-creator-conversation.v1、无 type、role 落在该写入器自己的角色集合(user/assistant/tool)内、content 为非空字符串的行,其余形状继续按损坏失败关闭 - user/assistant 旧行投影成与 direct_project_local_message_item 同形状的 message item:role 与 content 逐字节保留、不 trim、不改写、未知字段忽略,有 messageId 才带 id - tool 旧行已识别但不进 Codex 上下文:它不是 Responses item,无法还原成真正的工具 item,混入会造出假的工具消息;聊天投影本来也只展示 user/assistant - 抽出 direct_project_message_item 供本地补写与 legacy 投影共用,保证两种来源的 item 形状一致 - 补断言:旧行投影(role/content 逐字节、无 messageId 时无 id)、旧行与新行交替的混合文件按行顺序读取、tool 行被识别且不进上下文、非白名单异常行(role=system/developer、content 非字符串或为空、缺 role、缺 payload、坏 JSON)仍失败关闭、通用对话写入器产出的旧行形状落在白名单内 - 变异验证:去掉 legacy 兼容分支→投影/混合文件/通用写入器形状 3 个用例变红;把白名单放宽成「任意行都接受」→失败关闭用例变红
This commit is contained in:
@@ -9,6 +9,9 @@ use std::io::{BufRead, BufReader, Read, Seek, SeekFrom};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const DIRECT_PROJECT_HISTORY_RECORD_TYPE: &str = "response_item";
|
||||
/// 格式切换到 `response_item`(#282)之前,DirectProject 主对话通过通用对话写入器
|
||||
/// 落到同一份 `project.jsonl`,行形状是 `PersistedLocalConversationMessageRecord`。
|
||||
const DIRECT_PROJECT_HISTORY_LEGACY_SCHEMA_VERSION: &str = "game-creator-conversation.v1";
|
||||
const DIRECT_PROJECT_INTERNAL_CONTEXT_KINDS: &[&str] = &[
|
||||
"host_skills.instructions",
|
||||
"permissions.instructions",
|
||||
@@ -159,22 +162,90 @@ fn direct_project_history_item_from_line(
|
||||
}
|
||||
let parsed: Value = serde_json::from_slice(line)
|
||||
.map_err(|error| format!("解析 DirectProject 历史失败:{}: {error}", path.display()))?;
|
||||
if parsed.get("type").and_then(Value::as_str) != Some(DIRECT_PROJECT_HISTORY_RECORD_TYPE) {
|
||||
return Err(format!(
|
||||
"DirectProject 历史记录类型无效:{}",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
let item = parsed
|
||||
.get("payload")
|
||||
.cloned()
|
||||
.ok_or_else(|| format!("DirectProject 历史记录缺少 payload:{}", path.display()))?;
|
||||
let Some(item) = direct_project_history_item_from_parsed_line(path, &parsed)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
if is_direct_project_internal_context_item(&item) {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(item))
|
||||
}
|
||||
|
||||
/// 只接受两种行信封:`response_item`,以及白名单化的 legacy 行。返回 `None` 表示行已
|
||||
/// 被识别、但不该进入 Codex 上下文(legacy 的 `tool` 行)。
|
||||
/// 其余任何形状(含换了 `schemaVersion`、带 `type` 却不是 `response_item`、role 不在
|
||||
/// legacy 写入器自己的角色集合内、content 不是非空字符串)都判定为损坏并失败关闭。
|
||||
fn direct_project_history_item_from_parsed_line(
|
||||
path: &Path,
|
||||
parsed: &Value,
|
||||
) -> Result<Option<Value>, String> {
|
||||
if parsed.get("type").and_then(Value::as_str) == Some(DIRECT_PROJECT_HISTORY_RECORD_TYPE) {
|
||||
return parsed
|
||||
.get("payload")
|
||||
.cloned()
|
||||
.map(Some)
|
||||
.ok_or_else(|| format!("DirectProject 历史记录缺少 payload:{}", path.display()));
|
||||
}
|
||||
direct_project_legacy_row(parsed)
|
||||
.map(|row| row.message_item())
|
||||
.ok_or_else(|| format!("DirectProject 历史记录类型无效:{}", path.display()))
|
||||
}
|
||||
|
||||
/// 白名单化 legacy 行的投影结果。
|
||||
enum DirectProjectLegacyRow {
|
||||
/// user/assistant 行:投影成 message item 进入历史。
|
||||
Message(Value),
|
||||
/// 已识别但不可注入的行(`tool`):与 developer/system item 同样处理,不进 Codex
|
||||
/// 上下文。legacy 行不是 Responses item,`tool` 行无法还原成真正的工具 item,
|
||||
/// 注入会造出假的工具消息;聊天投影本来也只展示 user/assistant。
|
||||
NotChat,
|
||||
}
|
||||
|
||||
impl DirectProjectLegacyRow {
|
||||
fn message_item(self) -> Option<Value> {
|
||||
match self {
|
||||
DirectProjectLegacyRow::Message(item) => Some(item),
|
||||
DirectProjectLegacyRow::NotChat => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 格式切换前的 legacy 行投影。存量用户项目的历史文件全是这种行,读取时投影成与
|
||||
/// `direct_project_local_message_item` 同形状的 message item,`role` 与 `content`
|
||||
/// 原样保留(不 trim、不改写、不合并),未知字段忽略。
|
||||
///
|
||||
/// 角色白名单取的是 legacy 写入器自己的角色集合,也就是
|
||||
/// `project/conversation.rs` 里 `matches!(role, "user" | "assistant" | "tool")` 这一
|
||||
/// 条校验,所以「legacy 写入器能写出的行」被完整覆盖,不会有第三种角色漏进来;
|
||||
/// 白名单之外的角色(手改文件、未来写入器)仍按损坏失败关闭。
|
||||
fn direct_project_legacy_row(parsed: &Value) -> Option<DirectProjectLegacyRow> {
|
||||
let object = parsed.as_object()?;
|
||||
if object.contains_key("type")
|
||||
|| object.get("schemaVersion").and_then(Value::as_str)
|
||||
!= Some(DIRECT_PROJECT_HISTORY_LEGACY_SCHEMA_VERSION)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let role = object.get("role").and_then(Value::as_str)?;
|
||||
if !matches!(role, "user" | "assistant" | "tool") {
|
||||
return None;
|
||||
}
|
||||
let content = object.get("content").and_then(Value::as_str)?;
|
||||
if content.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if role == "tool" {
|
||||
return Some(DirectProjectLegacyRow::NotChat);
|
||||
}
|
||||
let message_id = object
|
||||
.get("messageId")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|message_id| !message_id.is_empty());
|
||||
Some(DirectProjectLegacyRow::Message(
|
||||
direct_project_message_item(role, content, message_id),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn append_direct_project_history_item_at(
|
||||
root: &Path,
|
||||
item: &Value,
|
||||
@@ -247,6 +318,16 @@ pub(crate) fn direct_project_local_message_item(
|
||||
if content.is_empty() || !matches!(role, "user" | "assistant") {
|
||||
return Err("DirectProject 只接受非空 user/assistant 历史消息".to_string());
|
||||
}
|
||||
Ok(direct_project_message_item(
|
||||
role,
|
||||
content,
|
||||
message_id.map(str::trim).filter(|value| !value.is_empty()),
|
||||
))
|
||||
}
|
||||
|
||||
/// 本地补写与 legacy 投影共用同一种 Responses message item 形状,两者的区别只在
|
||||
/// 是否对入参做 trim 校验:本地补写走上面的校验,legacy 行按文件内容逐字节投影。
|
||||
fn direct_project_message_item(role: &str, content: &str, message_id: Option<&str>) -> Value {
|
||||
let mut item = serde_json::json!({
|
||||
"type": "message",
|
||||
"role": role,
|
||||
@@ -255,10 +336,10 @@ pub(crate) fn direct_project_local_message_item(
|
||||
"text": content,
|
||||
}],
|
||||
});
|
||||
if let Some(message_id) = message_id.map(str::trim).filter(|value| !value.is_empty()) {
|
||||
if let Some(message_id) = message_id {
|
||||
item["id"] = Value::String(message_id.to_string());
|
||||
}
|
||||
Ok(item)
|
||||
item
|
||||
}
|
||||
|
||||
pub(crate) fn read_direct_project_history_items_at(root: &Path) -> Result<Vec<Value>, String> {
|
||||
@@ -293,16 +374,9 @@ pub(crate) fn read_direct_project_history_items_at(root: &Path) -> Result<Vec<Va
|
||||
));
|
||||
}
|
||||
};
|
||||
if parsed.get("type").and_then(Value::as_str) != Some(DIRECT_PROJECT_HISTORY_RECORD_TYPE) {
|
||||
return Err(format!(
|
||||
"DirectProject 历史记录类型无效:{}",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
let item = parsed
|
||||
.get("payload")
|
||||
.cloned()
|
||||
.ok_or_else(|| format!("DirectProject 历史记录缺少 payload:{}", path.display()))?;
|
||||
let Some(item) = direct_project_history_item_from_parsed_line(&path, &parsed)? else {
|
||||
continue;
|
||||
};
|
||||
if is_direct_project_internal_context_item(&item) {
|
||||
continue;
|
||||
}
|
||||
@@ -356,9 +430,27 @@ pub(crate) fn read_direct_project_chat_history_at(
|
||||
mod tests {
|
||||
use super::{
|
||||
append_direct_project_history_item_at, append_direct_project_user_message_at, history_path,
|
||||
is_direct_project_internal_context_item, read_direct_project_history_items_at,
|
||||
is_direct_project_internal_context_item, read_direct_project_chat_history_at,
|
||||
read_direct_project_history_items_at,
|
||||
};
|
||||
use serde_json::json;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
fn init_history_project(name: &str) -> tempfile::TempDir {
|
||||
let root = tempfile::tempdir().expect("temp project");
|
||||
crate::init_local_game_project_at(root.path(), name, "历史格式兼容测试")
|
||||
.expect("init project");
|
||||
root
|
||||
}
|
||||
|
||||
fn write_history_lines(root: &std::path::Path, lines: &[&str]) {
|
||||
let path = history_path(root);
|
||||
std::fs::create_dir_all(path.parent().expect("history parent")).expect("history dir");
|
||||
std::fs::write(&path, format!("{}\n", lines.join("\n"))).expect("write history fixture");
|
||||
}
|
||||
|
||||
const LEGACY_USER_ROW: &str = r#"{"schemaVersion":"game-creator-conversation.v1","role":"user","content":"请创建菜单","agentId":null,"messageId":"direct-codex:turn-0001:user","updatedAt":1757000000}"#;
|
||||
const LEGACY_ASSISTANT_ROW: &str = r#"{"schemaVersion":"game-creator-conversation.v1","role":"assistant","content":"已完成 第一行\n第二行 ","agentId":null,"updatedAt":1757000001}"#;
|
||||
const RESPONSE_ITEM_ROW: &str = r#"{"type":"response_item","payload":{"type":"message","role":"user","id":"codex-item-2","content":[{"type":"input_text","text":"再加一个按钮"}]}}"#;
|
||||
|
||||
#[test]
|
||||
fn filters_host_context_but_keeps_real_user_items() {
|
||||
@@ -464,4 +556,186 @@ mod tests {
|
||||
let items = read_direct_project_history_items_at(root.path()).expect("read history");
|
||||
assert_eq!(items, vec![user]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_conversation_rows_project_into_responses_message_items() {
|
||||
let root = init_history_project("legacy-projection");
|
||||
write_history_lines(
|
||||
root.path(),
|
||||
&[LEGACY_USER_ROW, LEGACY_ASSISTANT_ROW],
|
||||
);
|
||||
|
||||
let items =
|
||||
read_direct_project_history_items_at(root.path()).expect("read legacy history");
|
||||
assert_eq!(
|
||||
items,
|
||||
vec![
|
||||
json!({
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"id": "direct-codex:turn-0001:user",
|
||||
"content": [{"type": "input_text", "text": "请创建菜单"}],
|
||||
}),
|
||||
json!({
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "已完成 第一行\n第二行 "}],
|
||||
}),
|
||||
]
|
||||
);
|
||||
|
||||
let chat = read_direct_project_chat_history_at(root.path()).expect("read legacy chat");
|
||||
assert_eq!(chat.messages.len(), 2);
|
||||
assert_eq!(chat.messages[0].role, "user");
|
||||
assert_eq!(chat.messages[0].content, "请创建菜单");
|
||||
assert_eq!(
|
||||
chat.messages[0].message_id.as_deref(),
|
||||
Some("direct-codex:turn-0001:user")
|
||||
);
|
||||
assert_eq!(chat.messages[1].role, "assistant");
|
||||
assert_eq!(chat.messages[1].content, "已完成 第一行\n第二行 ");
|
||||
assert_eq!(chat.messages[1].message_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_legacy_and_response_item_history_reads_in_file_order() {
|
||||
let root = init_history_project("mixed-history");
|
||||
write_history_lines(
|
||||
root.path(),
|
||||
&[LEGACY_USER_ROW, RESPONSE_ITEM_ROW, LEGACY_ASSISTANT_ROW],
|
||||
);
|
||||
|
||||
let items = read_direct_project_history_items_at(root.path()).expect("read mixed history");
|
||||
assert_eq!(
|
||||
items
|
||||
.iter()
|
||||
.map(|item| item.get("id").and_then(Value::as_str))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![Some("direct-codex:turn-0001:user"), Some("codex-item-2"), None]
|
||||
);
|
||||
assert_eq!(
|
||||
items[1],
|
||||
json!({
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"id": "codex-item-2",
|
||||
"content": [{"type": "input_text", "text": "再加一个按钮"}],
|
||||
})
|
||||
);
|
||||
assert_eq!(items[2]["content"][0]["text"], json!("已完成 第一行\n第二行 "));
|
||||
|
||||
let chat = read_direct_project_chat_history_at(root.path()).expect("read mixed chat");
|
||||
assert_eq!(
|
||||
chat.messages
|
||||
.iter()
|
||||
.map(|message| (message.role.as_str(), message.content.as_str()))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
("user", "请创建菜单"),
|
||||
("user", "再加一个按钮"),
|
||||
("assistant", "已完成 第一行\n第二行 "),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_compat_keeps_unrecognized_rows_failing_closed() {
|
||||
let unrecognized_rows = [
|
||||
// 带 type 却不是 response_item
|
||||
r#"{"type":"message","role":"user","content":[{"type":"input_text","text":"x"}]}"#,
|
||||
// schemaVersion 不在白名单里
|
||||
r#"{"schemaVersion":"game-creator-conversation.v2","role":"user","content":"x","agentId":null,"updatedAt":1}"#,
|
||||
// legacy 写入器角色集合之外的角色
|
||||
r#"{"schemaVersion":"game-creator-conversation.v1","role":"system","content":"x","agentId":null,"updatedAt":1}"#,
|
||||
r#"{"schemaVersion":"game-creator-conversation.v1","role":"developer","content":"x","agentId":null,"updatedAt":1}"#,
|
||||
// content 不是字符串
|
||||
r#"{"schemaVersion":"game-creator-conversation.v1","role":"user","content":42,"agentId":null,"updatedAt":1}"#,
|
||||
// content 为空
|
||||
r#"{"schemaVersion":"game-creator-conversation.v1","role":"user","content":"","agentId":null,"updatedAt":1}"#,
|
||||
// 缺 role
|
||||
r#"{"schemaVersion":"game-creator-conversation.v1","content":"x","agentId":null,"updatedAt":1}"#,
|
||||
];
|
||||
for row in unrecognized_rows {
|
||||
let root = init_history_project("unrecognized-history");
|
||||
write_history_lines(root.path(), &[row]);
|
||||
let error = read_direct_project_history_items_at(root.path())
|
||||
.expect_err("unrecognized row must fail closed");
|
||||
assert!(
|
||||
error.starts_with("DirectProject 历史记录类型无效"),
|
||||
"{row}: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
let root = init_history_project("missing-payload");
|
||||
write_history_lines(root.path(), &[r#"{"type":"response_item"}"#]);
|
||||
let error = read_direct_project_history_items_at(root.path())
|
||||
.expect_err("response_item without payload must fail closed");
|
||||
assert!(
|
||||
error.starts_with("DirectProject 历史记录缺少 payload"),
|
||||
"{error}"
|
||||
);
|
||||
|
||||
let root = init_history_project("broken-json");
|
||||
write_history_lines(root.path(), &[r#"{"schemaVersion":"game-creator"#]);
|
||||
let error = read_direct_project_history_items_at(root.path())
|
||||
.expect_err("broken json must fail closed");
|
||||
assert!(error.starts_with("解析 DirectProject 历史失败"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_tool_row_is_recognized_but_stays_out_of_the_codex_context() {
|
||||
let root = init_history_project("legacy-tool-row");
|
||||
write_history_lines(
|
||||
root.path(),
|
||||
&[
|
||||
LEGACY_USER_ROW,
|
||||
r#"{"schemaVersion":"game-creator-conversation.v1","role":"tool","content":"{\"ok\":true}","agentId":null,"updatedAt":1757000002}"#,
|
||||
LEGACY_ASSISTANT_ROW,
|
||||
],
|
||||
);
|
||||
|
||||
let items = read_direct_project_history_items_at(root.path()).expect("read history");
|
||||
assert_eq!(
|
||||
items
|
||||
.iter()
|
||||
.map(|item| item["role"].as_str().unwrap_or_default())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["user", "assistant"]
|
||||
);
|
||||
|
||||
let chat = read_direct_project_chat_history_at(root.path()).expect("read chat");
|
||||
assert_eq!(
|
||||
chat.messages
|
||||
.iter()
|
||||
.map(|message| message.role.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["user", "assistant"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generic_writer_rows_are_covered_by_the_legacy_whitelist() {
|
||||
let root = init_history_project("generic-writer-shape");
|
||||
crate::append_local_conversation_message_at(
|
||||
root.path(),
|
||||
None,
|
||||
crate::LocalConversationMessage {
|
||||
role: "user".to_string(),
|
||||
content: "通用写入器写的旧格式回合".to_string(),
|
||||
agent_id: None,
|
||||
},
|
||||
)
|
||||
.expect("append legacy row through the generic writer");
|
||||
|
||||
let items =
|
||||
read_direct_project_history_items_at(root.path()).expect("read generic writer row");
|
||||
assert_eq!(
|
||||
items,
|
||||
vec![json!({
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "通用写入器写的旧格式回合"}],
|
||||
})]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user