DirectProject 读侧白名单兼容格式切换前的旧行,存量项目不再全部读不动
- direct_project_history.rs:两处行信封判定收敛到 direct_project_history_item_from_parsed_line,response_item 与白名单化的 legacy 行走同一条判定路径 - 新增 direct_project_legacy_row:只认 schemaVersion=game-creator-conversation.v1、无 type、role 落在该写入器自己的角色集合(user/assistant/tool)内、content 为非空字符串的行,其余形状继续按损坏失败关闭 - user/assistant 旧行投影成与 direct_project_local_message_item 同形状的 message item:role 与 content 逐字节保留、不 trim、不改写、未知字段忽略,有 messageId 才带 id - tool 旧行已识别但不进 Codex 上下文:它不是 Responses item,无法还原成真正的工具 item,混入会造出假的工具消息;聊天投影本来也只展示 user/assistant - 抽出 direct_project_message_item 供本地补写与 legacy 投影共用,保证两种来源的 item 形状一致 - 补断言:旧行投影(role/content 逐字节、无 messageId 时无 id)、旧行与新行交替的混合文件按行顺序读取、tool 行被识别且不进上下文、非白名单异常行(role=system/developer、content 非字符串或为空、缺 role、缺 payload、坏 JSON)仍失败关闭、通用对话写入器产出的旧行形状落在白名单内 - 变异验证:去掉 legacy 兼容分支→投影/混合文件/通用写入器形状 3 个用例变红;把白名单放宽成「任意行都接受」→失败关闭用例变红
This commit is contained in:
@@ -9,6 +9,9 @@ use std::io::{BufRead, BufReader, Read, Seek, SeekFrom};
|
|||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
const DIRECT_PROJECT_HISTORY_RECORD_TYPE: &str = "response_item";
|
const DIRECT_PROJECT_HISTORY_RECORD_TYPE: &str = "response_item";
|
||||||
|
/// 格式切换到 `response_item`(#282)之前,DirectProject 主对话通过通用对话写入器
|
||||||
|
/// 落到同一份 `project.jsonl`,行形状是 `PersistedLocalConversationMessageRecord`。
|
||||||
|
const DIRECT_PROJECT_HISTORY_LEGACY_SCHEMA_VERSION: &str = "game-creator-conversation.v1";
|
||||||
const DIRECT_PROJECT_INTERNAL_CONTEXT_KINDS: &[&str] = &[
|
const DIRECT_PROJECT_INTERNAL_CONTEXT_KINDS: &[&str] = &[
|
||||||
"host_skills.instructions",
|
"host_skills.instructions",
|
||||||
"permissions.instructions",
|
"permissions.instructions",
|
||||||
@@ -159,22 +162,90 @@ fn direct_project_history_item_from_line(
|
|||||||
}
|
}
|
||||||
let parsed: Value = serde_json::from_slice(line)
|
let parsed: Value = serde_json::from_slice(line)
|
||||||
.map_err(|error| format!("解析 DirectProject 历史失败:{}: {error}", path.display()))?;
|
.map_err(|error| format!("解析 DirectProject 历史失败:{}: {error}", path.display()))?;
|
||||||
if parsed.get("type").and_then(Value::as_str) != Some(DIRECT_PROJECT_HISTORY_RECORD_TYPE) {
|
let Some(item) = direct_project_history_item_from_parsed_line(path, &parsed)? else {
|
||||||
return Err(format!(
|
return Ok(None);
|
||||||
"DirectProject 历史记录类型无效:{}",
|
};
|
||||||
path.display()
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let item = parsed
|
|
||||||
.get("payload")
|
|
||||||
.cloned()
|
|
||||||
.ok_or_else(|| format!("DirectProject 历史记录缺少 payload:{}", path.display()))?;
|
|
||||||
if is_direct_project_internal_context_item(&item) {
|
if is_direct_project_internal_context_item(&item) {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
Ok(Some(item))
|
Ok(Some(item))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 只接受两种行信封:`response_item`,以及白名单化的 legacy 行。返回 `None` 表示行已
|
||||||
|
/// 被识别、但不该进入 Codex 上下文(legacy 的 `tool` 行)。
|
||||||
|
/// 其余任何形状(含换了 `schemaVersion`、带 `type` 却不是 `response_item`、role 不在
|
||||||
|
/// legacy 写入器自己的角色集合内、content 不是非空字符串)都判定为损坏并失败关闭。
|
||||||
|
fn direct_project_history_item_from_parsed_line(
|
||||||
|
path: &Path,
|
||||||
|
parsed: &Value,
|
||||||
|
) -> Result<Option<Value>, String> {
|
||||||
|
if parsed.get("type").and_then(Value::as_str) == Some(DIRECT_PROJECT_HISTORY_RECORD_TYPE) {
|
||||||
|
return parsed
|
||||||
|
.get("payload")
|
||||||
|
.cloned()
|
||||||
|
.map(Some)
|
||||||
|
.ok_or_else(|| format!("DirectProject 历史记录缺少 payload:{}", path.display()));
|
||||||
|
}
|
||||||
|
direct_project_legacy_row(parsed)
|
||||||
|
.map(|row| row.message_item())
|
||||||
|
.ok_or_else(|| format!("DirectProject 历史记录类型无效:{}", path.display()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 白名单化 legacy 行的投影结果。
|
||||||
|
enum DirectProjectLegacyRow {
|
||||||
|
/// user/assistant 行:投影成 message item 进入历史。
|
||||||
|
Message(Value),
|
||||||
|
/// 已识别但不可注入的行(`tool`):与 developer/system item 同样处理,不进 Codex
|
||||||
|
/// 上下文。legacy 行不是 Responses item,`tool` 行无法还原成真正的工具 item,
|
||||||
|
/// 注入会造出假的工具消息;聊天投影本来也只展示 user/assistant。
|
||||||
|
NotChat,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DirectProjectLegacyRow {
|
||||||
|
fn message_item(self) -> Option<Value> {
|
||||||
|
match self {
|
||||||
|
DirectProjectLegacyRow::Message(item) => Some(item),
|
||||||
|
DirectProjectLegacyRow::NotChat => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 格式切换前的 legacy 行投影。存量用户项目的历史文件全是这种行,读取时投影成与
|
||||||
|
/// `direct_project_local_message_item` 同形状的 message item,`role` 与 `content`
|
||||||
|
/// 原样保留(不 trim、不改写、不合并),未知字段忽略。
|
||||||
|
///
|
||||||
|
/// 角色白名单取的是 legacy 写入器自己的角色集合,也就是
|
||||||
|
/// `project/conversation.rs` 里 `matches!(role, "user" | "assistant" | "tool")` 这一
|
||||||
|
/// 条校验,所以「legacy 写入器能写出的行」被完整覆盖,不会有第三种角色漏进来;
|
||||||
|
/// 白名单之外的角色(手改文件、未来写入器)仍按损坏失败关闭。
|
||||||
|
fn direct_project_legacy_row(parsed: &Value) -> Option<DirectProjectLegacyRow> {
|
||||||
|
let object = parsed.as_object()?;
|
||||||
|
if object.contains_key("type")
|
||||||
|
|| object.get("schemaVersion").and_then(Value::as_str)
|
||||||
|
!= Some(DIRECT_PROJECT_HISTORY_LEGACY_SCHEMA_VERSION)
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let role = object.get("role").and_then(Value::as_str)?;
|
||||||
|
if !matches!(role, "user" | "assistant" | "tool") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let content = object.get("content").and_then(Value::as_str)?;
|
||||||
|
if content.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if role == "tool" {
|
||||||
|
return Some(DirectProjectLegacyRow::NotChat);
|
||||||
|
}
|
||||||
|
let message_id = object
|
||||||
|
.get("messageId")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.filter(|message_id| !message_id.is_empty());
|
||||||
|
Some(DirectProjectLegacyRow::Message(
|
||||||
|
direct_project_message_item(role, content, message_id),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn append_direct_project_history_item_at(
|
pub(crate) fn append_direct_project_history_item_at(
|
||||||
root: &Path,
|
root: &Path,
|
||||||
item: &Value,
|
item: &Value,
|
||||||
@@ -247,6 +318,16 @@ pub(crate) fn direct_project_local_message_item(
|
|||||||
if content.is_empty() || !matches!(role, "user" | "assistant") {
|
if content.is_empty() || !matches!(role, "user" | "assistant") {
|
||||||
return Err("DirectProject 只接受非空 user/assistant 历史消息".to_string());
|
return Err("DirectProject 只接受非空 user/assistant 历史消息".to_string());
|
||||||
}
|
}
|
||||||
|
Ok(direct_project_message_item(
|
||||||
|
role,
|
||||||
|
content,
|
||||||
|
message_id.map(str::trim).filter(|value| !value.is_empty()),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 本地补写与 legacy 投影共用同一种 Responses message item 形状,两者的区别只在
|
||||||
|
/// 是否对入参做 trim 校验:本地补写走上面的校验,legacy 行按文件内容逐字节投影。
|
||||||
|
fn direct_project_message_item(role: &str, content: &str, message_id: Option<&str>) -> Value {
|
||||||
let mut item = serde_json::json!({
|
let mut item = serde_json::json!({
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"role": role,
|
"role": role,
|
||||||
@@ -255,10 +336,10 @@ pub(crate) fn direct_project_local_message_item(
|
|||||||
"text": content,
|
"text": content,
|
||||||
}],
|
}],
|
||||||
});
|
});
|
||||||
if let Some(message_id) = message_id.map(str::trim).filter(|value| !value.is_empty()) {
|
if let Some(message_id) = message_id {
|
||||||
item["id"] = Value::String(message_id.to_string());
|
item["id"] = Value::String(message_id.to_string());
|
||||||
}
|
}
|
||||||
Ok(item)
|
item
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn read_direct_project_history_items_at(root: &Path) -> Result<Vec<Value>, String> {
|
pub(crate) fn read_direct_project_history_items_at(root: &Path) -> Result<Vec<Value>, String> {
|
||||||
@@ -293,16 +374,9 @@ pub(crate) fn read_direct_project_history_items_at(root: &Path) -> Result<Vec<Va
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if parsed.get("type").and_then(Value::as_str) != Some(DIRECT_PROJECT_HISTORY_RECORD_TYPE) {
|
let Some(item) = direct_project_history_item_from_parsed_line(&path, &parsed)? else {
|
||||||
return Err(format!(
|
continue;
|
||||||
"DirectProject 历史记录类型无效:{}",
|
};
|
||||||
path.display()
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let item = parsed
|
|
||||||
.get("payload")
|
|
||||||
.cloned()
|
|
||||||
.ok_or_else(|| format!("DirectProject 历史记录缺少 payload:{}", path.display()))?;
|
|
||||||
if is_direct_project_internal_context_item(&item) {
|
if is_direct_project_internal_context_item(&item) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -356,9 +430,27 @@ pub(crate) fn read_direct_project_chat_history_at(
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
append_direct_project_history_item_at, append_direct_project_user_message_at, history_path,
|
append_direct_project_history_item_at, append_direct_project_user_message_at, history_path,
|
||||||
is_direct_project_internal_context_item, read_direct_project_history_items_at,
|
is_direct_project_internal_context_item, read_direct_project_chat_history_at,
|
||||||
|
read_direct_project_history_items_at,
|
||||||
};
|
};
|
||||||
use serde_json::json;
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
|
fn init_history_project(name: &str) -> tempfile::TempDir {
|
||||||
|
let root = tempfile::tempdir().expect("temp project");
|
||||||
|
crate::init_local_game_project_at(root.path(), name, "历史格式兼容测试")
|
||||||
|
.expect("init project");
|
||||||
|
root
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_history_lines(root: &std::path::Path, lines: &[&str]) {
|
||||||
|
let path = history_path(root);
|
||||||
|
std::fs::create_dir_all(path.parent().expect("history parent")).expect("history dir");
|
||||||
|
std::fs::write(&path, format!("{}\n", lines.join("\n"))).expect("write history fixture");
|
||||||
|
}
|
||||||
|
|
||||||
|
const LEGACY_USER_ROW: &str = r#"{"schemaVersion":"game-creator-conversation.v1","role":"user","content":"请创建菜单","agentId":null,"messageId":"direct-codex:turn-0001:user","updatedAt":1757000000}"#;
|
||||||
|
const LEGACY_ASSISTANT_ROW: &str = r#"{"schemaVersion":"game-creator-conversation.v1","role":"assistant","content":"已完成 第一行\n第二行 ","agentId":null,"updatedAt":1757000001}"#;
|
||||||
|
const RESPONSE_ITEM_ROW: &str = r#"{"type":"response_item","payload":{"type":"message","role":"user","id":"codex-item-2","content":[{"type":"input_text","text":"再加一个按钮"}]}}"#;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn filters_host_context_but_keeps_real_user_items() {
|
fn filters_host_context_but_keeps_real_user_items() {
|
||||||
@@ -464,4 +556,186 @@ mod tests {
|
|||||||
let items = read_direct_project_history_items_at(root.path()).expect("read history");
|
let items = read_direct_project_history_items_at(root.path()).expect("read history");
|
||||||
assert_eq!(items, vec![user]);
|
assert_eq!(items, vec![user]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_conversation_rows_project_into_responses_message_items() {
|
||||||
|
let root = init_history_project("legacy-projection");
|
||||||
|
write_history_lines(
|
||||||
|
root.path(),
|
||||||
|
&[LEGACY_USER_ROW, LEGACY_ASSISTANT_ROW],
|
||||||
|
);
|
||||||
|
|
||||||
|
let items =
|
||||||
|
read_direct_project_history_items_at(root.path()).expect("read legacy history");
|
||||||
|
assert_eq!(
|
||||||
|
items,
|
||||||
|
vec![
|
||||||
|
json!({
|
||||||
|
"type": "message",
|
||||||
|
"role": "user",
|
||||||
|
"id": "direct-codex:turn-0001:user",
|
||||||
|
"content": [{"type": "input_text", "text": "请创建菜单"}],
|
||||||
|
}),
|
||||||
|
json!({
|
||||||
|
"type": "message",
|
||||||
|
"role": "assistant",
|
||||||
|
"content": [{"type": "output_text", "text": "已完成 第一行\n第二行 "}],
|
||||||
|
}),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
let chat = read_direct_project_chat_history_at(root.path()).expect("read legacy chat");
|
||||||
|
assert_eq!(chat.messages.len(), 2);
|
||||||
|
assert_eq!(chat.messages[0].role, "user");
|
||||||
|
assert_eq!(chat.messages[0].content, "请创建菜单");
|
||||||
|
assert_eq!(
|
||||||
|
chat.messages[0].message_id.as_deref(),
|
||||||
|
Some("direct-codex:turn-0001:user")
|
||||||
|
);
|
||||||
|
assert_eq!(chat.messages[1].role, "assistant");
|
||||||
|
assert_eq!(chat.messages[1].content, "已完成 第一行\n第二行 ");
|
||||||
|
assert_eq!(chat.messages[1].message_id, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mixed_legacy_and_response_item_history_reads_in_file_order() {
|
||||||
|
let root = init_history_project("mixed-history");
|
||||||
|
write_history_lines(
|
||||||
|
root.path(),
|
||||||
|
&[LEGACY_USER_ROW, RESPONSE_ITEM_ROW, LEGACY_ASSISTANT_ROW],
|
||||||
|
);
|
||||||
|
|
||||||
|
let items = read_direct_project_history_items_at(root.path()).expect("read mixed history");
|
||||||
|
assert_eq!(
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.map(|item| item.get("id").and_then(Value::as_str))
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec![Some("direct-codex:turn-0001:user"), Some("codex-item-2"), None]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
items[1],
|
||||||
|
json!({
|
||||||
|
"type": "message",
|
||||||
|
"role": "user",
|
||||||
|
"id": "codex-item-2",
|
||||||
|
"content": [{"type": "input_text", "text": "再加一个按钮"}],
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(items[2]["content"][0]["text"], json!("已完成 第一行\n第二行 "));
|
||||||
|
|
||||||
|
let chat = read_direct_project_chat_history_at(root.path()).expect("read mixed chat");
|
||||||
|
assert_eq!(
|
||||||
|
chat.messages
|
||||||
|
.iter()
|
||||||
|
.map(|message| (message.role.as_str(), message.content.as_str()))
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec![
|
||||||
|
("user", "请创建菜单"),
|
||||||
|
("user", "再加一个按钮"),
|
||||||
|
("assistant", "已完成 第一行\n第二行 "),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_compat_keeps_unrecognized_rows_failing_closed() {
|
||||||
|
let unrecognized_rows = [
|
||||||
|
// 带 type 却不是 response_item
|
||||||
|
r#"{"type":"message","role":"user","content":[{"type":"input_text","text":"x"}]}"#,
|
||||||
|
// schemaVersion 不在白名单里
|
||||||
|
r#"{"schemaVersion":"game-creator-conversation.v2","role":"user","content":"x","agentId":null,"updatedAt":1}"#,
|
||||||
|
// legacy 写入器角色集合之外的角色
|
||||||
|
r#"{"schemaVersion":"game-creator-conversation.v1","role":"system","content":"x","agentId":null,"updatedAt":1}"#,
|
||||||
|
r#"{"schemaVersion":"game-creator-conversation.v1","role":"developer","content":"x","agentId":null,"updatedAt":1}"#,
|
||||||
|
// content 不是字符串
|
||||||
|
r#"{"schemaVersion":"game-creator-conversation.v1","role":"user","content":42,"agentId":null,"updatedAt":1}"#,
|
||||||
|
// content 为空
|
||||||
|
r#"{"schemaVersion":"game-creator-conversation.v1","role":"user","content":"","agentId":null,"updatedAt":1}"#,
|
||||||
|
// 缺 role
|
||||||
|
r#"{"schemaVersion":"game-creator-conversation.v1","content":"x","agentId":null,"updatedAt":1}"#,
|
||||||
|
];
|
||||||
|
for row in unrecognized_rows {
|
||||||
|
let root = init_history_project("unrecognized-history");
|
||||||
|
write_history_lines(root.path(), &[row]);
|
||||||
|
let error = read_direct_project_history_items_at(root.path())
|
||||||
|
.expect_err("unrecognized row must fail closed");
|
||||||
|
assert!(
|
||||||
|
error.starts_with("DirectProject 历史记录类型无效"),
|
||||||
|
"{row}: {error}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let root = init_history_project("missing-payload");
|
||||||
|
write_history_lines(root.path(), &[r#"{"type":"response_item"}"#]);
|
||||||
|
let error = read_direct_project_history_items_at(root.path())
|
||||||
|
.expect_err("response_item without payload must fail closed");
|
||||||
|
assert!(
|
||||||
|
error.starts_with("DirectProject 历史记录缺少 payload"),
|
||||||
|
"{error}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let root = init_history_project("broken-json");
|
||||||
|
write_history_lines(root.path(), &[r#"{"schemaVersion":"game-creator"#]);
|
||||||
|
let error = read_direct_project_history_items_at(root.path())
|
||||||
|
.expect_err("broken json must fail closed");
|
||||||
|
assert!(error.starts_with("解析 DirectProject 历史失败"), "{error}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_tool_row_is_recognized_but_stays_out_of_the_codex_context() {
|
||||||
|
let root = init_history_project("legacy-tool-row");
|
||||||
|
write_history_lines(
|
||||||
|
root.path(),
|
||||||
|
&[
|
||||||
|
LEGACY_USER_ROW,
|
||||||
|
r#"{"schemaVersion":"game-creator-conversation.v1","role":"tool","content":"{\"ok\":true}","agentId":null,"updatedAt":1757000002}"#,
|
||||||
|
LEGACY_ASSISTANT_ROW,
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
let items = read_direct_project_history_items_at(root.path()).expect("read history");
|
||||||
|
assert_eq!(
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.map(|item| item["role"].as_str().unwrap_or_default())
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec!["user", "assistant"]
|
||||||
|
);
|
||||||
|
|
||||||
|
let chat = read_direct_project_chat_history_at(root.path()).expect("read chat");
|
||||||
|
assert_eq!(
|
||||||
|
chat.messages
|
||||||
|
.iter()
|
||||||
|
.map(|message| message.role.as_str())
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
vec!["user", "assistant"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn generic_writer_rows_are_covered_by_the_legacy_whitelist() {
|
||||||
|
let root = init_history_project("generic-writer-shape");
|
||||||
|
crate::append_local_conversation_message_at(
|
||||||
|
root.path(),
|
||||||
|
None,
|
||||||
|
crate::LocalConversationMessage {
|
||||||
|
role: "user".to_string(),
|
||||||
|
content: "通用写入器写的旧格式回合".to_string(),
|
||||||
|
agent_id: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("append legacy row through the generic writer");
|
||||||
|
|
||||||
|
let items =
|
||||||
|
read_direct_project_history_items_at(root.path()).expect("read generic writer row");
|
||||||
|
assert_eq!(
|
||||||
|
items,
|
||||||
|
vec![json!({
|
||||||
|
"type": "message",
|
||||||
|
"role": "user",
|
||||||
|
"content": [{"type": "input_text", "text": "通用写入器写的旧格式回合"}],
|
||||||
|
})]
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user