kdletters
202279c6d9
新增 Core、Engine、Runtime、SQLite、Provider、MCP、Skill、Codex、CLI 与 DAG crate 补齐 OpenAI endpoint 配置、Provider 实例/协议路由和统一工具权限边界 加入持久化、lease、checkpoint、reconciliation、审批恢复与消息历史回归 加入独立 workspace CI、依赖边界、能力集和 Fake Agent 测试脚本 同步建设计划、TODO、架构、测试与验收文档
216 lines
10 KiB
Bash
Executable File
216 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
# 最小真实 Provider eval。默认只列出用例；显式 --real 才会产生网络请求。
|
||
# 临时状态默认写入 ~/data/tmp（优先尊重显式 TMPDIR；可用 AGENT_EVAL_TMPDIR 覆盖），
|
||
# 退出时清理本轮目录，不把构建或测试产物写到用户 home 的其它位置。
|
||
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||
workspace_root="$(cd -- "$script_dir/.." && pwd)"
|
||
dataset="$workspace_root/tests/agent-llm-eval.jsonl"
|
||
tmp_parent="${AGENT_EVAL_TMPDIR:-${TMPDIR:-${HOME:?HOME 未设置}/data/tmp}}"
|
||
run_real=0
|
||
list_only=0
|
||
from_codex_config=0
|
||
|
||
usage() {
|
||
cat <<'EOF'
|
||
用法：
|
||
./scripts/run-agent-llm-eval.sh --real # 调用 OpenAI-compatible Provider
|
||
./scripts/run-agent-llm-eval.sh --real --from-codex-config # 使用当前 Codex 配置
|
||
./scripts/run-agent-llm-eval.sh --dataset tests/agent-llm-capability-eval.jsonl --real --from-codex-config
|
||
./scripts/run-agent-llm-eval.sh --list # 列出用例，不联网
|
||
|
||
真实运行需要 OPENAI_API_KEY（或 OPENAI_API_KEY_ENV/AGENT_OPENAI_API_KEY_ENV
|
||
指向的变量）。脚本不会读取或打印密钥值；可用 AGENT_MODEL/OPENAI_MODEL 选模型。
|
||
`--from-codex-config` 从 `$CODEX_HOME/config.toml`（默认 `~/.codex/config.toml`）
|
||
和同目录 `auth.json` 读取当前模型、Responses 网关和密钥，仅注入本轮子进程。
|
||
EOF
|
||
}
|
||
|
||
while (($# > 0)); do
|
||
case "$1" in
|
||
--dataset)
|
||
[[ $# -ge 2 ]] || { echo "--dataset 需要路径" >&2; exit 2; }
|
||
dataset="$2"
|
||
[[ "$dataset" = /* ]] || dataset="$workspace_root/$dataset"
|
||
shift
|
||
;;
|
||
--real) run_real=1 ;;
|
||
--from-codex-config) from_codex_config=1 ;;
|
||
--list) list_only=1 ;;
|
||
-h|--help) usage; exit 0 ;;
|
||
*) echo "未知参数：$1" >&2; usage >&2; exit 2 ;;
|
||
esac
|
||
shift
|
||
done
|
||
|
||
if [[ "$list_only" == 1 ]]; then
|
||
python3 - "$dataset" <<'PY'
|
||
import json, sys
|
||
for line in open(sys.argv[1], encoding="utf-8"):
|
||
if line.strip():
|
||
case = json.loads(line)
|
||
print(f"{case['id']}: {case['input']['stream'] and 'stream' or 'complete'}")
|
||
PY
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$run_real" != 1 ]]; then
|
||
echo "未执行：真实 eval 需要显式 --real（使用 --list 查看用例）" >&2
|
||
exit 2
|
||
fi
|
||
|
||
if [[ "$from_codex_config" == 1 ]]; then
|
||
codex_home="${CODEX_HOME:-${HOME:?HOME 未设置}/.codex}"
|
||
codex_config="$codex_home/config.toml"
|
||
codex_auth="$codex_home/auth.json"
|
||
if [[ ! -r "$codex_config" || ! -r "$codex_auth" ]]; then
|
||
echo "当前 Codex 配置不完整：需要 config.toml 和 auth.json（不会显示内容）" >&2
|
||
exit 2
|
||
fi
|
||
# 只在进程内搬运配置；stdout 为 base64 字段，不会直接回显凭据。
|
||
config_values="$(python3 - "$codex_config" "$codex_auth" <<'PY'
|
||
import base64, json, pathlib, sys, tomllib
|
||
config = tomllib.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))
|
||
auth = json.loads(pathlib.Path(sys.argv[2]).read_text(encoding="utf-8"))
|
||
provider_name = config.get("model_provider")
|
||
provider = (config.get("model_providers") or {}).get(provider_name or "", {})
|
||
model = str(config.get("model") or "")
|
||
url = str(provider.get("base_url") or "")
|
||
key = str(auth.get("OPENAI_API_KEY") or "")
|
||
if not model or not url or not key:
|
||
raise SystemExit("Codex 配置缺少 model、provider base_url 或 OPENAI_API_KEY")
|
||
enc = lambda value: base64.b64encode(value.encode()).decode()
|
||
print("\t".join((model, enc(url), enc(key))))
|
||
PY
|
||
)"
|
||
IFS=$'\t' read -r codex_model codex_base64_url codex_base64_key <<<"$config_values"
|
||
export AGENT_MODEL="$codex_model"
|
||
export OPENAI_BASE_URL="$(printf '%s' "$codex_base64_url" | base64 -d)"
|
||
export OPENAI_API_KEY="$(printf '%s' "$codex_base64_key" | base64 -d)"
|
||
fi
|
||
|
||
key_name="${AGENT_OPENAI_API_KEY_ENV:-${OPENAI_API_KEY_ENV:-OPENAI_API_KEY}}"
|
||
if [[ -z "${!key_name:-}" ]]; then
|
||
echo "缺少真实 Provider 凭据环境变量：$key_name（不会显示其值）" >&2
|
||
exit 2
|
||
fi
|
||
|
||
mkdir -p -- "$tmp_parent"
|
||
tmp_parent="$(cd -- "$tmp_parent" && pwd)"
|
||
suite_dir="$(mktemp -d "$tmp_parent/agent-llm-eval.XXXXXX")"
|
||
cleanup() { rm -rf -- "$suite_dir"; }
|
||
trap cleanup EXIT
|
||
export TMPDIR="$suite_dir/tmp"
|
||
export CARGO_TARGET_DIR="$suite_dir/target"
|
||
mkdir -p -- "$TMPDIR"
|
||
|
||
python3 - "$dataset" "$workspace_root" "$suite_dir" <<'PY'
|
||
import json, os, pathlib, re, subprocess, sys
|
||
|
||
dataset, workspace, suite_dir = map(pathlib.Path, sys.argv[1:])
|
||
|
||
def scrub_error(raw: bytes) -> str:
|
||
text = raw.decode("utf-8", errors="replace")
|
||
names = {"OPENAI_API_KEY", os.environ.get("OPENAI_API_KEY_ENV", ""), os.environ.get("AGENT_OPENAI_API_KEY_ENV", "")}
|
||
for name in names:
|
||
if name:
|
||
value = os.environ.get(name, "")
|
||
if value:
|
||
text = text.replace(value, "[redacted]")
|
||
text = re.sub(r"(?i)(authorization|api[-_ ]?key|token|password)\s*[:=]\s*[^\s,;]+", r"\1=[redacted]", text)
|
||
return next((line.strip() for line in reversed(text.splitlines()) if line.strip()), "无错误详情")
|
||
|
||
def run_case(case, case_dir):
|
||
env = os.environ.copy()
|
||
for name in ("AGENT_SKILL_ROOT", "AGENT_SKILL_ROOTS", "AGENT_SKILLS", "AGENT_MCP_STDIO_COMMAND", "AGENT_MCP_STDIO_ARGS", "AGENT_MCP_HTTP_URL", "AGENT_MCP_HTTP_HEADERS", "AGENT_MCP_ALLOW", "AGENT_SYSTEM_PROMPT", "AGENT_DEVELOPER_PROMPT", "AGENT_CONTEXT_PROMPT"):
|
||
env.pop(name, None)
|
||
env.update({"AGENT_PROVIDER": "openai", "AGENT_CONFIG": str(case_dir / "missing.toml"), "AGENT_DB": str(case_dir / "agent.db"), "AGENT_STREAM": "1" if case["input"].get("stream") else "0"})
|
||
if case["input"].get("model"):
|
||
env["AGENT_MODEL"] = case["input"]["model"]
|
||
for field, env_name in (("system_prompt", "AGENT_SYSTEM_PROMPT"), ("developer_prompt", "AGENT_DEVELOPER_PROMPT"), ("context_prompt", "AGENT_CONTEXT_PROMPT")):
|
||
value = case["input"].get(field)
|
||
if value:
|
||
env[env_name] = value
|
||
args = ["cargo", "run", "--locked", "-q", "-p", "agent-cli", "--", "run", "--jsonl", "--stream" if case["input"].get("stream") else "--no-stream", case["input"]["prompt"]]
|
||
proc = subprocess.run(args, cwd=workspace, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180, check=False)
|
||
if proc.returncode:
|
||
raise RuntimeError(f"退出码 {proc.returncode}：{scrub_error(proc.stderr)}")
|
||
result = None
|
||
for line in proc.stdout.decode("utf-8", errors="replace").splitlines():
|
||
try:
|
||
record = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
if record.get("type") == "result":
|
||
result = record.get("result")
|
||
if result is None:
|
||
raise RuntimeError("Provider 输出缺少 JSONL result 记录")
|
||
return result
|
||
|
||
def check(case, result):
|
||
out = result.get("output") or {}
|
||
events = out.get("events") or []
|
||
types = [e.get("type") for e in events]
|
||
assertion = case.get("assert", {})
|
||
if assertion.get("status"):
|
||
# completed 是 Engine 的 Finished 语义；CLI 结果中不重复返回状态字段。
|
||
if "finished" not in types:
|
||
raise AssertionError("缺少 finished 事件")
|
||
text = str(out.get("text") or "")
|
||
expected = assertion.get("final_text", {})
|
||
if expected.get("mode") == "contains_all" and any(v not in text for v in expected.get("values", [])):
|
||
raise AssertionError("最终文本未满足 contains_all")
|
||
steps = int(out.get("steps") or 0)
|
||
bounds = assertion.get("steps", {})
|
||
if steps < bounds.get("min", 0) or (bounds.get("max") is not None and steps > bounds["max"]):
|
||
raise AssertionError(f"steps={steps} 超出范围")
|
||
if len(out.get("stream_events") or []) < assertion.get("stream_events", {}).get("min", 0):
|
||
raise AssertionError("stream_events 不足")
|
||
tool_assert = assertion.get("tool_calls", {})
|
||
completed_tools = [
|
||
event.get("call") or {}
|
||
for event in events
|
||
if event.get("type") == "tool_requested"
|
||
]
|
||
min_tools = int(tool_assert.get("min_completed", 0))
|
||
max_tools = tool_assert.get("max_completed")
|
||
if len(completed_tools) < min_tools or (max_tools is not None and len(completed_tools) > int(max_tools)):
|
||
raise AssertionError(f"tool_requested 数量={len(completed_tools)} 超出范围")
|
||
for expected_tool in tool_assert.get("required", []):
|
||
matches = [call for call in completed_tools if call.get("name") == expected_tool.get("name")]
|
||
if not matches:
|
||
raise AssertionError(f"未调用工具 {expected_tool.get('name')}")
|
||
actual_args = matches[0].get("arguments") or {}
|
||
expected_args = expected_tool.get("arguments") or {}
|
||
if actual_args != expected_args:
|
||
raise AssertionError(f"工具 {expected_tool.get('name')} 参数不匹配")
|
||
if assertion.get("secrets_absent"):
|
||
for name in ("OPENAI_API_KEY", os.environ.get("OPENAI_API_KEY_ENV", ""), os.environ.get("AGENT_OPENAI_API_KEY_ENV", "")):
|
||
value = os.environ.get(name, "")
|
||
if value and value in text:
|
||
raise AssertionError("最终文本疑似包含 Provider 凭据")
|
||
weights = case.get("score", {}).get("weights", {})
|
||
observed = {"status": 1.0, "final_text": 1.0, "streaming": 1.0, "efficiency": 1.0, "tool_calls": 1.0}
|
||
total = sum(float(value) for value in weights.values())
|
||
score = (sum(float(weights.get(name, 0)) * observed.get(name, 0.0) for name in weights) / total) if total else 1.0
|
||
threshold = float(case.get("score", {}).get("pass_threshold", 1.0))
|
||
if score < threshold:
|
||
raise AssertionError(f"score={score:.2f} 低于阈值 {threshold:.2f}")
|
||
return score
|
||
|
||
cases = [json.loads(line) for line in dataset.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||
passed = 0
|
||
for case in cases:
|
||
case_dir = suite_dir / case["id"]
|
||
case_dir.mkdir(parents=True)
|
||
try:
|
||
score = check(case, run_case(case, case_dir))
|
||
passed += 1
|
||
print(f"PASS {case['id']} score={score:.2f}")
|
||
except Exception as exc:
|
||
print(f"FAIL {case['id']}: {exc}")
|
||
print(f"SUMMARY total={len(cases)} passed={passed} failed={len(cases)-passed}")
|
||
raise SystemExit(0 if passed == len(cases) else 1)
|
||
PY
|