#!/usr/bin/env bash
set -euo pipefail

# 最小真实 Provider eval。默认只列出用例；显式 --real 才会产生网络请求。
# 临时状态默认写入 ~/data/tmp（优先尊重显式 TMPDIR；可用 AGENT_EVAL_TMPDIR 覆盖），
# 退出时清理本轮目录，不把构建或测试产物写到用户 home 的其它位置。
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
workspace_root="$(cd -- "$script_dir/.." && pwd)"
dataset="$workspace_root/tests/agent-llm-eval.jsonl"
tmp_parent="${AGENT_EVAL_TMPDIR:-${TMPDIR:-${HOME:?HOME 未设置}/data/tmp}}"
run_real=0
list_only=0
from_codex_config=0

usage() {
    cat <<'EOF'
用法：
  ./scripts/run-agent-llm-eval.sh --real       # 调用 OpenAI-compatible Provider
  ./scripts/run-agent-llm-eval.sh --real --from-codex-config # 使用当前 Codex 配置
  ./scripts/run-agent-llm-eval.sh --dataset tests/agent-llm-capability-eval.jsonl --real --from-codex-config
  ./scripts/run-agent-llm-eval.sh --list       # 列出用例，不联网

真实运行需要 OPENAI_API_KEY（或 OPENAI_API_KEY_ENV/AGENT_OPENAI_API_KEY_ENV
指向的变量）。脚本不会读取或打印密钥值；可用 AGENT_MODEL/OPENAI_MODEL 选模型。
`--from-codex-config` 从 `$CODEX_HOME/config.toml`（默认 `~/.codex/config.toml`）
和同目录 `auth.json` 读取当前模型、Responses 网关和密钥，仅注入本轮子进程。
EOF
}

while (($# > 0)); do
    case "$1" in
        --dataset)
            [[ $# -ge 2 ]] || { echo "--dataset 需要路径" >&2; exit 2; }
            dataset="$2"
            [[ "$dataset" = /* ]] || dataset="$workspace_root/$dataset"
            shift
            ;;
        --real) run_real=1 ;;
        --from-codex-config) from_codex_config=1 ;;
        --list) list_only=1 ;;
        -h|--help) usage; exit 0 ;;
        *) echo "未知参数：$1" >&2; usage >&2; exit 2 ;;
    esac
    shift
done

if [[ "$list_only" == 1 ]]; then
    python3 - "$dataset" <<'PY'
import json, sys
for line in open(sys.argv[1], encoding="utf-8"):
    if line.strip():
        case = json.loads(line)
        print(f"{case['id']}: {case['input']['stream'] and 'stream' or 'complete'}")
PY
    exit 0
fi

if [[ "$run_real" != 1 ]]; then
    echo "未执行：真实 eval 需要显式 --real（使用 --list 查看用例）" >&2
    exit 2
fi

if [[ "$from_codex_config" == 1 ]]; then
    codex_home="${CODEX_HOME:-${HOME:?HOME 未设置}/.codex}"
    codex_config="$codex_home/config.toml"
    codex_auth="$codex_home/auth.json"
    if [[ ! -r "$codex_config" || ! -r "$codex_auth" ]]; then
        echo "当前 Codex 配置不完整：需要 config.toml 和 auth.json（不会显示内容）" >&2
        exit 2
    fi
    # 只在进程内搬运配置；stdout 为 base64 字段，不会直接回显凭据。
    config_values="$(python3 - "$codex_config" "$codex_auth" <<'PY'
import base64, json, pathlib, sys, tomllib
config = tomllib.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))
auth = json.loads(pathlib.Path(sys.argv[2]).read_text(encoding="utf-8"))
provider_name = config.get("model_provider")
provider = (config.get("model_providers") or {}).get(provider_name or "", {})
model = str(config.get("model") or "")
url = str(provider.get("base_url") or "")
key = str(auth.get("OPENAI_API_KEY") or "")
if not model or not url or not key:
    raise SystemExit("Codex 配置缺少 model、provider base_url 或 OPENAI_API_KEY")
enc = lambda value: base64.b64encode(value.encode()).decode()
print("\t".join((model, enc(url), enc(key))))
PY
)"
    IFS=$'\t' read -r codex_model codex_base64_url codex_base64_key <<<"$config_values"
    export AGENT_MODEL="$codex_model"
    export OPENAI_BASE_URL="$(printf '%s' "$codex_base64_url" | base64 -d)"
    export OPENAI_API_KEY="$(printf '%s' "$codex_base64_key" | base64 -d)"
fi

key_name="${AGENT_OPENAI_API_KEY_ENV:-${OPENAI_API_KEY_ENV:-OPENAI_API_KEY}}"
if [[ -z "${!key_name:-}" ]]; then
    echo "缺少真实 Provider 凭据环境变量：$key_name（不会显示其值）" >&2
    exit 2
fi

mkdir -p -- "$tmp_parent"
tmp_parent="$(cd -- "$tmp_parent" && pwd)"
suite_dir="$(mktemp -d "$tmp_parent/agent-llm-eval.XXXXXX")"
cleanup() { rm -rf -- "$suite_dir"; }
trap cleanup EXIT
export TMPDIR="$suite_dir/tmp"
export CARGO_TARGET_DIR="$suite_dir/target"
mkdir -p -- "$TMPDIR"

python3 - "$dataset" "$workspace_root" "$suite_dir" <<'PY'
import json, os, pathlib, re, subprocess, sys

dataset, workspace, suite_dir = map(pathlib.Path, sys.argv[1:])

def scrub_error(raw: bytes) -> str:
    text = raw.decode("utf-8", errors="replace")
    names = {"OPENAI_API_KEY", os.environ.get("OPENAI_API_KEY_ENV", ""), os.environ.get("AGENT_OPENAI_API_KEY_ENV", "")}
    for name in names:
        if name:
            value = os.environ.get(name, "")
            if value:
                text = text.replace(value, "[redacted]")
    text = re.sub(r"(?i)(authorization|api[-_ ]?key|token|password)\s*[:=]\s*[^\s,;]+", r"\1=[redacted]", text)
    return next((line.strip() for line in reversed(text.splitlines()) if line.strip()), "无错误详情")

def run_case(case, case_dir):
    env = os.environ.copy()
    for name in ("AGENT_SKILL_ROOT", "AGENT_SKILL_ROOTS", "AGENT_SKILLS", "AGENT_MCP_STDIO_COMMAND", "AGENT_MCP_STDIO_ARGS", "AGENT_MCP_HTTP_URL", "AGENT_MCP_HTTP_HEADERS", "AGENT_MCP_ALLOW", "AGENT_SYSTEM_PROMPT", "AGENT_DEVELOPER_PROMPT", "AGENT_CONTEXT_PROMPT"):
        env.pop(name, None)
    env.update({"AGENT_PROVIDER": "openai", "AGENT_CONFIG": str(case_dir / "missing.toml"), "AGENT_DB": str(case_dir / "agent.db"), "AGENT_STREAM": "1" if case["input"].get("stream") else "0"})
    if case["input"].get("model"):
        env["AGENT_MODEL"] = case["input"]["model"]
    for field, env_name in (("system_prompt", "AGENT_SYSTEM_PROMPT"), ("developer_prompt", "AGENT_DEVELOPER_PROMPT"), ("context_prompt", "AGENT_CONTEXT_PROMPT")):
        value = case["input"].get(field)
        if value:
            env[env_name] = value
    args = ["cargo", "run", "--locked", "-q", "-p", "agent-cli", "--", "run", "--jsonl", "--stream" if case["input"].get("stream") else "--no-stream", case["input"]["prompt"]]
    proc = subprocess.run(args, cwd=workspace, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180, check=False)
    if proc.returncode:
        raise RuntimeError(f"退出码 {proc.returncode}：{scrub_error(proc.stderr)}")
    result = None
    for line in proc.stdout.decode("utf-8", errors="replace").splitlines():
        try:
            record = json.loads(line)
        except json.JSONDecodeError:
            continue
        if record.get("type") == "result":
            result = record.get("result")
    if result is None:
        raise RuntimeError("Provider 输出缺少 JSONL result 记录")
    return result

def check(case, result):
    out = result.get("output") or {}
    events = out.get("events") or []
    types = [e.get("type") for e in events]
    assertion = case.get("assert", {})
    if assertion.get("status"):
        # completed 是 Engine 的 Finished 语义；CLI 结果中不重复返回状态字段。
        if "finished" not in types:
            raise AssertionError("缺少 finished 事件")
    text = str(out.get("text") or "")
    expected = assertion.get("final_text", {})
    if expected.get("mode") == "contains_all" and any(v not in text for v in expected.get("values", [])):
        raise AssertionError("最终文本未满足 contains_all")
    steps = int(out.get("steps") or 0)
    bounds = assertion.get("steps", {})
    if steps < bounds.get("min", 0) or (bounds.get("max") is not None and steps > bounds["max"]):
        raise AssertionError(f"steps={steps} 超出范围")
    if len(out.get("stream_events") or []) < assertion.get("stream_events", {}).get("min", 0):
        raise AssertionError("stream_events 不足")
    tool_assert = assertion.get("tool_calls", {})
    completed_tools = [
        event.get("call") or {}
        for event in events
        if event.get("type") == "tool_requested"
    ]
    min_tools = int(tool_assert.get("min_completed", 0))
    max_tools = tool_assert.get("max_completed")
    if len(completed_tools) < min_tools or (max_tools is not None and len(completed_tools) > int(max_tools)):
        raise AssertionError(f"tool_requested 数量={len(completed_tools)} 超出范围")
    for expected_tool in tool_assert.get("required", []):
        matches = [call for call in completed_tools if call.get("name") == expected_tool.get("name")]
        if not matches:
            raise AssertionError(f"未调用工具 {expected_tool.get('name')}")
        actual_args = matches[0].get("arguments") or {}
        expected_args = expected_tool.get("arguments") or {}
        if actual_args != expected_args:
            raise AssertionError(f"工具 {expected_tool.get('name')} 参数不匹配")
    if assertion.get("secrets_absent"):
        for name in ("OPENAI_API_KEY", os.environ.get("OPENAI_API_KEY_ENV", ""), os.environ.get("AGENT_OPENAI_API_KEY_ENV", "")):
            value = os.environ.get(name, "")
            if value and value in text:
                raise AssertionError("最终文本疑似包含 Provider 凭据")
    weights = case.get("score", {}).get("weights", {})
    observed = {"status": 1.0, "final_text": 1.0, "streaming": 1.0, "efficiency": 1.0, "tool_calls": 1.0}
    total = sum(float(value) for value in weights.values())
    score = (sum(float(weights.get(name, 0)) * observed.get(name, 0.0) for name in weights) / total) if total else 1.0
    threshold = float(case.get("score", {}).get("pass_threshold", 1.0))
    if score < threshold:
        raise AssertionError(f"score={score:.2f} 低于阈值 {threshold:.2f}")
    return score

cases = [json.loads(line) for line in dataset.read_text(encoding="utf-8").splitlines() if line.strip()]
passed = 0
for case in cases:
    case_dir = suite_dir / case["id"]
    case_dir.mkdir(parents=True)
    try:
        score = check(case, run_case(case, case_dir))
        passed += 1
        print(f"PASS {case['id']} score={score:.2f}")
    except Exception as exc:
        print(f"FAIL {case['id']}: {exc}")
print(f"SUMMARY total={len(cases)} passed={passed} failed={len(cases)-passed}")
raise SystemExit(0 if passed == len(cases) else 1)
PY