Genarrative/scripts/test-ve-llm.mjs

import { readFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const root = resolve(__dirname, '..');

function loadEnv(path) {
  const content = readFileSync(path, 'utf-8');
  const env = {};
  for (const line of content.split('\n')) {
    const trimmed = line.trim();
    if (!trimmed || trimmed.startsWith('#')) continue;
    const eqIndex = trimmed.indexOf('=');
    if (eqIndex === -1) continue;
    const key = trimmed.slice(0, eqIndex).trim();
    let value = trimmed.slice(eqIndex + 1).trim();
    if (
      (value.startsWith('"') && value.endsWith('"')) ||
      (value.startsWith("'") && value.endsWith("'"))
    ) {
      value = value.slice(1, -1);
    }
    env[key] = value;
  }
  return env;
}

const env = loadEnv(resolve(root, '.env.secrets.local'));
const BASE =
  env.TIANTOKEN_BASE_URL?.replace(/\/+$/, '') || 'https://api.tiantoken.com';
const KEY = env.TIANTOKEN_API_KEY || '';

if (!KEY) {
  console.error('未找到 TIANTOKEN_API_KEY');
  process.exit(1);
}

const TIMEOUT_MS = 60_000;

async function test(name, method, path, body = null) {
  const url = `${BASE}${path}`;
  const start = Date.now();
  try {
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
    const headers = {
      Authorization: `Bearer ${KEY}`,
      'Content-Type': 'application/json',
    };
    const options = { method, headers, signal: controller.signal };
    if (body) options.body = JSON.stringify(body);

    const resp = await fetch(url, options);
    clearTimeout(timer);
    const elapsed = Date.now() - start;
    const text = await resp.text();
    let json = null;
    try {
      json = JSON.parse(text);
    } catch {
      // 非 JSON 上游错误由下方原始文本分支处理。
    }

    if (resp.ok) {
      const model = json?.model || json?.data?.[0]?.id || '?';
      const summary = json?.choices?.[0]
        ? `choices[0]: ${json.choices[0].message?.content?.slice(0, 80)}`
        : json?.output_text
          ? `output_text: ${json.output_text.slice(0, 80)}`
          : json?.data
            ? `${json.data.length} models`
            : JSON.stringify(json).slice(0, 120);
      return { ok: true, elapsed, code: resp.status, model, summary };
    } else {
      const errMsg =
        json?.error?.message || json?.message || text.slice(0, 200);
      return { ok: false, elapsed, code: resp.status, error: errMsg };
    }
  } catch (e) {
    const elapsed = Date.now() - start;
    return {
      ok: false,
      elapsed,
      code: 0,
      error:
        e.name === 'AbortError' ? `超时(${TIMEOUT_MS / 1000}s)` : e.message,
    };
  }
}

console.log(`Tiantoken LLM 能力探测`);
console.log(`目标: ${BASE}\n`);

const tests = [
  // 1. 探测 /v1/models - 基础连通性 + 列出可用模型
  { name: 'GET /v1/models (列出可用模型)', method: 'GET', path: '/v1/models' },

  // 2. Chat Completions - 最标准协议，项目已有 LlmProvider::OpenAiCompatible 支持
  {
    name: 'POST /v1/chat/completions (Chat)',
    method: 'POST',
    path: '/v1/chat/completions',
    body: {
      model: 'gpt-5.4-mini',
      messages: [{ role: 'user', content: '回复 ok，不要解释' }],
      max_completion_tokens: 10,
    },
  },

  // 3. Responses - 仅作兼容探测，creative_agent 默认走 Chat Completions
  {
    name: 'POST /v1/responses (Responses)',
    method: 'POST',
    path: '/v1/responses',
    body: {
      model: 'gpt-5.4-mini',
      input: [
        {
          role: 'user',
          content: [{ type: 'input_text', text: '回复 ok，不要解释' }],
        },
      ],
    },
  },

  // 4. 测试 gpt-5.4-mini (creative_agent 模型)
  {
    name: 'POST /v1/chat/completions (gpt-5.4-mini, Chat)',
    method: 'POST',
    path: '/v1/chat/completions',
    body: {
      model: 'gpt-5.4-mini',
      messages: [{ role: 'user', content: '回复 ok' }],
      max_completion_tokens: 10,
    },
  },

  // 5. 抓大鹅生成需要的 JSON 输出能力验证
  {
    name: 'POST /v1/chat/completions (JSON 输出: 抓大鹅物品)',
    method: 'POST',
    path: '/v1/chat/completions',
    body: {
      model: 'gpt-5.4-mini',
      messages: [
        { role: 'system', content: '你是抓大鹅游戏编辑，只返回 JSON。' },
        {
          role: 'user',
          content:
            '题材：水果。请生成 JSON：{"gameName":"水果切切乐","items":[{"name":"苹果","itemSize":"中"},{"name":"西瓜","itemSize":"大"}]}',
        },
      ],
      max_completion_tokens: 200,
    },
  },
];

let pass = 0;
let fail = 0;

for (let i = 0; i < tests.length; i++) {
  const t = tests[i];
  console.log(`[${i + 1}/${tests.length}] ${t.name}`);
  const result = await test(t.name, t.method, t.path, t.body);

  if (result.ok) {
    console.log(
      `  ✅ HTTP ${result.code}  ${result.elapsed}ms  model: ${result.model}`,
    );
    console.log(`     ${result.summary}`);
    pass++;
  } else {
    const codeStr = result.code === 0 ? 'NET' : `HTTP ${result.code}`;
    console.log(`  ❌ ${codeStr}  ${result.elapsed}ms  ${result.error}`);
    fail++;
  }
  console.log();
}

console.log(
  `=== 结果: ${pass}/${tests.length} 通过, ${fail}/${tests.length} 失败 ===`,
);

// 结论
if (pass >= 3) {
  console.log('\n✅ Tiantoken 支持 LLM 文本调用，可替代 Apimart。');
  console.log(
    '   将 .env.secrets.local 中 TIANTOKEN_BASE_URL / TIANTOKEN_API_KEY 配好即可。',
  );
} else if (pass <= 1) {
  console.log('\n❌ Tiantoken 不支持 LLM 文本调用。');
} else {
  console.log('\n⚠️ 部分支持，需进一步评估。');
}