Genarrative/scripts/test-ve-llm.mjs

import { readFileSync } from 'node:fs';
import { resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const root = resolve(__dirname, '..');

function loadEnv(path) {
  const content = readFileSync(path, 'utf-8');
  const env = {};
  for (const line of content.split('\n')) {
    const trimmed = line.trim();
    if (!trimmed || trimmed.startsWith('#')) continue;
    const eqIndex = trimmed.indexOf('=');
    if (eqIndex === -1) continue;
    const key = trimmed.slice(0, eqIndex).trim();
    let value = trimmed.slice(eqIndex + 1).trim();
    if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
      value = value.slice(1, -1);
    }
    env[key] = value;
  }
  return env;
}

const env = loadEnv(resolve(root, '.env.secrets.local'));
const BASE = env.VECTOR_ENGINE_BASE_URL?.replace(/\/+$/, '') || 'https://api.vectorengine.cn';
const KEY = env.VECTOR_ENGINE_API_KEY || '';

if (!KEY) {
  console.error('未找到 VECTOR_ENGINE_API_KEY');
  process.exit(1);
}

const TIMEOUT_MS = 60_000;

async function test(name, method, path, body = null) {
  const url = `${BASE}${path}`;
  const start = Date.now();
  try {
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
    const headers = {
      'Authorization': `Bearer ${KEY}`,
      'Content-Type': 'application/json',
    };
    const options = { method, headers, signal: controller.signal };
    if (body) options.body = JSON.stringify(body);

    const resp = await fetch(url, options);
    clearTimeout(timer);
    const elapsed = Date.now() - start;
    const text = await resp.text();
    let json = null;
    try { json = JSON.parse(text); } catch {}

    if (resp.ok) {
      const model = json?.model || json?.data?.[0]?.id || '?';
      const summary = json?.choices?.[0] ? `choices[0]: ${json.choices[0].message?.content?.slice(0, 80)}` :
                      json?.output_text ? `output_text: ${json.output_text.slice(0, 80)}` :
                      json?.data ? `${json.data.length} models` : JSON.stringify(json).slice(0, 120);
      return { ok: true, elapsed, code: resp.status, model, summary };
    } else {
      const errMsg = json?.error?.message || json?.message || text.slice(0, 200);
      return { ok: false, elapsed, code: resp.status, error: errMsg };
    }
  } catch (e) {
    const elapsed = Date.now() - start;
    return { ok: false, elapsed, code: 0, error: e.name === 'AbortError' ? `超时(${TIMEOUT_MS / 1000}s)` : e.message };
  }
}

console.log(`VectorEngine LLM 能力探测`);
console.log(`目标: ${BASE}\n`);

const tests = [
  // 1. 探测 /v1/models - 基础连通性 + 列出可用模型
  { name: 'GET /v1/models (列出可用模型)', method: 'GET', path: '/v1/models' },

  // 2. Chat Completions - 最标准协议，项目已有 LlmProvider::OpenAiCompatible 支持
  {
    name: 'POST /v1/chat/completions (Chat)',
    method: 'POST',
    path: '/v1/chat/completions',
    body: {
      model: 'gpt-4o',
      messages: [{ role: 'user', content: '回复 ok，不要解释' }],
      max_tokens: 10,
    },
  },

  // 3. Responses - Apimart 当前使用的协议
  {
    name: 'POST /v1/responses (Responses)',
    method: 'POST',
    path: '/v1/responses',
    body: {
      model: 'gpt-4o',
      input: [
        { role: 'user', content: [{ type: 'input_text', text: '回复 ok，不要解释' }] },
      ],
    },
  },

  // 4. 测试 gpt-5 (creative_agent 模型)
  {
    name: 'POST /v1/chat/completions (gpt-5, Chat)',
    method: 'POST',
    path: '/v1/chat/completions',
    body: {
      model: 'gpt-5',
      messages: [{ role: 'user', content: '回复 ok' }],
      max_tokens: 10,
    },
  },

  // 5. 抓大鹅生成需要的 JSON 输出能力验证
  {
    name: 'POST /v1/chat/completions (JSON 输出: 抓大鹅物品)',
    method: 'POST',
    path: '/v1/chat/completions',
    body: {
      model: 'gpt-4o',
      messages: [
        { role: 'system', content: '你是抓大鹅游戏编辑，只返回 JSON。' },
        { role: 'user', content: '题材：水果。请生成 JSON：{"gameName":"水果切切乐","items":[{"name":"苹果","itemSize":"中"},{"name":"西瓜","itemSize":"大"}]}' },
      ],
      max_tokens: 200,
    },
  },
];

let pass = 0;
let fail = 0;

for (let i = 0; i < tests.length; i++) {
  const t = tests[i];
  console.log(`[${i + 1}/${tests.length}] ${t.name}`);
  const result = await test(t.name, t.method, t.path, t.body);

  if (result.ok) {
    console.log(`  ✅ HTTP ${result.code}  ${result.elapsed}ms  model: ${result.model}`);
    console.log(`     ${result.summary}`);
    pass++;
  } else {
    const codeStr = result.code === 0 ? 'NET' : `HTTP ${result.code}`;
    console.log(`  ❌ ${codeStr}  ${result.elapsed}ms  ${result.error}`);
    fail++;
  }
  console.log();
}

console.log(`=== 结果: ${pass}/${tests.length} 通过, ${fail}/${tests.length} 失败 ===`);

// 结论
if (pass >= 3) {
  console.log('\n✅ VectorEngine 支持 LLM 文本调用，可替代 Apimart。');
  console.log('   将 .env.secrets.local 中 APIMART_BASE_URL 改为 VectorEngine 地址即可。');
} else if (pass <= 1) {
  console.log('\n❌ VectorEngine 不支持 LLM 文本调用。');
} else {
  console.log('\n⚠️ 部分支持，需进一步评估。');
}