新方案v1角色结构理解

2026-09-17 17:44:24 +08:00
parent 2e79bc4a12
commit 30e82af4f1
21 changed files with 1287 additions and 14 deletions
@@ -0,0 +1,53 @@
+import {dataUrl} from '../lib/images.mjs';
+import {requestJson} from './http.mjs';
+
+export function buildVisionRequest(stage, image, schema) {
+  const instruction = `${stage.prompt}\n\nCanvas: ${image.width} x ${image.height} pixels.\nReturn ONLY JSON matching this schema:\n${JSON.stringify(schema)}`;
+  if (stage.protocol === 'chat-completions') {
+    const body = { ...stage.params, model: stage.model, stream: stage.stream,
+      messages: [{role: 'user', content: [
+        {type: 'text', text: instruction},
+        {type: 'image_url', image_url: {url: dataUrl(image)}},
+      ]}],
+    };
+    if (stage.outputMode === 'json-object') body.response_format = {type: 'json_object'};
+    if (stage.outputMode === 'json-schema') body.response_format = {type: 'json_schema', json_schema: {name: 'semantic_parts', strict: true, schema}};
+    return {route: 'chat/completions', body};
+  }
+  if (stage.protocol === 'responses') {
+    const body = {...stage.params, model: stage.model, stream: stage.stream, instructions: stage.prompt, input: [{role: 'user', content: [
+      {type: 'input_text', text: instruction},
+      {type: 'input_image', image_url: dataUrl(image)},
+    ]}]};
+    if (stage.outputMode === 'json-object') body.text = {format: {type: 'json_object'}};
+    if (stage.outputMode === 'json-schema') body.text = {format: {type: 'json_schema', name: 'semantic_parts', strict: true, schema}};
+    return {route: 'responses', body};
+  }
+  throw new Error(`未知理解协议：${stage.protocol}`);
+}
+
+export function extractVisionText(payload, protocol) {
+  if (protocol === 'chat-completions') {
+    const choice = payload.choices?.[0];
+    if (choice?.message?.refusal) throw new Error('结构理解请求被模型拒绝');
+    if (choice?.finish_reason && choice.finish_reason !== 'stop') throw new Error(`结构理解响应未完整结束：${choice.finish_reason}`);
+    const content = choice?.message?.content;
+    if (typeof content === 'string' && content.trim()) return content;
+    if (Array.isArray(content)) {
+      const text = content.filter((item) => item.type === 'text').map((item) => item.text).join('\n');
+      if (text.trim()) return text;
+    }
+  } else {
+    if (payload.status && payload.status !== 'completed') throw new Error(`结构理解响应状态：${payload.status}`);
+    const items = (payload.output || []).flatMap((item) => item.content || []);
+    if (items.some((item) => item.type === 'refusal')) throw new Error('结构理解请求被模型拒绝');
+    const text = payload.output_text || items.filter((item) => item.type === 'output_text').map((item) => item.text).join('\n');
+    if (typeof text === 'string' && text.trim()) return text;
+  }
+  throw new Error('服务端响应中没有结构理解文本');
+}
+
+export async function analyze(stage, image, schema, options) {
+  const {route, body} = buildVisionRequest(stage, image, schema);
+  return requestJson(stage, route, () => ({method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify(body)}), options);
+}