Ukawka
54 lines
3.0 KiB
JavaScript
54 lines
3.0 KiB
JavaScript
import {dataUrl} from '../lib/images.mjs';
|
|
import {requestJson} from './http.mjs';
|
|
|
|
export function buildVisionRequest(stage, image, schema) {
|
|
const instruction = `${stage.prompt}\n\nCanvas: ${image.width} x ${image.height} pixels.\nReturn ONLY JSON matching this schema:\n${JSON.stringify(schema)}`;
|
|
if (stage.protocol === 'chat-completions') {
|
|
const body = { ...stage.params, model: stage.model, stream: stage.stream,
|
|
messages: [{role: 'user', content: [
|
|
{type: 'text', text: instruction},
|
|
{type: 'image_url', image_url: {url: dataUrl(image)}},
|
|
]}],
|
|
};
|
|
if (stage.outputMode === 'json-object') body.response_format = {type: 'json_object'};
|
|
if (stage.outputMode === 'json-schema') body.response_format = {type: 'json_schema', json_schema: {name: 'semantic_parts', strict: true, schema}};
|
|
return {route: 'chat/completions', body};
|
|
}
|
|
if (stage.protocol === 'responses') {
|
|
const body = {...stage.params, model: stage.model, stream: stage.stream, instructions: stage.prompt, input: [{role: 'user', content: [
|
|
{type: 'input_text', text: instruction},
|
|
{type: 'input_image', image_url: dataUrl(image)},
|
|
]}]};
|
|
if (stage.outputMode === 'json-object') body.text = {format: {type: 'json_object'}};
|
|
if (stage.outputMode === 'json-schema') body.text = {format: {type: 'json_schema', name: 'semantic_parts', strict: true, schema}};
|
|
return {route: 'responses', body};
|
|
}
|
|
throw new Error(`未知理解协议：${stage.protocol}`);
|
|
}
|
|
|
|
export function extractVisionText(payload, protocol) {
|
|
if (protocol === 'chat-completions') {
|
|
const choice = payload.choices?.[0];
|
|
if (choice?.message?.refusal) throw new Error('结构理解请求被模型拒绝');
|
|
if (choice?.finish_reason && choice.finish_reason !== 'stop') throw new Error(`结构理解响应未完整结束：${choice.finish_reason}`);
|
|
const content = choice?.message?.content;
|
|
if (typeof content === 'string' && content.trim()) return content;
|
|
if (Array.isArray(content)) {
|
|
const text = content.filter((item) => item.type === 'text').map((item) => item.text).join('\n');
|
|
if (text.trim()) return text;
|
|
}
|
|
} else {
|
|
if (payload.status && payload.status !== 'completed') throw new Error(`结构理解响应状态：${payload.status}`);
|
|
const items = (payload.output || []).flatMap((item) => item.content || []);
|
|
if (items.some((item) => item.type === 'refusal')) throw new Error('结构理解请求被模型拒绝');
|
|
const text = payload.output_text || items.filter((item) => item.type === 'output_text').map((item) => item.text).join('\n');
|
|
if (typeof text === 'string' && text.trim()) return text;
|
|
}
|
|
throw new Error('服务端响应中没有结构理解文本');
|
|
}
|
|
|
|
export async function analyze(stage, image, schema, options) {
|
|
const {route, body} = buildVisionRequest(stage, image, schema);
|
|
return requestJson(stage, route, () => ({method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify(body)}), options);
|
|
}
|