92 lines
2.2 KiB
TypeScript
92 lines
2.2 KiB
TypeScript
import type { LlmAdapter } from "./types.js";
|
|
|
|
interface OpenAiResponse {
|
|
choices: Array<{ message: { content: string } }>;
|
|
usage: {
|
|
prompt_tokens: number;
|
|
completion_tokens: number;
|
|
total_tokens: number;
|
|
};
|
|
timings?: { prompt_ms: number; predicted_ms: number };
|
|
}
|
|
|
|
export class OpenAiCompatibleAdapter implements LlmAdapter {
|
|
private url: string;
|
|
private apiKey: string | undefined;
|
|
private model: string | undefined;
|
|
|
|
constructor(url: string, apiKey?: string, model?: string) {
|
|
this.url = url;
|
|
this.apiKey = apiKey;
|
|
this.model = model;
|
|
}
|
|
|
|
async call(
|
|
words: string[],
|
|
systemPrompt: string,
|
|
): Promise<{
|
|
content: string;
|
|
promptTokens: number;
|
|
completionTokens: number;
|
|
totalTokens: number;
|
|
promptTimeMs: number | null;
|
|
completionTimeMs: number | null;
|
|
totalTimeMs: number;
|
|
}> {
|
|
const payload: Record<string, unknown> = {
|
|
messages: [
|
|
{ role: "system", content: systemPrompt },
|
|
{ role: "user", content: JSON.stringify(words) },
|
|
],
|
|
temperature: 0.1,
|
|
top_p: 0.9,
|
|
max_tokens: Math.ceil(words.length * 250 * 1.2),
|
|
};
|
|
|
|
if (this.model) {
|
|
payload["model"] = this.model;
|
|
}
|
|
|
|
const headers: Record<string, string> = {
|
|
"Content-Type": "application/json",
|
|
};
|
|
|
|
if (this.apiKey) {
|
|
headers["Authorization"] = `Bearer ${this.apiKey}`;
|
|
}
|
|
|
|
const startTime = Date.now();
|
|
|
|
const response = await fetch(this.url, {
|
|
method: "POST",
|
|
headers,
|
|
body: JSON.stringify(payload),
|
|
});
|
|
|
|
const totalTimeMs = Date.now() - startTime;
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`LLM server responded with status: ${response.status}`);
|
|
}
|
|
|
|
const json = (await response.json()) as OpenAiResponse;
|
|
|
|
const content = json.choices[0]?.message?.content;
|
|
if (!content) {
|
|
throw new Error("LLM response content is empty");
|
|
}
|
|
|
|
const promptTokens = json.usage.prompt_tokens;
|
|
const completionTokens = json.usage.completion_tokens;
|
|
|
|
return {
|
|
content,
|
|
promptTokens,
|
|
completionTokens,
|
|
totalTokens: json.usage.total_tokens,
|
|
promptTimeMs: json.timings?.prompt_ms ?? null,
|
|
completionTimeMs: json.timings?.predicted_ms ?? null,
|
|
totalTimeMs,
|
|
};
|
|
}
|
|
}
|