From a23f430e2afd7d9ea462f71c2fd1568d8e1dba38 Mon Sep 17 00:00:00 2001 From: polwex Date: Wed, 23 Jul 2025 07:42:10 +0700 Subject: turns out kimi an others dont implement the new openai sdk --- src/generic.ts | 187 ++++++++++++++++++++++++++------------------------------- 1 file changed, 86 insertions(+), 101 deletions(-) (limited to 'src/generic.ts') diff --git a/src/generic.ts b/src/generic.ts index 3690dc6..e8dfa13 100644 --- a/src/generic.ts +++ b/src/generic.ts @@ -1,17 +1,16 @@ import OpenAI from "openai"; import { MAX_TOKENS, RESPONSE_LENGTH } from "./logic/constants"; -import type { AIModelAPI, InputToken } from "./types"; +import type { AIModelAPI, ChatMessage, InputToken } from "./types"; import type { AsyncRes } from "sortug"; -import type { - ResponseCreateParamsBase, - ResponseCreateParamsNonStreaming, - ResponseCreateParamsStreaming, - ResponseInput, -} from "openai/resources/responses/responses.mjs"; +import type { ChatCompletionContentPart } from "openai/resources"; + +type OChoice = OpenAI.Chat.Completions.ChatCompletion.Choice; +type Message = OpenAI.Chat.Completions.ChatCompletionUserMessageParam; +type OMessage = OpenAI.Chat.Completions.ChatCompletionMessageParam; type Props = { baseURL: string; - apiKey: string | undefined; + apiKey: string; model?: string; maxTokens?: number; tokenizer?: (text: string) => number; @@ -25,8 +24,6 @@ export default class OpenAIAPI implements AIModelAPI { model; constructor(props: Props) { - if (!props.apiKey) throw new Error("NO API KEY"); - console.log({ props }); this.apiKey = props.apiKey; this.baseURL = props.baseURL; this.api = new OpenAI({ baseURL: this.baseURL, apiKey: this.apiKey }); @@ -37,36 +34,43 @@ export default class OpenAIAPI implements AIModelAPI { public setModel(model: string) { this.model = model; } - - public buildInput(tokens: InputToken[]): ResponseInput { - return [ - { - role: "user", - content: tokens.map((t) => - "text" in t - ? { type: "input_text", text: t.text } - : "img" in t - ? { type: "input_image", image_url: t.img, detail: "auto" } - : { type: "input_text", text: "oy vey" }, - ), - }, - ]; + private mapMessages(input: ChatMessage[]): Message[] { + return input.map((m) => { + return { role: m.author as any, content: m.text, name: m.author }; + }); + } + private buildInput(tokens: InputToken[]): Message[] { + const content: ChatCompletionContentPart[] = tokens.map((t) => { + if ("text" in t) return { type: "text", text: t.text }; + if ("img" in t) return { type: "image_url", image_url: { url: t.img } }; + else return { type: "text", text: "oy vey" }; + }); + return [{ role: "user", content }]; } - // OpenAI SDK has three kinds ReponseInputContent: text image and file - // images can be URLs or base64 dataurl thingies - // public async send( - inpt: string | InputToken[], + input: string | InputToken[], sys?: string, ): AsyncRes { - const input = typeof inpt === "string" ? inpt : this.buildInput(inpt); - const params = sys ? { instructions: sys, input } : { input }; - const res = await this.apiCall(params); + const messages: Message[] = + typeof input === "string" + ? [{ role: "user" as const, content: input }] + : this.buildInput(input); + // const messages = this.mapMessages(input); + const allMessages: OMessage[] = sys + ? [{ role: "system", content: sys }, ...messages] + : messages; + const truncated = this.truncateHistory(allMessages); + const res = await this.apiCall(truncated); if ("error" in res) return res; else { try { - return { ok: res.ok.output_text }; + // TODO type this properly + const choices: OChoice[] = res.ok; + const resText = choices.reduce((acc, item) => { + return `${acc}\n${item.message.content || ""}`; + }, ""); + return { ok: resText }; } catch (e) { return { error: `${e}` }; } @@ -74,30 +78,46 @@ export default class OpenAIAPI implements AIModelAPI { } public async stream( - inpt: string | InputToken[], + input: string | InputToken[], handle: (c: string) => void, sys?: string, ) { - const input = typeof inpt === "string" ? inpt : this.buildInput(inpt); - const params = sys ? { instructions: sys, input } : { input }; - await this.apiCallStream(params, handle); + const messages: Message[] = + typeof input === "string" + ? [{ role: "user" as const, content: input }] + : this.buildInput(input); + // const messages = this.mapMessages(input); + const allMessages: OMessage[] = sys + ? [{ role: "system", content: sys }, ...messages] + : messages; + const truncated = this.truncateHistory(allMessages); + await this.apiCallStream(truncated, handle); + } + + private truncateHistory(messages: OMessage[]): OMessage[] { + const totalTokens = messages.reduce((total, message) => { + return total + this.tokenizer(message.content as string); + }, 0); + while (totalTokens > this.maxTokens && messages.length > 1) { + // Always keep the system message if it exists + const startIndex = messages[0].role === "system" ? 1 : 0; + messages.splice(startIndex, 1); + } + return messages; } // TODO custom temperature? - private async apiCall( - params: ResponseCreateParamsNonStreaming, - ): AsyncRes { + private async apiCall(messages: OMessage[]): AsyncRes { + // console.log({ messages }, "at the very end"); try { - const res = await this.api.responses.create({ - ...params, + const completion = await this.api.chat.completions.create({ // temperature: 1.3, - model: params.model || this.model, - input: params.input, - max_output_tokens: params.max_output_tokens || RESPONSE_LENGTH, - stream: false, + model: this.model, + messages, + max_tokens: RESPONSE_LENGTH, }); - // TODO damn there's a lot of stuff here - return { ok: res }; + if (!completion) return { error: "null response from openai" }; + return { ok: completion.choices }; } catch (e) { console.log(e, "error in openai api"); return { error: `${e}` }; @@ -105,65 +125,30 @@ export default class OpenAIAPI implements AIModelAPI { } private async apiCallStream( - params: ResponseCreateParamsBase, - handler: (c: string) => void, - ) { - // temperature: 1.3, - const pms: ResponseCreateParamsStreaming = { - ...params, - stream: true, - model: params.model || this.model, - input: params.input, - max_output_tokens: params.max_output_tokens || RESPONSE_LENGTH, - }; + messages: OMessage[], + handle: (c: string) => void, + ): Promise { try { - const stream = await this.api.responses.create(pms); - for await (const event of stream) { - console.log(event); - switch (event.type) { - // TODO deal with audio and whatever - case "response.output_text.delta": - handler(event.delta); - break; - case "response.completed": - break; - default: - break; + const stream = await this.api.chat.completions.create({ + temperature: 1.3, + model: this.model, + messages, + max_tokens: RESPONSE_LENGTH, + stream: true, + }); + + for await (const chunk of stream) { + for (const choice of chunk.choices) { + console.log({ choice }); + if (!choice.delta) continue; + const cont = choice.delta.content; + if (!cont) continue; + handle(cont); } - // if (event.type === "response.completed") - // wtf how do we use this } } catch (e) { console.log(e, "error in openai api"); - return { error: `${e}` }; + handle(`Error streaming OpenAI, ${e}`); } } - - // private async apiCallStream( - // messages: Message[], - // handle: (c: string) => void, - // ): Promise { - // try { - // const stream = await this.api.chat.completions.create({ - // temperature: 1.3, - // model: this.model, - // messages, - // max_tokens: RESPONSE_LENGTH, - // stream: true, - // }); - - // for await (const chunk of stream) { - // for (const choice of chunk.choices) { - // console.log({ choice }); - // if (!choice.delta) continue; - // const cont = choice.delta.content; - // if (!cont) continue; - // handle(cont); - // } - // } - // } catch (e) { - // console.log(e, "error in openai api"); - // handle(`Error streaming OpenAI, ${e}`); - // } - // } } -- cgit v1.2.3