m

5 files changed, 85 insertions, 28 deletions

diff --git a/index.ts b/index.ts
index 3bc3a34..9776285 100644
--- a/index.ts
+++ b/index.ts
@@ -16,25 +16,25 @@ export default function (choice: LLMChoice): AIModelAPI {
         : "chatgpt" in choice
           ? new Generic({
               baseURL: "https://api.openai.com/v1",
-              apiKey: Bun.env.OPENAI_API_KEY!,
+              apiKey: Bun.env.OPENAI_API_KEY,
               model: choice.chatgpt || "o4-mini",
             })
           : "deepseek" in choice
             ? new Generic({
                 baseURL: "https://api.deepseek.com",
-                apiKey: Bun.env.DEEPSEEK_API_KEY!,
+                apiKey: Bun.env.DEEPSEEK_API_KEY,
                 model: "deepseek-reasoner",
               })
             : "kimi" in choice
               ? new Generic({
                   baseURL: "https://api.moonshot.ai/v1",
-                  apiKey: Bun.env.MOONSHOT_API_KEY!,
+                  apiKey: Bun.env.MOONSHOT_API_KEY,
                   model: "kimi-k2-0711-preview", // "kimi-latest"?
                 })
               : "grok" in choice
                 ? new Generic({
                     baseURL: "https://api.x.ai/v1",
-                    apiKey: Bun.env.XAI_API_KEY!,
+                    apiKey: Bun.env.XAI_API_KEY,
                     model: "grok-4", // "kimi-latest"?
                   })
                 : new Generic({
diff --git a/src/claude.ts b/src/claude.ts
index 629f9c2..8269378 100644
--- a/src/claude.ts
+++ b/src/claude.ts
@@ -1,9 +1,13 @@
 import Claude from "@anthropic-ai/sdk";
 import { RESPONSE_LENGTH } from "./logic/constants";
-import type { AIModelAPI, ChatMessage } from "./types";
+import type { AIModelAPI, ChatMessage, InputToken } from "./types";
 import { BOOKWORM_SYS } from "./prompts";
 import type { AsyncRes } from "sortug";
-import type { MessageCreateParamsStreaming } from "@anthropic-ai/sdk/resources";
+import type {
+  ImageBlockParam,
+  MessageCreateParamsStreaming,
+  TextBlockParam,
+} from "@anthropic-ai/sdk/resources";
 
 type Message = Claude.Messages.MessageParam;
 
@@ -30,14 +34,31 @@ export default class ClaudeAPI implements AIModelAPI {
       return { role, content: m.text };
     });
   }
+  private buildInput(tokens: InputToken[]): Message[] {
+    // can do base64 for images too
+    const content = tokens.map((t) => {
+      const content =
+        "text" in t
+          ? ({ type: "text", text: t.text } as TextBlockParam)
+          : "img" in t
+            ? ({
+                type: "image",
+                source: { type: "url", url: t.img },
+              } as ImageBlockParam)
+            : ({ type: "text", text: "oy vey" } as TextBlockParam);
+      return content;
+    });
+
+    return [{ role: "user", content }];
+  }
 
-  public async send(input: string | ChatMessage[], sys?: string) {
-    const msgs: ChatMessage[] =
+  // https://docs.anthropic.com/en/api/messages-examples#vision
+  public async send(input: string | InputToken[], sys?: string) {
+    const msgs: Message[] =
       typeof input === "string"
-        ? [{ author: "user", text: input, sent: 0 }]
-        : input;
-    const messages = this.mapMessages(msgs);
-    const truncated = this.truncateHistory(messages);
+        ? [{ role: "user", content: input }]
+        : this.buildInput(input);
+    const truncated = this.truncateHistory(msgs);
     const res = await this.apiCall(truncated, sys);
     return res;
   }
@@ -62,16 +83,15 @@ export default class ClaudeAPI implements AIModelAPI {
   }
 
   public async stream(
-    input: string | ChatMessage[],
+    input: string | InputToken[],
     handle: (c: any) => void,
     sys?: string,
   ) {
-    const msgs: ChatMessage[] =
+    const msgs: Message[] =
       typeof input === "string"
-        ? [{ author: "user", text: input, sent: 0 }]
-        : input;
-    const messages = this.mapMessages(msgs);
-    const truncated = this.truncateHistory(messages);
+        ? [{ role: "user", content: input }]
+        : this.buildInput(input);
+    const truncated = this.truncateHistory(msgs);
     await this.apiCallStream(truncated, handle, sys);
   }
 
diff --git a/src/gemini.ts b/src/gemini.ts
index 5c7267b..aac5338 100644
--- a/src/gemini.ts
+++ b/src/gemini.ts
@@ -6,6 +6,7 @@ import {
   createUserContent,
   GoogleGenAI,
   type Content,
+  type ContentListUnion,
   type GeneratedImage,
   type GeneratedVideo,
   type Part,
@@ -81,6 +82,17 @@ export default class GeminiAPI implements AIModelAPI {
       return { ok: part };
     } else return { ok: createPartFromBase64(imageString, mimeType) };
   }
+  async inlineImage(imageURI: URL): AsyncRes<Part> {
+    try {
+      const imgdata = await fetch(imageURI);
+      const imageArrayBuffer = await imgdata.arrayBuffer();
+      const base64ImageData = Buffer.from(imageArrayBuffer).toString("base64");
+      const mimeType = imgdata.headers.get("content-type") || "image/jpeg";
+      return { ok: { inlineData: { mimeType, data: base64ImageData } } };
+    } catch (e) {
+      return { error: `${e}` };
+    }
+  }
   public buildInput(tokens: InputToken[]): Result<Content> {
     try {
       const input = createUserContent(
@@ -100,11 +112,21 @@ export default class GeminiAPI implements AIModelAPI {
     }
   }
 
-  async send(input: string | Content, systemPrompt?: string): AsyncRes<string> {
+  async send(
+    input: string | InputToken[],
+    systemPrompt?: string,
+  ): AsyncRes<string> {
+    let contents: ContentListUnion;
+    if (typeof input === "string") contents = input;
+    else {
+      const built = this.buildInput(input);
+      if ("error" in built) return built;
+      else contents = built.ok;
+    }
     try {
       const opts = {
         model: this.model,
-        contents: input,
+        contents,
       };
       const fopts = systemPrompt
         ? { ...opts, config: { systemInstruction: systemPrompt } }
@@ -117,13 +139,20 @@ export default class GeminiAPI implements AIModelAPI {
     }
   }
   async stream(
-    input: string | Content,
+    input: string | InputToken[],
     handler: (s: string) => void,
     systemPrompt?: string,
   ) {
+    let contents: ContentListUnion;
+    if (typeof input === "string") contents = input;
+    else {
+      const built = this.buildInput(input);
+      if ("error" in built) return built;
+      else contents = built.ok;
+    }
     const opts = {
       model: this.model,
-      contents: input,
+      contents,
     };
     const fopts = systemPrompt
       ? { ...opts, config: { systemInstruction: systemPrompt } }
diff --git a/src/generic.ts b/src/generic.ts
index ac6b55b..50b19c3 100644
--- a/src/generic.ts
+++ b/src/generic.ts
@@ -1,6 +1,6 @@
 import OpenAI from "openai";
 import { MAX_TOKENS, RESPONSE_LENGTH } from "./logic/constants";
-import type { AIModelAPI, ChatMessage, InputToken, OChoice } from "./types";
+import type { AIModelAPI, InputToken } from "./types";
 import type { AsyncRes } from "sortug";
 import type {
   ResponseCreateParamsBase,
@@ -11,7 +11,7 @@ import type {
 
 type Props = {
   baseURL: string;
-  apiKey: string;
+  apiKey: string | undefined;
   model?: string;
   maxTokens?: number;
   tokenizer?: (text: string) => number;
@@ -25,9 +25,12 @@ export default class OpenAIAPI implements AIModelAPI {
   model;
 
   constructor(props: Props) {
+    if (!props.apiKey) throw new Error("NO API KEY");
+    console.log({ props });
     this.apiKey = props.apiKey;
     this.baseURL = props.baseURL;
     this.api = new OpenAI({ baseURL: this.baseURL, apiKey: this.apiKey });
+    console.log(this.api);
     this.model = props.model || "";
     if (props.maxTokens) this.maxTokens = props.maxTokens;
     if (props.tokenizer) this.tokenizer = props.tokenizer;
@@ -55,9 +58,10 @@ export default class OpenAIAPI implements AIModelAPI {
   // images can be URLs or base64 dataurl thingies
   //
   public async send(
-    input: string | ResponseInput,
+    inpt: string | InputToken[],
     sys?: string,
   ): AsyncRes<string> {
+    const input = typeof inpt === "string" ? inpt : this.buildInput(inpt);
     const params = sys ? { instructions: sys, input } : { input };
     const res = await this.apiCall(params);
     if ("error" in res) return res;
@@ -71,10 +75,11 @@ export default class OpenAIAPI implements AIModelAPI {
   }
 
   public async stream(
-    input: string,
+    inpt: string | InputToken[],
     handle: (c: string) => void,
     sys?: string,
   ) {
+    const input = typeof inpt === "string" ? inpt : this.buildInput(inpt);
     const params = sys ? { instructions: sys, input } : { input };
     await this.apiCallStream(params, handle);
   }
diff --git a/src/types/index.ts b/src/types/index.ts
index 6c16e0d..1e4d57d 100644
--- a/src/types/index.ts
+++ b/src/types/index.ts
@@ -22,9 +22,12 @@ export interface AIModelAPI {
   tokenizer: (text: string) => number;
   maxTokens: number;
 
-  send: (input: string, systemPrompt?: string) => AsyncRes<string>;
+  send: (
+    input: string | InputToken[],
+    systemPrompt?: string,
+  ) => AsyncRes<string>;
   stream: (
-    input: string,
+    input: string | InputToken[],
     handler: (data: string) => void,
     systemPrompt?: string,
   ) => void;