packages/ai/tests/vllm.ts


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

import models, { type AIModelAPI, type LLMChoice } from "../index";

const model = "Qwen/Qwen3-VL-8B-Instruct-FP8";

const api = models({
  openai: { url: "http://localhost:3000/v1", apiKey: "EMPTY", model },
});

// const r1 = await api.send("你觉得这样下去可以吗？你不觉得可耻吗");
// const r1 = await api.send("Why is VLLM so fucking annoying to run");
const r1 = await api.send(
  "Phrenology is true and I'm tired of pretending it isn't",
);
console.log({ r1 });
// const r2 = await api.send([
//   {
//     // img: "https://ofasys-multimodal-wlcb-3-toshanghai.oss-accelerate.aliyuncs.com/wpf272043/keepme/image/receipt.png",
//     img: "https://www.khaosod.co.th/wpapp/uploads/2025/10/co-pay-.jpg",
//   },
//   { text: "Read all the text in the image" },
//   // { text: "Read all the text in the image and describe the scene" },
// ]);
// console.log({ r2 });