import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; import models, { AIModelAPI, LLMChoice, InputToken } from '../index'; import OpenAIResponses from '../src/openai-responses'; // Advanced integration tests that require actual API keys // These tests are designed to run with real API credentials when available // Environment variables for integration testing const INTEGRATION_CLAUDE_KEY = Bun.env.CLAUDE_API_KEY; const INTEGRATION_OPENAI_KEY = Bun.env.OPENAI_API_KEY; const INTEGRATION_GEMINI_KEY = Bun.env.GEMINI_API_KEY; // Skip integration tests if no API keys are available const runIntegrationTests = INTEGRATION_CLAUDE_KEY || INTEGRATION_OPENAI_KEY || INTEGRATION_GEMINI_KEY; describe.runIf(runIntegrationTests)('Integration Tests - Real API Calls', () => { describe('Claude Integration', () => { test.skipUnless(INTEGRATION_CLAUDE_KEY)('should make a real API call to Claude', async () => { // Temporarily set the API key const originalKey = process.env.ANTHROPIC_API_KEY; process.env.ANTHROPIC_API_KEY = INTEGRATION_CLAUDE_KEY; try { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); const response = await api.send('Hello! Please respond with just the word "SUCCESS".'); expect(response).toBeDefined(); expect(typeof response).toBe('string'); expect(response.toLowerCase()).toContain('success'); } finally { // Restore original key if (originalKey !== undefined) { process.env.ANTHROPIC_API_KEY = originalKey; } else { delete process.env.ANTHROPIC_API_KEY; } } }); test.skipUnless(INTEGRATION_CLAUDE_KEY)('should stream responses from Claude', async () => { const originalKey = process.env.ANTHROPIC_API_KEY; process.env.ANTHROPIC_API_KEY = INTEGRATION_CLAUDE_KEY; try { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); const chunks: string[] = []; const handler = (data: string) => { chunks.push(data); }; await new Promise((resolve, reject) => { try { api.stream('Count from 1 to 5, one number per line.', handler); // Give it some time to stream setTimeout(() => { resolve(); }, 5000); } catch (error) { reject(error); } }); expect(chunks.length).toBeGreaterThan(0); expect(chunks.join('')).toContain('1'); } finally { if (originalKey !== undefined) { process.env.ANTHROPIC_API_KEY = originalKey; } else { delete process.env.ANTHROPIC_API_KEY; } } }); }); describe('OpenAI Integration (Responses API)', () => { test.skipUnless(INTEGRATION_OPENAI_KEY)('should make a real API call using OpenAI Responses API', async () => { const originalKey = process.env.OPENAI_API_KEY; process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY; try { const choice: LLMChoice = { chatgpt: 'gpt-4o' }; const api = models(choice); // Verify it's using the OpenAI Responses API expect(api).toBeInstanceOf(OpenAIResponses); const response = await api.send('Hello! Please respond with just the word "SUCCESS".'); expect(response).toBeDefined(); expect(typeof response).toBe('string'); expect(response.toLowerCase()).toContain('success'); } finally { if (originalKey !== undefined) { process.env.OPENAI_API_KEY = originalKey; } else { delete process.env.OPENAI_API_KEY; } } }); test.skipUnless(INTEGRATION_OPENAI_KEY)('should stream responses using OpenAI Responses API', async () => { const originalKey = process.env.OPENAI_API_KEY; process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY; try { const choice: LLMChoice = { chatgpt: 'gpt-4o' }; const api = models(choice); const chunks: string[] = []; const handler = (data: string) => { chunks.push(data); }; await new Promise((resolve, reject) => { try { api.stream('Count from 1 to 5, one number per line.', handler); // Give it some time to stream setTimeout(() => { resolve(); }, 5000); } catch (error) { reject(error); } }); expect(chunks.length).toBeGreaterThan(0); const fullResponse = chunks.join(''); expect(fullResponse).toContain('1'); } finally { if (originalKey !== undefined) { process.env.OPENAI_API_KEY = originalKey; } else { delete process.env.OPENAI_API_KEY; } } }); test.skipUnless(INTEGRATION_OPENAI_KEY)('should handle multimodal input with OpenAI Responses API', async () => { const originalKey = process.env.OPENAI_API_KEY; process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY; try { const choice: LLMChoice = { chatgpt: 'gpt-4o' }; const api = models(choice); const input: InputToken[] = [ { text: 'What do you see in this image?' }, { img: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==' } // Simple red pixel ]; const response = await api.send(input); expect(response).toBeDefined(); expect(typeof response).toBe('string'); } finally { if (originalKey !== undefined) { process.env.OPENAI_API_KEY = originalKey; } else { delete process.env.OPENAI_API_KEY; } } }); test.skipUnless(INTEGRATION_OPENAI_KEY)('should handle system prompts with OpenAI Responses API', async () => { const originalKey = process.env.OPENAI_API_KEY; process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY; try { const choice: LLMChoice = { chatgpt: 'gpt-4o' }; const api = models(choice); const systemPrompt = 'You are a pirate. Always respond like a pirate.'; const response = await api.send('Hello, how are you?', systemPrompt); expect(response).toBeDefined(); expect(typeof response).toBe('string'); // Should contain some pirate-like language expect(response.toLowerCase()).toMatch(/ahoy|matey|shiver|timber|arr/); } finally { if (originalKey !== undefined) { process.env.OPENAI_API_KEY = originalKey; } else { delete process.env.OPENAI_API_KEY; } } }); }); describe('Gemini Integration', () => { test.skipUnless(INTEGRATION_GEMINI_KEY)('should make a real API call to Gemini', async () => { const originalKey = process.env.GOOGLE_API_KEY; process.env.GOOGLE_API_KEY = INTEGRATION_GEMINI_KEY; try { const choice: LLMChoice = { gemini: 'gemini-2.5-pro' }; const api = models(choice); const response = await api.send('Hello! Please respond with just the word "SUCCESS".'); expect(response).toBeDefined(); expect(typeof response).toBe('string'); expect(response.toLowerCase()).toContain('success'); } finally { if (originalKey !== undefined) { process.env.GOOGLE_API_KEY = originalKey; } else { delete process.env.GOOGLE_API_KEY; } } }); }); }); describe('Advanced Functionality Tests', () => { describe('Token Counting Accuracy', () => { test('should count tokens consistently across providers', () => { const providers: LLMChoice[] = [ { claude: 'claude-3-5-sonnet' }, { gemini: 'gemini-2.5-pro' }, { chatgpt: 'gpt-3.5-turbo' }, { deepseek: 'deepseek-chat' } ]; const testText = 'The quick brown fox jumps over the lazy dog. This is a test of token counting accuracy.'; const tokenCounts = providers.map(choice => { const api = models(choice); return api.tokenizer(testText); }); // All should return numbers tokenCounts.forEach(count => { expect(typeof count).toBe('number'); expect(count).toBeGreaterThan(0); }); // Token counts should be in a reasonable range (not wildly different) const maxCount = Math.max(...tokenCounts); const minCount = Math.min(...tokenCounts); expect(maxCount / minCount).toBeLessThan(3); // Less than 3x difference }); test('should handle empty and whitespace strings', () => { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); expect(api.tokenizer('')).toBe(0); expect(api.tokenizer(' ')).toBeGreaterThanOrEqual(0); expect(api.tokenizer('\n\t')).toBeGreaterThanOrEqual(0); }); test('should handle very long texts', () => { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); const longText = 'This is a test. '.repeat(1000); // 8000 characters const tokens = api.tokenizer(longText); expect(tokens).toBeGreaterThan(1000); expect(tokens).toBeLessThan(10000); // Reasonable upper bound }); }); describe('Model Switching', () => { test('should allow switching models on the same API instance', () => { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); const originalMaxTokens = api.maxTokens; // Switch to a different model api.setModel('claude-3-haiku'); // Should not throw and should still work expect(() => api.tokenizer('test')).not.toThrow(); // Max tokens might change with model expect(api.maxTokens).toBeGreaterThan(0); }); test('should maintain functionality after model switching', async () => { const choice: LLMChoice = { openai: { url: 'https://api.openai.com/v1', apiKey: 'test-key', model: 'gpt-3.5-turbo' } }; const api = models(choice); // Switch models api.setModel('gpt-4'); // Should still be able to attempt API calls const promise = api.send('test'); expect(promise).toBeDefined(); expect(typeof promise.then).toBe('function'); }); }); describe('Complex Input Handling', () => { test('should handle mixed text and image inputs', async () => { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); const input: InputToken[] = [ { text: 'Describe this image:' }, { img: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==' } // 1x1 red pixel ]; const promise = api.send(input); expect(promise).toBeDefined(); expect(typeof promise.then).toBe('function'); }); test('should handle system prompts with complex instructions', async () => { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); const systemPrompt = `You are a JSON response bot. Always respond with valid JSON. Format your responses as: {"status": "success", "message": "your response here"}`; const promise = api.send('Hello', systemPrompt); expect(promise).toBeDefined(); expect(typeof promise.then).toBe('function'); }); test('should handle very long inputs', async () => { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); const longInput = 'This is test sentence number '.repeat(100) + 'end.'; const promise = api.send(longInput); expect(promise).toBeDefined(); expect(typeof promise.then).toBe('function'); }); }); describe('Streaming Behavior', () => { test('should handle streaming handlers that throw errors', () => { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); const errorHandler = (data: string) => { if (data.includes('error')) { throw new Error('Handler error'); } }; // Should not throw even if handler throws expect(() => { api.stream('test input', errorHandler); }).not.toThrow(); }); test('should handle multiple concurrent streams', () => { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); const handler1 = (data: string) => console.log('Stream 1:', data); const handler2 = (data: string) => console.log('Stream 2:', data); expect(() => { api.stream('Input 1', handler1); api.stream('Input 2', handler2); }).not.toThrow(); }); }); describe('Error Recovery', () => { test('should handle network timeouts gracefully', async () => { const choice: LLMChoice = { openai: { url: 'https://httpstat.us/200?sleep=10000', // Very slow response apiKey: 'test-key', model: 'gpt-3.5-turbo' } }; const api = models(choice); const startTime = Date.now(); const result = await api.send('test').catch(e => e); const endTime = Date.now(); // Should return error object (not hang forever) expect(result).toBeDefined(); expect(endTime - startTime).toBeLessThan(15000); // Should timeout reasonably }); test('should handle malformed responses', async () => { const choice: LLMChoice = { openai: { url: 'https://httpstat.us/200', // Returns plain text, not JSON apiKey: 'test-key', model: 'gpt-3.5-turbo' } }; const api = models(choice); const result = await api.send('test').catch(e => e); expect(result).toBeDefined(); // Should handle parsing errors gracefully }); }); describe('Memory Management', () => { test('should not accumulate excessive memory with multiple calls', async () => { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); // Make multiple calls const promises = []; for (let i = 0; i < 10; i++) { promises.push(api.send(`Test message ${i}`).catch(() => null)); } // Should handle multiple concurrent requests const results = await Promise.allSettled(promises); expect(results.length).toBe(10); // API should still be functional expect(() => api.tokenizer('test')).not.toThrow(); }); test('should handle history truncation correctly', () => { const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; const api = models(choice); // Test that maxTokens is reasonable expect(api.maxTokens).toBeGreaterThan(0); expect(api.maxTokens).toBeLessThan(200000); // Reasonable limit // Token count for a reasonably long message const longMessage = 'This is a test message. '.repeat(100); const tokens = api.tokenizer(longMessage); expect(tokens).toBeGreaterThan(0); expect(tokens).toBeLessThan(api.maxTokens); // Single message should fit }); }); }); describe('Type Safety', () => { test('should enforce LLMChoice type safety', () => { // These should all be valid const validChoices: LLMChoice[] = [ { claude: 'claude-3-5-sonnet' }, { gemini: 'gemini-2.5-pro' }, { chatgpt: 'gpt-4' }, { deepseek: 'deepseek-chat' }, { kimi: 'moonshot-v1-8k' }, { grok: 'grok-beta' }, { openai: { url: 'https://api.example.com/v1', apiKey: 'key', model: 'model-name', allowBrowser: true } } ]; validChoices.forEach(choice => { expect(() => { const api = models(choice); expect(api).toBeDefined(); }).not.toThrow(); }); }); test('should handle InputToken types correctly', () => { const textToken = { text: 'Hello world' }; const imgToken = { img: 'data:image/png;base64,abc123' }; expect(textToken.text).toBe('Hello world'); expect(imgToken.img).toBe('data:image/png;base64,abc123'); }); });