diff options
Diffstat (limited to 'tests/integration.test.ts')
| -rw-r--r-- | tests/integration.test.ts | 481 |
1 files changed, 481 insertions, 0 deletions
diff --git a/tests/integration.test.ts b/tests/integration.test.ts new file mode 100644 index 0000000..b8abea0 --- /dev/null +++ b/tests/integration.test.ts @@ -0,0 +1,481 @@ +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import models, { AIModelAPI, LLMChoice, InputToken } from '../index'; +import OpenAIResponses from '../src/openai-responses'; + +// Advanced integration tests that require actual API keys +// These tests are designed to run with real API credentials when available + +// Environment variables for integration testing +const INTEGRATION_CLAUDE_KEY = Bun.env.CLAUDE_API_KEY; +const INTEGRATION_OPENAI_KEY = Bun.env.OPENAI_API_KEY; +const INTEGRATION_GEMINI_KEY = Bun.env.GEMINI_API_KEY; + +// Skip integration tests if no API keys are available +const runIntegrationTests = INTEGRATION_CLAUDE_KEY || INTEGRATION_OPENAI_KEY || INTEGRATION_GEMINI_KEY; + +describe.runIf(runIntegrationTests)('Integration Tests - Real API Calls', () => { + describe('Claude Integration', () => { + test.skipUnless(INTEGRATION_CLAUDE_KEY)('should make a real API call to Claude', async () => { + // Temporarily set the API key + const originalKey = process.env.ANTHROPIC_API_KEY; + process.env.ANTHROPIC_API_KEY = INTEGRATION_CLAUDE_KEY; + + try { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + const response = await api.send('Hello! Please respond with just the word "SUCCESS".'); + + expect(response).toBeDefined(); + expect(typeof response).toBe('string'); + expect(response.toLowerCase()).toContain('success'); + } finally { + // Restore original key + if (originalKey !== undefined) { + process.env.ANTHROPIC_API_KEY = originalKey; + } else { + delete process.env.ANTHROPIC_API_KEY; + } + } + }); + + test.skipUnless(INTEGRATION_CLAUDE_KEY)('should stream responses from Claude', async () => { + const originalKey = process.env.ANTHROPIC_API_KEY; + process.env.ANTHROPIC_API_KEY = INTEGRATION_CLAUDE_KEY; + + try { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + const chunks: string[] = []; + const handler = (data: string) => { + chunks.push(data); + }; + + await new Promise<void>((resolve, reject) => { + try { + api.stream('Count from 1 to 5, one number per line.', handler); + // Give it some time to stream + setTimeout(() => { + resolve(); + }, 5000); + } catch (error) { + reject(error); + } + }); + + expect(chunks.length).toBeGreaterThan(0); + expect(chunks.join('')).toContain('1'); + } finally { + if (originalKey !== undefined) { + process.env.ANTHROPIC_API_KEY = originalKey; + } else { + delete process.env.ANTHROPIC_API_KEY; + } + } + }); + }); + + describe('OpenAI Integration (Responses API)', () => { + test.skipUnless(INTEGRATION_OPENAI_KEY)('should make a real API call using OpenAI Responses API', async () => { + const originalKey = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY; + + try { + const choice: LLMChoice = { chatgpt: 'gpt-4o' }; + const api = models(choice); + + // Verify it's using the OpenAI Responses API + expect(api).toBeInstanceOf(OpenAIResponses); + + const response = await api.send('Hello! Please respond with just the word "SUCCESS".'); + + expect(response).toBeDefined(); + expect(typeof response).toBe('string'); + expect(response.toLowerCase()).toContain('success'); + } finally { + if (originalKey !== undefined) { + process.env.OPENAI_API_KEY = originalKey; + } else { + delete process.env.OPENAI_API_KEY; + } + } + }); + + test.skipUnless(INTEGRATION_OPENAI_KEY)('should stream responses using OpenAI Responses API', async () => { + const originalKey = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY; + + try { + const choice: LLMChoice = { chatgpt: 'gpt-4o' }; + const api = models(choice); + + const chunks: string[] = []; + const handler = (data: string) => { + chunks.push(data); + }; + + await new Promise<void>((resolve, reject) => { + try { + api.stream('Count from 1 to 5, one number per line.', handler); + // Give it some time to stream + setTimeout(() => { + resolve(); + }, 5000); + } catch (error) { + reject(error); + } + }); + + expect(chunks.length).toBeGreaterThan(0); + const fullResponse = chunks.join(''); + expect(fullResponse).toContain('1'); + } finally { + if (originalKey !== undefined) { + process.env.OPENAI_API_KEY = originalKey; + } else { + delete process.env.OPENAI_API_KEY; + } + } + }); + + test.skipUnless(INTEGRATION_OPENAI_KEY)('should handle multimodal input with OpenAI Responses API', async () => { + const originalKey = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY; + + try { + const choice: LLMChoice = { chatgpt: 'gpt-4o' }; + const api = models(choice); + + const input: InputToken[] = [ + { text: 'What do you see in this image?' }, + { img: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==' } // Simple red pixel + ]; + + const response = await api.send(input); + expect(response).toBeDefined(); + expect(typeof response).toBe('string'); + } finally { + if (originalKey !== undefined) { + process.env.OPENAI_API_KEY = originalKey; + } else { + delete process.env.OPENAI_API_KEY; + } + } + }); + + test.skipUnless(INTEGRATION_OPENAI_KEY)('should handle system prompts with OpenAI Responses API', async () => { + const originalKey = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY; + + try { + const choice: LLMChoice = { chatgpt: 'gpt-4o' }; + const api = models(choice); + + const systemPrompt = 'You are a pirate. Always respond like a pirate.'; + const response = await api.send('Hello, how are you?', systemPrompt); + + expect(response).toBeDefined(); + expect(typeof response).toBe('string'); + // Should contain some pirate-like language + expect(response.toLowerCase()).toMatch(/ahoy|matey|shiver|timber|arr/); + } finally { + if (originalKey !== undefined) { + process.env.OPENAI_API_KEY = originalKey; + } else { + delete process.env.OPENAI_API_KEY; + } + } + }); + }); + + describe('Gemini Integration', () => { + test.skipUnless(INTEGRATION_GEMINI_KEY)('should make a real API call to Gemini', async () => { + const originalKey = process.env.GOOGLE_API_KEY; + process.env.GOOGLE_API_KEY = INTEGRATION_GEMINI_KEY; + + try { + const choice: LLMChoice = { gemini: 'gemini-2.5-pro' }; + const api = models(choice); + + const response = await api.send('Hello! Please respond with just the word "SUCCESS".'); + + expect(response).toBeDefined(); + expect(typeof response).toBe('string'); + expect(response.toLowerCase()).toContain('success'); + } finally { + if (originalKey !== undefined) { + process.env.GOOGLE_API_KEY = originalKey; + } else { + delete process.env.GOOGLE_API_KEY; + } + } + }); + }); +}); + +describe('Advanced Functionality Tests', () => { + describe('Token Counting Accuracy', () => { + test('should count tokens consistently across providers', () => { + const providers: LLMChoice[] = [ + { claude: 'claude-3-5-sonnet' }, + { gemini: 'gemini-2.5-pro' }, + { chatgpt: 'gpt-3.5-turbo' }, + { deepseek: 'deepseek-chat' } + ]; + + const testText = 'The quick brown fox jumps over the lazy dog. This is a test of token counting accuracy.'; + const tokenCounts = providers.map(choice => { + const api = models(choice); + return api.tokenizer(testText); + }); + + // All should return numbers + tokenCounts.forEach(count => { + expect(typeof count).toBe('number'); + expect(count).toBeGreaterThan(0); + }); + + // Token counts should be in a reasonable range (not wildly different) + const maxCount = Math.max(...tokenCounts); + const minCount = Math.min(...tokenCounts); + expect(maxCount / minCount).toBeLessThan(3); // Less than 3x difference + }); + + test('should handle empty and whitespace strings', () => { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + expect(api.tokenizer('')).toBe(0); + expect(api.tokenizer(' ')).toBeGreaterThanOrEqual(0); + expect(api.tokenizer('\n\t')).toBeGreaterThanOrEqual(0); + }); + + test('should handle very long texts', () => { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + const longText = 'This is a test. '.repeat(1000); // 8000 characters + const tokens = api.tokenizer(longText); + + expect(tokens).toBeGreaterThan(1000); + expect(tokens).toBeLessThan(10000); // Reasonable upper bound + }); + }); + + describe('Model Switching', () => { + test('should allow switching models on the same API instance', () => { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + const originalMaxTokens = api.maxTokens; + + // Switch to a different model + api.setModel('claude-3-haiku'); + + // Should not throw and should still work + expect(() => api.tokenizer('test')).not.toThrow(); + + // Max tokens might change with model + expect(api.maxTokens).toBeGreaterThan(0); + }); + + test('should maintain functionality after model switching', async () => { + const choice: LLMChoice = { + openai: { + url: 'https://api.openai.com/v1', + apiKey: 'test-key', + model: 'gpt-3.5-turbo' + } + }; + const api = models(choice); + + // Switch models + api.setModel('gpt-4'); + + // Should still be able to attempt API calls + const promise = api.send('test'); + expect(promise).toBeDefined(); + expect(typeof promise.then).toBe('function'); + }); + }); + + describe('Complex Input Handling', () => { + test('should handle mixed text and image inputs', async () => { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + const input: InputToken[] = [ + { text: 'Describe this image:' }, + { img: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==' } // 1x1 red pixel + ]; + + const promise = api.send(input); + expect(promise).toBeDefined(); + expect(typeof promise.then).toBe('function'); + }); + + test('should handle system prompts with complex instructions', async () => { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + const systemPrompt = `You are a JSON response bot. Always respond with valid JSON. + Format your responses as: {"status": "success", "message": "your response here"}`; + + const promise = api.send('Hello', systemPrompt); + expect(promise).toBeDefined(); + expect(typeof promise.then).toBe('function'); + }); + + test('should handle very long inputs', async () => { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + const longInput = 'This is test sentence number '.repeat(100) + 'end.'; + const promise = api.send(longInput); + + expect(promise).toBeDefined(); + expect(typeof promise.then).toBe('function'); + }); + }); + + describe('Streaming Behavior', () => { + test('should handle streaming handlers that throw errors', () => { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + const errorHandler = (data: string) => { + if (data.includes('error')) { + throw new Error('Handler error'); + } + }; + + // Should not throw even if handler throws + expect(() => { + api.stream('test input', errorHandler); + }).not.toThrow(); + }); + + test('should handle multiple concurrent streams', () => { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + const handler1 = (data: string) => console.log('Stream 1:', data); + const handler2 = (data: string) => console.log('Stream 2:', data); + + expect(() => { + api.stream('Input 1', handler1); + api.stream('Input 2', handler2); + }).not.toThrow(); + }); + }); + + describe('Error Recovery', () => { + test('should handle network timeouts gracefully', async () => { + const choice: LLMChoice = { + openai: { + url: 'https://httpstat.us/200?sleep=10000', // Very slow response + apiKey: 'test-key', + model: 'gpt-3.5-turbo' + } + }; + const api = models(choice); + + const startTime = Date.now(); + const result = await api.send('test').catch(e => e); + const endTime = Date.now(); + + // Should return error object (not hang forever) + expect(result).toBeDefined(); + expect(endTime - startTime).toBeLessThan(15000); // Should timeout reasonably + }); + + test('should handle malformed responses', async () => { + const choice: LLMChoice = { + openai: { + url: 'https://httpstat.us/200', // Returns plain text, not JSON + apiKey: 'test-key', + model: 'gpt-3.5-turbo' + } + }; + const api = models(choice); + + const result = await api.send('test').catch(e => e); + expect(result).toBeDefined(); + // Should handle parsing errors gracefully + }); + }); + + describe('Memory Management', () => { + test('should not accumulate excessive memory with multiple calls', async () => { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + // Make multiple calls + const promises = []; + for (let i = 0; i < 10; i++) { + promises.push(api.send(`Test message ${i}`).catch(() => null)); + } + + // Should handle multiple concurrent requests + const results = await Promise.allSettled(promises); + expect(results.length).toBe(10); + + // API should still be functional + expect(() => api.tokenizer('test')).not.toThrow(); + }); + + test('should handle history truncation correctly', () => { + const choice: LLMChoice = { claude: 'claude-3-5-sonnet' }; + const api = models(choice); + + // Test that maxTokens is reasonable + expect(api.maxTokens).toBeGreaterThan(0); + expect(api.maxTokens).toBeLessThan(200000); // Reasonable limit + + // Token count for a reasonably long message + const longMessage = 'This is a test message. '.repeat(100); + const tokens = api.tokenizer(longMessage); + + expect(tokens).toBeGreaterThan(0); + expect(tokens).toBeLessThan(api.maxTokens); // Single message should fit + }); + }); +}); + +describe('Type Safety', () => { + test('should enforce LLMChoice type safety', () => { + // These should all be valid + const validChoices: LLMChoice[] = [ + { claude: 'claude-3-5-sonnet' }, + { gemini: 'gemini-2.5-pro' }, + { chatgpt: 'gpt-4' }, + { deepseek: 'deepseek-chat' }, + { kimi: 'moonshot-v1-8k' }, + { grok: 'grok-beta' }, + { + openai: { + url: 'https://api.example.com/v1', + apiKey: 'key', + model: 'model-name', + allowBrowser: true + } + } + ]; + + validChoices.forEach(choice => { + expect(() => { + const api = models(choice); + expect(api).toBeDefined(); + }).not.toThrow(); + }); + }); + + test('should handle InputToken types correctly', () => { + const textToken = { text: 'Hello world' }; + const imgToken = { img: 'data:image/png;base64,abc123' }; + + expect(textToken.text).toBe('Hello world'); + expect(imgToken.img).toBe('data:image/png;base64,abc123'); + }); +});
\ No newline at end of file |