path: root/tests/integration.test.ts

import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import models, { AIModelAPI, LLMChoice, InputToken } from '../index';
import OpenAIResponses from '../src/openai-responses';

// Advanced integration tests that require actual API keys
// These tests are designed to run with real API credentials when available

// Environment variables for integration testing
const INTEGRATION_CLAUDE_KEY = Bun.env.CLAUDE_API_KEY;
const INTEGRATION_OPENAI_KEY = Bun.env.OPENAI_API_KEY;
const INTEGRATION_GEMINI_KEY = Bun.env.GEMINI_API_KEY;

// Skip integration tests if no API keys are available
const runIntegrationTests = INTEGRATION_CLAUDE_KEY || INTEGRATION_OPENAI_KEY || INTEGRATION_GEMINI_KEY;

describe.runIf(runIntegrationTests)('Integration Tests - Real API Calls', () => {
  describe('Claude Integration', () => {
    test.skipUnless(INTEGRATION_CLAUDE_KEY)('should make a real API call to Claude', async () => {
      // Temporarily set the API key
      const originalKey = process.env.ANTHROPIC_API_KEY;
      process.env.ANTHROPIC_API_KEY = INTEGRATION_CLAUDE_KEY;

      try {
        const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
        const api = models(choice);

        const response = await api.send('Hello! Please respond with just the word "SUCCESS".');

        expect(response).toBeDefined();
        expect(typeof response).toBe('string');
        expect(response.toLowerCase()).toContain('success');
      } finally {
        // Restore original key
        if (originalKey !== undefined) {
          process.env.ANTHROPIC_API_KEY = originalKey;
        } else {
          delete process.env.ANTHROPIC_API_KEY;
        }
      }
    });

    test.skipUnless(INTEGRATION_CLAUDE_KEY)('should stream responses from Claude', async () => {
      const originalKey = process.env.ANTHROPIC_API_KEY;
      process.env.ANTHROPIC_API_KEY = INTEGRATION_CLAUDE_KEY;

      try {
        const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
        const api = models(choice);

        const chunks: string[] = [];
        const handler = (data: string) => {
          chunks.push(data);
        };

        await new Promise<void>((resolve, reject) => {
          try {
            api.stream('Count from 1 to 5, one number per line.', handler);
            // Give it some time to stream
            setTimeout(() => {
              resolve();
            }, 5000);
          } catch (error) {
            reject(error);
          }
        });

        expect(chunks.length).toBeGreaterThan(0);
        expect(chunks.join('')).toContain('1');
      } finally {
        if (originalKey !== undefined) {
          process.env.ANTHROPIC_API_KEY = originalKey;
        } else {
          delete process.env.ANTHROPIC_API_KEY;
        }
      }
    });
  });

  describe('OpenAI Integration (Responses API)', () => {
    test.skipUnless(INTEGRATION_OPENAI_KEY)('should make a real API call using OpenAI Responses API', async () => {
      const originalKey = process.env.OPENAI_API_KEY;
      process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY;

      try {
        const choice: LLMChoice = { chatgpt: 'gpt-4o' };
        const api = models(choice);

        // Verify it's using the OpenAI Responses API
        expect(api).toBeInstanceOf(OpenAIResponses);

        const response = await api.send('Hello! Please respond with just the word "SUCCESS".');

        expect(response).toBeDefined();
        expect(typeof response).toBe('string');
        expect(response.toLowerCase()).toContain('success');
      } finally {
        if (originalKey !== undefined) {
          process.env.OPENAI_API_KEY = originalKey;
        } else {
          delete process.env.OPENAI_API_KEY;
        }
      }
    });

    test.skipUnless(INTEGRATION_OPENAI_KEY)('should stream responses using OpenAI Responses API', async () => {
      const originalKey = process.env.OPENAI_API_KEY;
      process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY;

      try {
        const choice: LLMChoice = { chatgpt: 'gpt-4o' };
        const api = models(choice);

        const chunks: string[] = [];
        const handler = (data: string) => {
          chunks.push(data);
        };

        await new Promise<void>((resolve, reject) => {
          try {
            api.stream('Count from 1 to 5, one number per line.', handler);
            // Give it some time to stream
            setTimeout(() => {
              resolve();
            }, 5000);
          } catch (error) {
            reject(error);
          }
        });

        expect(chunks.length).toBeGreaterThan(0);
        const fullResponse = chunks.join('');
        expect(fullResponse).toContain('1');
      } finally {
        if (originalKey !== undefined) {
          process.env.OPENAI_API_KEY = originalKey;
        } else {
          delete process.env.OPENAI_API_KEY;
        }
      }
    });

    test.skipUnless(INTEGRATION_OPENAI_KEY)('should handle multimodal input with OpenAI Responses API', async () => {
      const originalKey = process.env.OPENAI_API_KEY;
      process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY;

      try {
        const choice: LLMChoice = { chatgpt: 'gpt-4o' };
        const api = models(choice);

        const input: InputToken[] = [
          { text: 'What do you see in this image?' },
          { img: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==' } // Simple red pixel
        ];

        const response = await api.send(input);
        expect(response).toBeDefined();
        expect(typeof response).toBe('string');
      } finally {
        if (originalKey !== undefined) {
          process.env.OPENAI_API_KEY = originalKey;
        } else {
          delete process.env.OPENAI_API_KEY;
        }
      }
    });

    test.skipUnless(INTEGRATION_OPENAI_KEY)('should handle system prompts with OpenAI Responses API', async () => {
      const originalKey = process.env.OPENAI_API_KEY;
      process.env.OPENAI_API_KEY = INTEGRATION_OPENAI_KEY;

      try {
        const choice: LLMChoice = { chatgpt: 'gpt-4o' };
        const api = models(choice);

        const systemPrompt = 'You are a pirate. Always respond like a pirate.';
        const response = await api.send('Hello, how are you?', systemPrompt);

        expect(response).toBeDefined();
        expect(typeof response).toBe('string');
        // Should contain some pirate-like language
        expect(response.toLowerCase()).toMatch(/ahoy|matey|shiver|timber|arr/);
      } finally {
        if (originalKey !== undefined) {
          process.env.OPENAI_API_KEY = originalKey;
        } else {
          delete process.env.OPENAI_API_KEY;
        }
      }
    });
  });

  describe('Gemini Integration', () => {
    test.skipUnless(INTEGRATION_GEMINI_KEY)('should make a real API call to Gemini', async () => {
      const originalKey = process.env.GOOGLE_API_KEY;
      process.env.GOOGLE_API_KEY = INTEGRATION_GEMINI_KEY;

      try {
        const choice: LLMChoice = { gemini: 'gemini-2.5-pro' };
        const api = models(choice);

        const response = await api.send('Hello! Please respond with just the word "SUCCESS".');

        expect(response).toBeDefined();
        expect(typeof response).toBe('string');
        expect(response.toLowerCase()).toContain('success');
      } finally {
        if (originalKey !== undefined) {
          process.env.GOOGLE_API_KEY = originalKey;
        } else {
          delete process.env.GOOGLE_API_KEY;
        }
      }
    });
  });
});

describe('Advanced Functionality Tests', () => {
  describe('Token Counting Accuracy', () => {
    test('should count tokens consistently across providers', () => {
      const providers: LLMChoice[] = [
        { claude: 'claude-3-5-sonnet' },
        { gemini: 'gemini-2.5-pro' },
        { chatgpt: 'gpt-3.5-turbo' },
        { deepseek: 'deepseek-chat' }
      ];

      const testText = 'The quick brown fox jumps over the lazy dog. This is a test of token counting accuracy.';
      const tokenCounts = providers.map(choice => {
        const api = models(choice);
        return api.tokenizer(testText);
      });

      // All should return numbers
      tokenCounts.forEach(count => {
        expect(typeof count).toBe('number');
        expect(count).toBeGreaterThan(0);
      });

      // Token counts should be in a reasonable range (not wildly different)
      const maxCount = Math.max(...tokenCounts);
      const minCount = Math.min(...tokenCounts);
      expect(maxCount / minCount).toBeLessThan(3); // Less than 3x difference
    });

    test('should handle empty and whitespace strings', () => {
      const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
      const api = models(choice);

      expect(api.tokenizer('')).toBe(0);
      expect(api.tokenizer('   ')).toBeGreaterThanOrEqual(0);
      expect(api.tokenizer('\n\t')).toBeGreaterThanOrEqual(0);
    });

    test('should handle very long texts', () => {
      const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
      const api = models(choice);

      const longText = 'This is a test. '.repeat(1000); // 8000 characters
      const tokens = api.tokenizer(longText);

      expect(tokens).toBeGreaterThan(1000);
      expect(tokens).toBeLessThan(10000); // Reasonable upper bound
    });
  });

  describe('Model Switching', () => {
    test('should allow switching models on the same API instance', () => {
      const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
      const api = models(choice);

      const originalMaxTokens = api.maxTokens;

      // Switch to a different model
      api.setModel('claude-3-haiku');

      // Should not throw and should still work
      expect(() => api.tokenizer('test')).not.toThrow();

      // Max tokens might change with model
      expect(api.maxTokens).toBeGreaterThan(0);
    });

    test('should maintain functionality after model switching', async () => {
      const choice: LLMChoice = {
        openai: {
          url: 'https://api.openai.com/v1',
          apiKey: 'test-key',
          model: 'gpt-3.5-turbo'
        }
      };
      const api = models(choice);

      // Switch models
      api.setModel('gpt-4');

      // Should still be able to attempt API calls
      const promise = api.send('test');
      expect(promise).toBeDefined();
      expect(typeof promise.then).toBe('function');
    });
  });

  describe('Complex Input Handling', () => {
    test('should handle mixed text and image inputs', async () => {
      const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
      const api = models(choice);

      const input: InputToken[] = [
        { text: 'Describe this image:' },
        { img: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==' } // 1x1 red pixel
      ];

      const promise = api.send(input);
      expect(promise).toBeDefined();
      expect(typeof promise.then).toBe('function');
    });

    test('should handle system prompts with complex instructions', async () => {
      const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
      const api = models(choice);

      const systemPrompt = `You are a JSON response bot. Always respond with valid JSON.
      Format your responses as: {"status": "success", "message": "your response here"}`;

      const promise = api.send('Hello', systemPrompt);
      expect(promise).toBeDefined();
      expect(typeof promise.then).toBe('function');
    });

    test('should handle very long inputs', async () => {
      const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
      const api = models(choice);

      const longInput = 'This is test sentence number '.repeat(100) + 'end.';
      const promise = api.send(longInput);

      expect(promise).toBeDefined();
      expect(typeof promise.then).toBe('function');
    });
  });

  describe('Streaming Behavior', () => {
    test('should handle streaming handlers that throw errors', () => {
      const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
      const api = models(choice);

      const errorHandler = (data: string) => {
        if (data.includes('error')) {
          throw new Error('Handler error');
        }
      };

      // Should not throw even if handler throws
      expect(() => {
        api.stream('test input', errorHandler);
      }).not.toThrow();
    });

    test('should handle multiple concurrent streams', () => {
      const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
      const api = models(choice);

      const handler1 = (data: string) => console.log('Stream 1:', data);
      const handler2 = (data: string) => console.log('Stream 2:', data);

      expect(() => {
        api.stream('Input 1', handler1);
        api.stream('Input 2', handler2);
      }).not.toThrow();
    });
  });

  describe('Error Recovery', () => {
    test('should handle network timeouts gracefully', async () => {
      const choice: LLMChoice = {
        openai: {
          url: 'https://httpstat.us/200?sleep=10000', // Very slow response
          apiKey: 'test-key',
          model: 'gpt-3.5-turbo'
        }
      };
      const api = models(choice);

      const startTime = Date.now();
      const result = await api.send('test').catch(e => e);
      const endTime = Date.now();

      // Should return error object (not hang forever)
      expect(result).toBeDefined();
      expect(endTime - startTime).toBeLessThan(15000); // Should timeout reasonably
    });

    test('should handle malformed responses', async () => {
      const choice: LLMChoice = {
        openai: {
          url: 'https://httpstat.us/200', // Returns plain text, not JSON
          apiKey: 'test-key',
          model: 'gpt-3.5-turbo'
        }
      };
      const api = models(choice);

      const result = await api.send('test').catch(e => e);
      expect(result).toBeDefined();
      // Should handle parsing errors gracefully
    });
  });

  describe('Memory Management', () => {
    test('should not accumulate excessive memory with multiple calls', async () => {
      const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
      const api = models(choice);

      // Make multiple calls
      const promises = [];
      for (let i = 0; i < 10; i++) {
        promises.push(api.send(`Test message ${i}`).catch(() => null));
      }

      // Should handle multiple concurrent requests
      const results = await Promise.allSettled(promises);
      expect(results.length).toBe(10);

      // API should still be functional
      expect(() => api.tokenizer('test')).not.toThrow();
    });

    test('should handle history truncation correctly', () => {
      const choice: LLMChoice = { claude: 'claude-3-5-sonnet' };
      const api = models(choice);

      // Test that maxTokens is reasonable
      expect(api.maxTokens).toBeGreaterThan(0);
      expect(api.maxTokens).toBeLessThan(200000); // Reasonable limit

      // Token count for a reasonably long message
      const longMessage = 'This is a test message. '.repeat(100);
      const tokens = api.tokenizer(longMessage);

      expect(tokens).toBeGreaterThan(0);
      expect(tokens).toBeLessThan(api.maxTokens); // Single message should fit
    });
  });
});

describe('Type Safety', () => {
  test('should enforce LLMChoice type safety', () => {
    // These should all be valid
    const validChoices: LLMChoice[] = [
      { claude: 'claude-3-5-sonnet' },
      { gemini: 'gemini-2.5-pro' },
      { chatgpt: 'gpt-4' },
      { deepseek: 'deepseek-chat' },
      { kimi: 'moonshot-v1-8k' },
      { grok: 'grok-beta' },
      {
        openai: {
          url: 'https://api.example.com/v1',
          apiKey: 'key',
          model: 'model-name',
          allowBrowser: true
        }
      }
    ];

    validChoices.forEach(choice => {
      expect(() => {
        const api = models(choice);
        expect(api).toBeDefined();
      }).not.toThrow();
    });
  });

  test('should handle InputToken types correctly', () => {
    const textToken = { text: 'Hello world' };
    const imgToken = { img: 'data:image/png;base64,abc123' };

    expect(textToken.text).toBe('Hello world');
    expect(imgToken.img).toBe('data:image/png;base64,abc123');
  });
});