← back to Goodquestion
src/lib/ai-provider-multi.ts
321 lines
import Anthropic from '@anthropic-ai/sdk'
import { GoogleGenerativeAI } from '@google/generative-ai'
import { HfInference } from '@huggingface/inference'
/**
* Multi-provider AI system with automatic fallback
* Priority: Anthropic Claude > Google Gemini > Hugging Face (free)
*/
export interface AIResponse {
content: string
provider: 'anthropic' | 'gemini' | 'huggingface'
model?: string
tokensUsed?: number
}
export interface AIProviderStats {
provider: string
requestCount: number
totalTokens: number
lastError?: string
lastErrorTime?: Date
}
class MultiAIProviderManager {
private anthropic: Anthropic
private gemini: GoogleGenerativeAI
private hf: HfInference
private stats: Map<string, AIProviderStats> = new Map()
// Token tracking per minute
private tokenUsageWindow: { timestamp: number; tokens: number; provider: string }[] = []
// Thresholds
private readonly ANTHROPIC_TOKEN_LIMIT_PER_MINUTE = 4000
private readonly RATE_LIMIT_COOLDOWN_MS = 60000 // 1 minute
// Hugging Face models to try (in order of preference)
private readonly HF_MODELS = [
'deepseek-ai/DeepSeek-R1:fireworks-ai', // DeepSeek R1 via Fireworks - high quality reasoning
'mistralai/Mixtral-8x7B-Instruct-v0.1', // High quality, good for analysis
'meta-llama/Meta-Llama-3-8B-Instruct', // Fast and reliable
'microsoft/Phi-3-mini-4k-instruct', // Compact but capable
'HuggingFaceH4/zephyr-7b-beta', // Good general purpose
]
constructor() {
this.anthropic = new Anthropic({
apiKey: process.env.ANTHROPIC_API_KEY || '',
})
this.gemini = new GoogleGenerativeAI(
process.env.GOOGLE_API_KEY || ''
)
this.hf = new HfInference(
process.env.HUGGINGFACE_API_KEY || '' // Free tier works without key for many models
)
// Initialize stats
this.stats.set('anthropic', { provider: 'anthropic', requestCount: 0, totalTokens: 0 })
this.stats.set('gemini', { provider: 'gemini', requestCount: 0, totalTokens: 0 })
this.stats.set('huggingface', { provider: 'huggingface', requestCount: 0, totalTokens: 0 })
}
/**
* Track token usage in a rolling window
*/
private trackTokenUsage(provider: string, tokens: number) {
const now = Date.now()
this.tokenUsageWindow.push({ timestamp: now, tokens, provider })
// Remove entries older than 1 minute
this.tokenUsageWindow = this.tokenUsageWindow.filter(
entry => now - entry.timestamp < 60000
)
// Update stats
const stats = this.stats.get(provider)
if (stats) {
stats.requestCount++
stats.totalTokens += tokens
this.stats.set(provider, stats)
}
console.log(`[AI Provider] ${provider} used ${tokens} tokens. Total in last minute: ${this.getTokensInLastMinute(provider)}`)
}
/**
* Get total tokens used in the last minute for a provider
*/
private getTokensInLastMinute(provider: string): number {
const now = Date.now()
return this.tokenUsageWindow
.filter(entry => entry.provider === provider && now - entry.timestamp < 60000)
.reduce((sum, entry) => sum + entry.tokens, 0)
}
/**
* Check if a provider is currently rate limited
*/
private isProviderRateLimited(provider: string): boolean {
const stats = this.stats.get(provider)
if (!stats?.lastError) return false
// Check if the last error was a rate limit and within cooldown period
if (stats.lastError.includes('rate_limit') ||
stats.lastError.includes('credit balance') ||
stats.lastError.includes('429') ||
stats.lastError.includes('quota')) {
const timeSinceError = stats.lastErrorTime ? Date.now() - stats.lastErrorTime.getTime() : Infinity
return timeSinceError < this.RATE_LIMIT_COOLDOWN_MS
}
return false
}
/**
* Check if we're approaching Anthropic's token limit
*/
private isApproachingTokenLimit(): boolean {
const tokensUsed = this.getTokensInLastMinute('anthropic')
// Use 80% of limit as threshold
return tokensUsed >= this.ANTHROPIC_TOKEN_LIMIT_PER_MINUTE * 0.8
}
/**
* Determine which provider to use
* Priority: Anthropic (best quality) > Hugging Face (free) > Gemini (backup)
*/
private selectProvider(): 'anthropic' | 'gemini' | 'huggingface' {
// Use Anthropic first (highest quality, API key is working)
if (!this.isProviderRateLimited('anthropic') && !this.isApproachingTokenLimit() && process.env.ANTHROPIC_API_KEY) {
console.log('[AI Provider] Using Anthropic (primary provider)')
return 'anthropic'
}
// Try Hugging Face second (free)
if (!this.isProviderRateLimited('huggingface')) {
console.log('[AI Provider] Using Hugging Face (Anthropic unavailable)')
return 'huggingface'
}
// Fallback to Gemini if configured
if (!this.isProviderRateLimited('gemini') && process.env.GOOGLE_API_KEY) {
console.log('[AI Provider] Using Gemini (other providers unavailable)')
return 'gemini'
}
// Last resort: try Anthropic anyway
console.log('[AI Provider] All providers rate limited, retrying Anthropic')
return 'anthropic'
}
/**
* Try Hugging Face with multiple models using chatCompletion API
*/
private async tryHuggingFaceModels(prompt: string, maxTokens: number): Promise<AIResponse> {
let lastError: Error | null = null
for (const model of this.HF_MODELS) {
try {
console.log(`[AI Provider] Trying Hugging Face model: ${model}`)
const response = await this.hf.chatCompletion({
model,
messages: [{ role: 'user', content: prompt }],
max_tokens: maxTokens,
temperature: 0.7,
})
const content = response.choices?.[0]?.message?.content || ''
const estimatedTokens = Math.ceil(content.length / 4)
this.trackTokenUsage('huggingface', estimatedTokens)
return {
content,
provider: 'huggingface',
model,
tokensUsed: estimatedTokens,
}
} catch (error: any) {
console.log(`[AI Provider] HF model ${model} failed: ${error.message}`)
lastError = error
// Try next model
continue
}
}
throw lastError || new Error('All Hugging Face models failed')
}
/**
* Generate a completion using the best available provider
*/
async generateCompletion(
prompt: string,
options: {
maxTokens?: number
temperature?: number
forceProvider?: 'anthropic' | 'gemini' | 'huggingface'
} = {}
): Promise<AIResponse> {
const provider = options.forceProvider || this.selectProvider()
const maxTokens = options.maxTokens || 5000
const temperature = options.temperature || 1.0
try {
if (provider === 'anthropic') {
const message = await this.anthropic.messages.create({
model: 'claude-opus-4-8',
max_tokens: maxTokens,
temperature,
messages: [{ role: 'user', content: prompt }],
})
const content = message.content[0].type === 'text' ? message.content[0].text : ''
const tokensUsed = message.usage?.output_tokens || 0
this.trackTokenUsage('anthropic', tokensUsed)
return {
content,
provider: 'anthropic',
tokensUsed,
}
} else if (provider === 'gemini') {
const model = this.gemini.getGenerativeModel({
model: 'gemini-pro',
generationConfig: {
maxOutputTokens: maxTokens,
temperature,
},
})
const result = await model.generateContent(prompt)
const response = result.response
const content = response.text()
const estimatedTokens = Math.ceil(content.length / 4)
this.trackTokenUsage('gemini', estimatedTokens)
return {
content,
provider: 'gemini',
tokensUsed: estimatedTokens,
}
} else {
// Try Hugging Face with multiple models
return await this.tryHuggingFaceModels(prompt, maxTokens)
}
} catch (error: any) {
// Record error for rate limiting detection
const stats = this.stats.get(provider)
if (stats) {
stats.lastError = error?.message || String(error)
stats.lastErrorTime = new Date()
this.stats.set(provider, stats)
}
// If primary provider fails, try next in chain
if (!options.forceProvider) {
const errorMsg = error?.message || String(error)
console.log(`[AI Provider] ${provider} failed (${errorMsg}), trying fallback`)
// Anthropic failed -> try Hugging Face
if (provider === 'anthropic') {
return this.generateCompletion(prompt, { ...options, forceProvider: 'huggingface' })
}
// Hugging Face failed -> try Gemini if available, otherwise Anthropic
if (provider === 'huggingface') {
if (process.env.GOOGLE_API_KEY) {
return this.generateCompletion(prompt, { ...options, forceProvider: 'gemini' })
} else if (process.env.ANTHROPIC_API_KEY) {
return this.generateCompletion(prompt, { ...options, forceProvider: 'anthropic' })
}
}
// Gemini failed -> try Anthropic
if (provider === 'gemini') {
if (process.env.ANTHROPIC_API_KEY) {
return this.generateCompletion(prompt, { ...options, forceProvider: 'anthropic' })
}
}
}
throw error
}
}
/**
* Get current provider statistics
*/
getStats(): Record<string, AIProviderStats> {
const result: Record<string, AIProviderStats> = {}
this.stats.forEach((value, key) => {
result[key] = { ...value }
})
return result
}
/**
* Reset statistics
*/
resetStats() {
this.stats.forEach((value, key) => {
this.stats.set(key, {
provider: value.provider,
requestCount: 0,
totalTokens: 0,
})
})
this.tokenUsageWindow = []
}
}
// Export singleton instance
export const aiProvider = new MultiAIProviderManager()