20 min
Lesson 2 of 4
Multi-Provider Routing
From the course AI API Gateway Patterns
Implement intelligent routing across multiple AI providers for reliability and cost optimization.
Routing Strategies
Cost-Based Routing
const providerCosts = {
'gpt-4': { input: 0.03, output: 0.06 },
'claude-3-opus': { input: 0.015, output: 0.075 },
'gemini-pro': { input: 0.00025, output: 0.0005 },
};
function selectCheapestProvider(request: AIRequest): string {
const estimatedTokens = estimateTokens(request);
let cheapest = { provider: '', cost: Infinity };
for (const [provider, pricing] of Object.entries(providerCosts)) {
if (!meetsCapabilityRequirements(provider, request)) continue;
const cost = estimatedTokens.input * pricing.input +
estimatedTokens.output * pricing.output;
if (cost < cheapest.cost) {
cheapest = { provider, cost };
}
}
return cheapest.provider;
}
Capability-Based Routing
const providerCapabilities = {
'gpt-4-vision': ['text', 'image', 'code'],
'claude-3-opus': ['text', 'code', 'long-context'],
'gemini-pro-vision': ['text', 'image', 'video'],
};
function selectByCapability(request: AIRequest): string {
const requiredCapabilities = inferCapabilities(request);
for (const [provider, caps] of Object.entries(providerCapabilities)) {
if (requiredCapabilities.every(c => caps.includes(c))) {
return provider;
}
}
throw new Error('No provider supports required capabilities');
}
Load Balancing with Health Checks
class LoadBalancer {
private healthScores: Map<string, number> = new Map();
async selectProvider(): Promise<string> {
const healthy = this.getHealthyProviders();
// Weighted random selection based on health scores
const totalWeight = healthy.reduce((sum, p) => sum + this.healthScores.get(p)!, 0);
let random = Math.random() * totalWeight;
for (const provider of healthy) {
random -= this.healthScores.get(provider)!;
if (random <= 0) return provider;
}
return healthy[0];
}
updateHealth(provider: string, success: boolean, latency: number): void {
const current = this.healthScores.get(provider) || 100;
const adjustment = success ? Math.min(5, (1000 - latency) / 200) : -20;
this.healthScores.set(provider, Math.max(0, Math.min(100, current + adjustment)));
}
}
Key Takeaways
- •Cost-based routing estimates tokens and selects cheapest capable provider
- •Capability-based routing matches request requirements to provider features
- •Load balancing uses weighted random selection based on health scores
- •Health scores adjust based on success/failure and latency of each provider
0 of 4 lessons marked complete
Progress is kept in this browser only.