AI Model Wiki

    Skip to main content
    20 min
    Lesson 2 of 4

    Multi-Provider Routing

    From the course AI API Gateway Patterns

    Implement intelligent routing across multiple AI providers for reliability and cost optimization.

    Routing Strategies

    Cost-Based Routing

    const providerCosts = {
      'gpt-4': { input: 0.03, output: 0.06 },
      'claude-3-opus': { input: 0.015, output: 0.075 },
      'gemini-pro': { input: 0.00025, output: 0.0005 },
    };
    
    function selectCheapestProvider(request: AIRequest): string {
      const estimatedTokens = estimateTokens(request);
      
      let cheapest = { provider: '', cost: Infinity };
      
      for (const [provider, pricing] of Object.entries(providerCosts)) {
        if (!meetsCapabilityRequirements(provider, request)) continue;
        
        const cost = estimatedTokens.input * pricing.input + 
                     estimatedTokens.output * pricing.output;
        
        if (cost < cheapest.cost) {
          cheapest = { provider, cost };
        }
      }
      
      return cheapest.provider;
    }
    

    Capability-Based Routing

    const providerCapabilities = {
      'gpt-4-vision': ['text', 'image', 'code'],
      'claude-3-opus': ['text', 'code', 'long-context'],
      'gemini-pro-vision': ['text', 'image', 'video'],
    };
    
    function selectByCapability(request: AIRequest): string {
      const requiredCapabilities = inferCapabilities(request);
      
      for (const [provider, caps] of Object.entries(providerCapabilities)) {
        if (requiredCapabilities.every(c => caps.includes(c))) {
          return provider;
        }
      }
      
      throw new Error('No provider supports required capabilities');
    }
    

    Load Balancing with Health Checks

    class LoadBalancer {
      private healthScores: Map<string, number> = new Map();
      
      async selectProvider(): Promise<string> {
        const healthy = this.getHealthyProviders();
        
        // Weighted random selection based on health scores
        const totalWeight = healthy.reduce((sum, p) => sum + this.healthScores.get(p)!, 0);
        let random = Math.random() * totalWeight;
        
        for (const provider of healthy) {
          random -= this.healthScores.get(provider)!;
          if (random <= 0) return provider;
        }
        
        return healthy[0];
      }
      
      updateHealth(provider: string, success: boolean, latency: number): void {
        const current = this.healthScores.get(provider) || 100;
        const adjustment = success ? Math.min(5, (1000 - latency) / 200) : -20;
        this.healthScores.set(provider, Math.max(0, Math.min(100, current + adjustment)));
      }
    }
    

    Key Takeaways

    • •Cost-based routing estimates tokens and selects cheapest capable provider
    • •Capability-based routing matches request requirements to provider features
    • •Load balancing uses weighted random selection based on health scores
    • •Health scores adjust based on success/failure and latency of each provider
    0 of 4 lessons marked complete

    Progress is kept in this browser only.