AI Model Wiki

    Skip to main content
    17 min
    Lesson 3 of 4

    Caching & Rate Limiting

    From the course AI API Gateway Patterns

    Implement caching and rate limiting strategies optimized for AI workloads.

    Semantic Caching

    import { embeddings } from './embedding-service';
    
    class SemanticCache {
      private vectorStore: VectorStore;
      private similarityThreshold = 0.95;
      
      async get(request: AIRequest): Promise<AIResponse | null> {
        // Generate embedding for the request
        const embedding = await embeddings.create(request.prompt);
        
        // Search for similar cached requests
        const results = await this.vectorStore.search(embedding, { topK: 1 });
        
        if (results[0]?.score >= this.similarityThreshold) {
          return results[0].metadata.response;
        }
        
        return null;
      }
      
      async set(request: AIRequest, response: AIResponse): Promise<void> {
        const embedding = await embeddings.create(request.prompt);
        
        await this.vectorStore.upsert({
          id: generateId(request),
          embedding,
          metadata: {
            request,
            response,
            timestamp: Date.now(),
            ttl: this.calculateTTL(request),
          },
        });
      }
    }
    

    Tiered Rate Limiting

    const rateLimitTiers = {
      free: { rpm: 10, tpm: 10000, dailyLimit: 100 },
      pro: { rpm: 60, tpm: 100000, dailyLimit: 1000 },
      enterprise: { rpm: 600, tpm: 1000000, dailyLimit: null },
    };
    
    class RateLimiter {
      async checkLimit(userId: string, tokens: number): Promise<void> {
        const tier = await this.getUserTier(userId);
        const limits = rateLimitTiers[tier];
        
        // Check requests per minute
        const rpm = await this.redis.incr(\`rpm:${userId}\`);
        if (rpm === 1) await this.redis.expire(\`rpm:${userId}\`, 60);
        if (rpm > limits.rpm) throw new RateLimitError('RPM exceeded');
        
        // Check tokens per minute
        const tpm = await this.redis.incrby(\`tpm:${userId}\`, tokens);
        if (tpm > limits.tpm) throw new RateLimitError('TPM exceeded');
        
        // Check daily limit
        if (limits.dailyLimit) {
          const daily = await this.redis.incr(\`daily:${userId}\`);
          if (daily > limits.dailyLimit) throw new RateLimitError('Daily limit exceeded');
        }
      }
    }
    

    Key Takeaways

    • •Semantic caching uses embeddings to find similar previous requests (95% threshold)
    • •Cache hits on semantically similar prompts can dramatically reduce API costs
    • •Tiered rate limiting controls RPM, TPM, and daily limits by subscription tier
    • •Redis TTLs automatically expire rate limit counters for sliding window enforcement
    0 of 4 lessons marked complete

    Progress is kept in this browser only.