17 min
Lesson 3 of 4
Caching & Rate Limiting
From the course AI API Gateway Patterns
Implement caching and rate limiting strategies optimized for AI workloads.
Semantic Caching
import { embeddings } from './embedding-service';
class SemanticCache {
private vectorStore: VectorStore;
private similarityThreshold = 0.95;
async get(request: AIRequest): Promise<AIResponse | null> {
// Generate embedding for the request
const embedding = await embeddings.create(request.prompt);
// Search for similar cached requests
const results = await this.vectorStore.search(embedding, { topK: 1 });
if (results[0]?.score >= this.similarityThreshold) {
return results[0].metadata.response;
}
return null;
}
async set(request: AIRequest, response: AIResponse): Promise<void> {
const embedding = await embeddings.create(request.prompt);
await this.vectorStore.upsert({
id: generateId(request),
embedding,
metadata: {
request,
response,
timestamp: Date.now(),
ttl: this.calculateTTL(request),
},
});
}
}
Tiered Rate Limiting
const rateLimitTiers = {
free: { rpm: 10, tpm: 10000, dailyLimit: 100 },
pro: { rpm: 60, tpm: 100000, dailyLimit: 1000 },
enterprise: { rpm: 600, tpm: 1000000, dailyLimit: null },
};
class RateLimiter {
async checkLimit(userId: string, tokens: number): Promise<void> {
const tier = await this.getUserTier(userId);
const limits = rateLimitTiers[tier];
// Check requests per minute
const rpm = await this.redis.incr(\`rpm:${userId}\`);
if (rpm === 1) await this.redis.expire(\`rpm:${userId}\`, 60);
if (rpm > limits.rpm) throw new RateLimitError('RPM exceeded');
// Check tokens per minute
const tpm = await this.redis.incrby(\`tpm:${userId}\`, tokens);
if (tpm > limits.tpm) throw new RateLimitError('TPM exceeded');
// Check daily limit
if (limits.dailyLimit) {
const daily = await this.redis.incr(\`daily:${userId}\`);
if (daily > limits.dailyLimit) throw new RateLimitError('Daily limit exceeded');
}
}
}
Key Takeaways
- •Semantic caching uses embeddings to find similar previous requests (95% threshold)
- •Cache hits on semantically similar prompts can dramatically reduce API costs
- •Tiered rate limiting controls RPM, TPM, and daily limits by subscription tier
- •Redis TTLs automatically expire rate limit counters for sliding window enforcement
0 of 4 lessons marked complete
Progress is kept in this browser only.