Implement technical optimizations that dramatically reduce AI operational costs.

Prompt Optimization

Token Reduction Techniques

# Before: Verbose prompt (estimated 400 tokens)
verbose_prompt = """
You are a helpful assistant. I want you to help me with something.
Could you please analyze the following customer review and tell me:
1. What is the overall sentiment (positive, negative, or neutral)?
2. What specific features or aspects does the customer mention?
3. Would you say this customer is likely to purchase again?
Please provide a detailed response with your analysis.

Customer Review: {review_text}
"""

# After: Optimized prompt (estimated 60 tokens)
optimized_prompt = """Analyze this review. Reply in JSON:
{"sentiment": "positive/negative/neutral", "features": [], "repurchase_likely": true/false}

Review: {review_text}"""

# Savings: ~85% token reduction per request
# At 100K requests/month = $500 → $75/month with Claude Haiku

Structured Output

from openai import OpenAI
from pydantic import BaseModel

class ReviewAnalysis(BaseModel):
    sentiment: str
    features: list[str]
    repurchase_likely: bool

# Structured output uses fewer tokens for parsing
response = client.beta.chat.completions.parse(
    model="gpt-4o-mini",
    messages=[{"role": "user", "content": f"Analyze: {review}"}],
    response_format=ReviewAnalysis,
)

Caching Strategies

Response Caching

import hashlib
import json
import redis

redis_client = redis.Redis(host='localhost', port=6379)

def cached_ai_call(prompt, model="gpt-4o-mini", ttl=3600):
    # Create cache key
    cache_key = hashlib.sha256(
        json.dumps({"prompt": prompt, "model": model}).encode()
    ).hexdigest()
    
    # Check cache
    cached = redis_client.get(cache_key)
    if cached:
        return json.loads(cached)  # Cache hit - no API cost!
    
    # Cache miss - call API
    response = client.chat.completions.create(
        model=model,
        messages=[{"role": "user", "content": prompt}]
    )
    
    result = response.choices[0].message.content
    
    # Store in cache
    redis_client.setex(cache_key, ttl, json.dumps(result))
    
    return result

# For similar/repeated prompts, cache hit rate of 30-60% is achievable

Semantic Caching

from sentence_transformers import SentenceTransformer
import numpy as np

model = SentenceTransformer('all-MiniLM-L6-v2')

def semantic_cache_lookup(query, similarity_threshold=0.95):
    query_embedding = model.encode([query])[0]
    
    # Search vector store for similar past queries
    similar = vector_store.search(
        query_embedding,
        top_k=1,
        min_score=similarity_threshold
    )
    
    if similar:
        return similar[0]['response']  # Return cached similar response
    
    return None  # Cache miss

Batch Processing

Batch API for 50% Savings

# OpenAI Batch API - 50% cheaper, 24h turnaround
import jsonl

# Create batch input file
requests = []
for i, text in enumerate(documents_to_classify):
    requests.append({
        "custom_id": f"request-{i}",
        "method": "POST",
        "url": "/v1/chat/completions",
        "body": {
            "model": "gpt-4o-mini",
            "messages": [{"role": "user", "content": f"Classify: {text}"}]
        }
    })

# Submit batch
batch = client.batches.create(
    input_file_id=upload_jsonl(requests),
    endpoint="/v1/chat/completions",
    completion_window="24h"  # Up to 24 hours
)

# Retrieve results when complete
results = client.batches.retrieve(batch.id)