Implement streaming inference and continuous batching for optimal throughput and perceived latency.

Token Streaming

Server-Sent Events Streaming

from fastapi import FastAPI
from fastapi.responses import StreamingResponse
import asyncio

app = FastAPI()

@app.post("/chat/stream")
async def stream_chat(request: dict):
    prompt = request["prompt"]
    
    async def generate_tokens():
        # Stream each token as it's generated
        for token in model.stream_generate(prompt):
            # SSE format
            yield f"data: {json.dumps({'token': token})}

"
            await asyncio.sleep(0)  # Yield control
        
        yield "data: [DONE]

"
    
    return StreamingResponse(
        generate_tokens(),
        media_type="text/event-stream",
        headers={
            "Cache-Control": "no-cache",
            "X-Accel-Buffering": "no"  # Disable nginx buffering
        }
    )

Client-Side Streaming

const response = await fetch('/chat/stream', {
    method: 'POST',
    headers: {'Content-Type': 'application/json'},
    body: JSON.stringify({prompt: userMessage})
});

const reader = response.body.getReader();
const decoder = new TextDecoder();

while (true) {
    const {done, value} = await reader.read();
    if (done) break;
    
    const text = decoder.decode(value);
    const lines = text.split('

');
    
    for (const line of lines) {
        if (line.startsWith('data: ') && line !== 'data: [DONE]') {
            const data = JSON.parse(line.slice(6));
            appendToUI(data.token);  // Show each token immediately
        }
    }
}

Continuous Batching

vLLM PagedAttention

from vllm import LLM, SamplingParams

# vLLM implements continuous batching automatically
llm = LLM(
    model="meta-llama/Llama-3-8B",
    tensor_parallel_size=4,  # Multi-GPU
    gpu_memory_utilization=0.90
)

# Multiple requests processed concurrently with optimal batching
prompts = [f"Question {i}: ..." for i in range(100)]

sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=512
)

outputs = llm.generate(prompts, sampling_params)
# vLLM 20-24x higher throughput vs HuggingFace Transformers

KV Cache Management

# Prefix caching for shared system prompts
class KVCacheManager:
    def __init__(self, max_cache_size_gb=40):
        self.cache = {}
        self.max_size = max_cache_size_gb * 1e9
    
    def get_or_compute_kv(self, prefix):
        prefix_hash = hash(prefix)
        if prefix_hash in self.cache:
            return self.cache[prefix_hash]  # Cache hit!
        
        kv = self.compute_kv(prefix)
        self.cache[prefix_hash] = kv
        return kv

# For RAG systems: cache document KVs
# For multi-turn: cache conversation history
# For system prompts: cache across all users