A model that works in a notebook can fail silently in production. This lesson covers the monitoring, alerting, and hardening steps required before serving real users.

Metrics to Instrument

Every production inference endpoint should emit these metrics:

import time
from prometheus_client import Histogram, Counter, Gauge, start_http_server

# Start Prometheus scrape endpoint on port 8001
start_http_server(8001)

REQUEST_LATENCY = Histogram(
    "inference_latency_seconds",
    "End-to-end latency per request",
    buckets=[0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0]
)
TOKENS_GENERATED = Counter("tokens_generated_total", "Total tokens produced")
QUEUE_DEPTH = Gauge("request_queue_depth", "Pending requests in queue")
ERROR_RATE = Counter("inference_errors_total", "Failed inference requests", ["error_type"])

def instrumented_generate(prompt: str) -> str:
    QUEUE_DEPTH.inc()
    start = time.perf_counter()
    try:
        output = model.generate(prompt)
        elapsed = time.perf_counter() - start
        REQUEST_LATENCY.observe(elapsed)
        TOKENS_GENERATED.inc(len(output.split()))
        return output
    except torch.cuda.OutOfMemoryError as e:
        ERROR_RATE.labels(error_type="oom").inc()
        raise
    except Exception as e:
        ERROR_RATE.labels(error_type="unknown").inc()
        raise
    finally:
        QUEUE_DEPTH.dec()

Alert Thresholds

Configure these in Grafana or any Prometheus-compatible alerting system:

Output Quality Monitoring

Silent quality degradation is harder to detect than errors. Implement lightweight quality checks:

import re

def quality_check(prompt: str, output: str) -> dict:
    issues = []

    # Length anomaly: suspiciously short responses
    if len(output.split()) < 10:
        issues.append("SHORT_OUTPUT")

    # Repetition detection (common quantization artifact)
    words = output.lower().split()
    if len(words) > 20:
        bigrams = [f"{words[i]} {words[i+1]}" for i in range(len(words)-1)]
        unique_ratio = len(set(bigrams)) / len(bigrams)
        if unique_ratio < 0.5:
            issues.append("REPETITION_LOOP")

    # Truncation (hit max_tokens limit)
    if output.endswith(("...", "—")) or len(output.split()) >= 120:
        issues.append("POSSIBLE_TRUNCATION")

    return {"output": output, "quality_issues": issues, "pass": len(issues) == 0}

Graceful Degradation

async def generate_with_fallback(prompt: str, primary_model, fallback_api_fn) -> str:
    """
    Try the local optimized model first; fall back to API if it fails.
    Prevents complete outage when GPU has transient issues.
    """
    try:
        return await asyncio.wait_for(
            primary_model.agenerate(prompt),
            timeout=10.0
        )
    except (asyncio.TimeoutError, RuntimeError) as e:
        # Log the failure for later investigation
        ERROR_RATE.labels(error_type="fallback_triggered").inc()
        return await fallback_api_fn(prompt)

Load Testing Before Launch

# Locust load test targeting your vLLM endpoint
pip install locust

# locustfile.py
# from locust import HttpUser, task, between
# class InferenceUser(HttpUser):
#     wait_time = between(0.5, 2)
#     @task
#     def generate(self):
#         self.client.post("/generate", json={"prompt": "Hello", "max_tokens": 64})

locust -f locustfile.py --host=http://localhost:8000 --users=50 --spawn-rate=5

Run the load test for at least 10 minutes and verify p99 latency stays within your SLA before shipping to production.