Deploy AI models at scale with proper infrastructure, monitoring, and auto-scaling.

Deployment Frameworks

NVIDIA Triton Inference Server

# Model repository structure
model_repository/
├── llama-3-8b/
│   ├── config.pbtxt
│   └── 1/
│       └── model.plan  # TensorRT engine

# config.pbtxt
name: "llama-3-8b"
platform: "tensorrt_plan"
max_batch_size: 32
dynamic_batching {
  max_queue_delay_microseconds: 100
}
instance_group [
  {
    count: 2  # 2 model instances per GPU
    kind: KIND_GPU
  }
]

Ray Serve

from ray import serve
import torch

@serve.deployment(
    num_replicas=4,
    ray_actor_options={"num_gpus": 1}
)
class LLMDeployment:
    def __init__(self):
        self.model = AutoModelForCausalLM.from_pretrained(
            "meta-llama/Llama-3-8b",
            torch_dtype=torch.float16,
            device_map="auto"
        )
    
    async def __call__(self, request):
        prompt = await request.json()
        outputs = self.model.generate(**prompt)
        return {"generated_text": outputs}

serve.run(LLMDeployment.bind())

Auto-Scaling

Kubernetes HPA with Custom Metrics

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: llm-inference-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: llm-inference
  minReplicas: 2
  maxReplicas: 20
  metrics:
  - type: Pods
    pods:
      metric:
        name: inference_queue_depth
      target:
        type: AverageValue
        averageValue: "10"
  - type: Resource
    resource:
      name: nvidia.com/gpu
      target:
        type: Utilization
        averageUtilization: 80

Model Management

A/B Testing

from fastapi import FastAPI
import random

app = FastAPI()

models = {
    "model_a": load_model("llama-3-8b"),
    "model_b": load_model("llama-3-8b-finetuned")
}

@app.post("/predict")
async def predict(request: dict):
    # Route 10% to model B
    model_version = "model_b" if random.random() < 0.1 else "model_a"
    
    model = models[model_version]
    output = model.generate(request["prompt"])
    
    # Log for analysis
    log_prediction(model_version, request, output)
    
    return {"result": output, "model": model_version}