22 min
Lesson 3 of 4
Alerting and Automated Retraining
From the course AI Model Monitoring
Build automated alerting and retraining pipelines that maintain model performance continuously.
Alerting Framework
Tiered Alert System
from enum import Enum
class AlertSeverity(Enum):
INFO = "info"
WARNING = "warning"
CRITICAL = "critical"
class ModelAlertManager:
def evaluate_health(self, model_id, metrics):
alerts = []
# Performance degradation
if metrics["accuracy"] < 0.80:
alerts.append(Alert(
severity=AlertSeverity.CRITICAL,
message=f"Model {model_id} accuracy dropped to {metrics['accuracy']:.2%}",
action="IMMEDIATE_INVESTIGATION"
))
elif metrics["accuracy"] < 0.85:
alerts.append(Alert(
severity=AlertSeverity.WARNING,
message=f"Model {model_id} accuracy below threshold: {metrics['accuracy']:.2%}",
action="SCHEDULE_RETRAINING"
))
# Drift detection
if metrics["psi_max"] > 0.25:
alerts.append(Alert(
severity=AlertSeverity.WARNING,
message=f"Significant data drift detected: PSI={metrics['psi_max']:.3f}",
action="RETRAIN_WITHIN_WEEK"
))
return alerts
Automated Retraining
Trigger-Based Retraining
class RetrainingOrchestrator:
def check_retraining_triggers(self, model_id):
triggers = []
# Schedule-based (every 30 days)
if self.days_since_training(model_id) > 30:
triggers.append("SCHEDULED")
# Performance-based
if self.get_rolling_accuracy(model_id) < self.thresholds["min_accuracy"]:
triggers.append("PERFORMANCE_DEGRADATION")
# Drift-based
if self.get_max_psi(model_id) > 0.25:
triggers.append("DATA_DRIFT")
# Volume-based (enough new labels)
if self.get_new_labels_count(model_id) > 10000:
triggers.append("NEW_DATA")
return triggers
def trigger_retraining(self, model_id, reason):
# Submit retraining job
self.pipeline.submit({
"model_id": model_id,
"trigger": reason,
"training_data": self.get_latest_data(),
"validation_data": self.get_holdout_data()
})
Canary Deployment for Model Updates
class CanaryDeployment:
def deploy_new_model(self, new_model, canary_pct=10):
# Route 10% traffic to new model
self.traffic_router.set_weights({
"current": 100 - canary_pct,
"canary": canary_pct
})
# Monitor for 24 hours
time.sleep(86400)
# Compare performance
current_metrics = self.get_model_metrics("current")
canary_metrics = self.get_model_metrics("canary")
if canary_metrics["accuracy"] >= current_metrics["accuracy"] - 0.01:
# Gradually promote
self.traffic_router.set_weights({"current": 0, "canary": 100})
print("Canary promoted to production")
else:
# Rollback
self.traffic_router.set_weights({"current": 100, "canary": 0})
print("Canary rolled back — performance regression detected")
Key Takeaways
- •Tiered alerting with INFO/WARNING/CRITICAL enables proportional response to degradation
- •Multi-trigger retraining covers schedules, performance thresholds, drift, and new data
- •Canary deployments validate new models on 10% traffic before full promotion
- •Automated rollback on performance regression prevents production impact
0 of 4 lessons marked complete
Progress is kept in this browser only.