AI Model Wiki

    Skip to main content
    22 min
    Lesson 3 of 4

    Alerting and Automated Retraining

    From the course AI Model Monitoring

    Build automated alerting and retraining pipelines that maintain model performance continuously.

    Alerting Framework

    Tiered Alert System

    from enum import Enum
    
    class AlertSeverity(Enum):
        INFO = "info"
        WARNING = "warning"
        CRITICAL = "critical"
    
    class ModelAlertManager:
        def evaluate_health(self, model_id, metrics):
            alerts = []
            
            # Performance degradation
            if metrics["accuracy"] < 0.80:
                alerts.append(Alert(
                    severity=AlertSeverity.CRITICAL,
                    message=f"Model {model_id} accuracy dropped to {metrics['accuracy']:.2%}",
                    action="IMMEDIATE_INVESTIGATION"
                ))
            elif metrics["accuracy"] < 0.85:
                alerts.append(Alert(
                    severity=AlertSeverity.WARNING,
                    message=f"Model {model_id} accuracy below threshold: {metrics['accuracy']:.2%}",
                    action="SCHEDULE_RETRAINING"
                ))
            
            # Drift detection
            if metrics["psi_max"] > 0.25:
                alerts.append(Alert(
                    severity=AlertSeverity.WARNING,
                    message=f"Significant data drift detected: PSI={metrics['psi_max']:.3f}",
                    action="RETRAIN_WITHIN_WEEK"
                ))
            
            return alerts
    

    Automated Retraining

    Trigger-Based Retraining

    class RetrainingOrchestrator:
        def check_retraining_triggers(self, model_id):
            triggers = []
            
            # Schedule-based (every 30 days)
            if self.days_since_training(model_id) > 30:
                triggers.append("SCHEDULED")
            
            # Performance-based
            if self.get_rolling_accuracy(model_id) < self.thresholds["min_accuracy"]:
                triggers.append("PERFORMANCE_DEGRADATION")
            
            # Drift-based
            if self.get_max_psi(model_id) > 0.25:
                triggers.append("DATA_DRIFT")
            
            # Volume-based (enough new labels)
            if self.get_new_labels_count(model_id) > 10000:
                triggers.append("NEW_DATA")
            
            return triggers
        
        def trigger_retraining(self, model_id, reason):
            # Submit retraining job
            self.pipeline.submit({
                "model_id": model_id,
                "trigger": reason,
                "training_data": self.get_latest_data(),
                "validation_data": self.get_holdout_data()
            })
    

    Canary Deployment for Model Updates

    class CanaryDeployment:
        def deploy_new_model(self, new_model, canary_pct=10):
            # Route 10% traffic to new model
            self.traffic_router.set_weights({
                "current": 100 - canary_pct,
                "canary": canary_pct
            })
            
            # Monitor for 24 hours
            time.sleep(86400)
            
            # Compare performance
            current_metrics = self.get_model_metrics("current")
            canary_metrics = self.get_model_metrics("canary")
            
            if canary_metrics["accuracy"] >= current_metrics["accuracy"] - 0.01:
                # Gradually promote
                self.traffic_router.set_weights({"current": 0, "canary": 100})
                print("Canary promoted to production")
            else:
                # Rollback
                self.traffic_router.set_weights({"current": 100, "canary": 0})
                print("Canary rolled back — performance regression detected")
    

    Key Takeaways

    • •Tiered alerting with INFO/WARNING/CRITICAL enables proportional response to degradation
    • •Multi-trigger retraining covers schedules, performance thresholds, drift, and new data
    • •Canary deployments validate new models on 10% traffic before full promotion
    • •Automated rollback on performance regression prevents production impact
    0 of 4 lessons marked complete

    Progress is kept in this browser only.