AI Model Wiki

    Skip to main content
    25 min
    Lesson 2 of 4

    Drift Detection Implementation

    From the course AI Model Monitoring

    Implement statistical drift detection for production ML systems.

    Statistical Tests for Drift

    Kolmogorov-Smirnov Test

    from scipy import stats
    import numpy as np
    
    def detect_drift_ks(reference_data, current_data, threshold=0.05):
        """
        Detect distribution drift using KS test.
        Returns True if drift detected.
        """
        ks_statistic, p_value = stats.ks_2samp(reference_data, current_data)
        
        drift_detected = p_value < threshold
        
        return {
            "drift_detected": drift_detected,
            "ks_statistic": ks_statistic,
            "p_value": p_value,
            "severity": "high" if ks_statistic > 0.3 else "medium" if ks_statistic > 0.1 else "low"
        }
    
    # Monitor each feature
    for feature in feature_columns:
        result = detect_drift_ks(
            reference_data[feature].values,
            current_data[feature].values
        )
        if result["drift_detected"]:
            send_alert(f"Drift detected in {feature}: {result}")
    

    Population Stability Index (PSI)

    def calculate_psi(expected, actual, buckets=10):
        """
        PSI < 0.1: No significant drift
        PSI 0.1-0.25: Moderate drift (investigate)
        PSI > 0.25: Significant drift (retrain)
        """
        expected_bins = np.histogram(expected, bins=buckets)[0]
        actual_bins = np.histogram(actual, bins=buckets)[0]
        
        expected_pct = (expected_bins + 0.0001) / len(expected)
        actual_pct = (actual_bins + 0.0001) / len(actual)
        
        psi = np.sum((actual_pct - expected_pct) * np.log(actual_pct / expected_pct))
        
        return psi
    
    # Evaluate all features weekly
    weekly_psi = {}
    for feature in features:
        psi = calculate_psi(training_data[feature], production_data[feature])
        weekly_psi[feature] = psi
        
        if psi > 0.25:
            trigger_retraining_alert(feature, psi)
    

    Evidently AI Integration

    from evidently.report import Report
    from evidently.metric_preset import DataDriftPreset, ClassificationPreset
    
    # Comprehensive monitoring report
    report = Report(metrics=[
        DataDriftPreset(),
        ClassificationPreset()
    ])
    
    report.run(
        reference_data=reference_df,
        current_data=production_df,
        column_mapping=column_mapping
    )
    
    # Generate HTML report
    report.save_html("monitoring_report.html")
    
    # Get JSON metrics for alerting
    results = report.as_dict()
    if results["metrics"][0]["result"]["dataset_drift"]:
        trigger_retraining_pipeline()
    

    Prediction Monitoring

    class PredictionMonitor:
        def __init__(self, model_id, window_hours=24):
            self.model_id = model_id
            self.window = window_hours
        
        def log_prediction(self, inputs, prediction, ground_truth=None):
            self.db.insert({
                "model_id": self.model_id,
                "inputs": json.dumps(inputs),
                "prediction": prediction,
                "ground_truth": ground_truth,
                "timestamp": datetime.utcnow()
            })
        
        def calculate_rolling_accuracy(self):
            recent = self.db.query("""
                SELECT prediction, ground_truth 
                FROM predictions
                WHERE model_id = ? 
                AND ground_truth IS NOT NULL
                AND timestamp > NOW() - INTERVAL ? HOUR
            """, (self.model_id, self.window))
            
            if len(recent) < 100:
                return None  # Not enough data
            
            accuracy = sum(r.prediction == r.ground_truth for r in recent) / len(recent)
            return accuracy
    

    Key Takeaways

    • •KS test and PSI provide statistical measures of feature distribution drift
    • •PSI >0.25 indicates significant drift requiring model retraining
    • •Evidently AI generates comprehensive monitoring reports comparing reference to production
    • •Rolling accuracy on labeled production data provides ground truth performance metrics
    0 of 4 lessons marked complete

    Progress is kept in this browser only.