25 min
Lesson 2 of 4
Drift Detection Implementation
From the course AI Model Monitoring
Implement statistical drift detection for production ML systems.
Statistical Tests for Drift
Kolmogorov-Smirnov Test
from scipy import stats
import numpy as np
def detect_drift_ks(reference_data, current_data, threshold=0.05):
"""
Detect distribution drift using KS test.
Returns True if drift detected.
"""
ks_statistic, p_value = stats.ks_2samp(reference_data, current_data)
drift_detected = p_value < threshold
return {
"drift_detected": drift_detected,
"ks_statistic": ks_statistic,
"p_value": p_value,
"severity": "high" if ks_statistic > 0.3 else "medium" if ks_statistic > 0.1 else "low"
}
# Monitor each feature
for feature in feature_columns:
result = detect_drift_ks(
reference_data[feature].values,
current_data[feature].values
)
if result["drift_detected"]:
send_alert(f"Drift detected in {feature}: {result}")
Population Stability Index (PSI)
def calculate_psi(expected, actual, buckets=10):
"""
PSI < 0.1: No significant drift
PSI 0.1-0.25: Moderate drift (investigate)
PSI > 0.25: Significant drift (retrain)
"""
expected_bins = np.histogram(expected, bins=buckets)[0]
actual_bins = np.histogram(actual, bins=buckets)[0]
expected_pct = (expected_bins + 0.0001) / len(expected)
actual_pct = (actual_bins + 0.0001) / len(actual)
psi = np.sum((actual_pct - expected_pct) * np.log(actual_pct / expected_pct))
return psi
# Evaluate all features weekly
weekly_psi = {}
for feature in features:
psi = calculate_psi(training_data[feature], production_data[feature])
weekly_psi[feature] = psi
if psi > 0.25:
trigger_retraining_alert(feature, psi)
Evidently AI Integration
from evidently.report import Report
from evidently.metric_preset import DataDriftPreset, ClassificationPreset
# Comprehensive monitoring report
report = Report(metrics=[
DataDriftPreset(),
ClassificationPreset()
])
report.run(
reference_data=reference_df,
current_data=production_df,
column_mapping=column_mapping
)
# Generate HTML report
report.save_html("monitoring_report.html")
# Get JSON metrics for alerting
results = report.as_dict()
if results["metrics"][0]["result"]["dataset_drift"]:
trigger_retraining_pipeline()
Prediction Monitoring
class PredictionMonitor:
def __init__(self, model_id, window_hours=24):
self.model_id = model_id
self.window = window_hours
def log_prediction(self, inputs, prediction, ground_truth=None):
self.db.insert({
"model_id": self.model_id,
"inputs": json.dumps(inputs),
"prediction": prediction,
"ground_truth": ground_truth,
"timestamp": datetime.utcnow()
})
def calculate_rolling_accuracy(self):
recent = self.db.query("""
SELECT prediction, ground_truth
FROM predictions
WHERE model_id = ?
AND ground_truth IS NOT NULL
AND timestamp > NOW() - INTERVAL ? HOUR
""", (self.model_id, self.window))
if len(recent) < 100:
return None # Not enough data
accuracy = sum(r.prediction == r.ground_truth for r in recent) / len(recent)
return accuracy
Key Takeaways
- •KS test and PSI provide statistical measures of feature distribution drift
- •PSI >0.25 indicates significant drift requiring model retraining
- •Evidently AI generates comprehensive monitoring reports comparing reference to production
- •Rolling accuracy on labeled production data provides ground truth performance metrics
0 of 4 lessons marked complete
Progress is kept in this browser only.