Implement comprehensive bias and fairness testing for AI systems.

Fairness Metrics

Statistical Fairness Measures

import pandas as pd
from sklearn.metrics import confusion_matrix
import numpy as np

def comprehensive_fairness_audit(predictions, actuals, sensitive_attribute):
    groups = predictions[sensitive_attribute].unique()
    results = {}
    
    for group in groups:
        mask = predictions[sensitive_attribute] == group
        group_preds = predictions.loc[mask, "prediction"]
        group_actual = actuals.loc[mask, "actual"]
        
        tn, fp, fn, tp = confusion_matrix(group_actual, group_preds).ravel()
        
        results[group] = {
            # Selection rate (Demographic Parity)
            "selection_rate": (tp + fp) / len(group_preds),
            
            # True positive rate (Equal Opportunity)
            "tpr": tp / (tp + fn) if (tp + fn) > 0 else 0,
            
            # False positive rate
            "fpr": fp / (fp + tn) if (fp + tn) > 0 else 0,
            
            # Accuracy
            "accuracy": (tp + tn) / len(group_preds),
            
            # Positive predictive value
            "ppv": tp / (tp + fp) if (tp + fp) > 0 else 0
        }
    
    # Calculate fairness ratios between groups
    groups_list = list(groups)
    fairness_analysis = {}
    
    for i, group_a in enumerate(groups_list):
        for group_b in groups_list[i+1:]:
            # Disparate Impact Ratio (aim for > 0.8)
            dir = min(
                results[group_a]["selection_rate"],
                results[group_b]["selection_rate"]
            ) / max(
                results[group_a]["selection_rate"],
                results[group_b]["selection_rate"]
            )
            
            fairness_analysis[f"{group_a}_vs_{group_b}"] = {
                "disparate_impact_ratio": dir,
                "passes_80_pct_rule": dir >= 0.8,
                "tpr_difference": abs(results[group_a]["tpr"] - results[group_b]["tpr"]),
                "fpr_difference": abs(results[group_a]["fpr"] - results[group_b]["fpr"])
            }
    
    return results, fairness_analysis

# Generate audit report
group_metrics, fairness_ratios = comprehensive_fairness_audit(
    pred_df, actual_df, "gender"
)

Intersectional Fairness

def intersectional_analysis(predictions, actuals, attributes):
    # Create intersectional groups (e.g., Black women, White men)
    pred_df = predictions.copy()
    pred_df["intersectional_group"] = pred_df[attributes].apply(
        lambda row: "_".join(row.astype(str)), axis=1
    )
    
    # Analyze each intersectional subgroup
    groups = pred_df["intersectional_group"].unique()
    
    for group in groups:
        if len(pred_df[pred_df["intersectional_group"] == group]) < 50:
            continue  # Skip groups too small for reliable analysis
        
        group_preds = pred_df[pred_df["intersectional_group"] == group]
        # Calculate metrics for each intersectional group
        accuracy = (group_preds["prediction"] == group_preds["actual"]).mean()
        
        if accuracy < 0.80:
            print(f"WARNING: Low accuracy for {group}: {accuracy:.2%}")

Fairness-Aware Model Development

from fairlearn.reductions import ExponentiatedGradient, EqualizedOdds
from sklearn.linear_model import LogisticRegression

# Constrained optimization for fairness
base_model = LogisticRegression()
constraint = EqualizedOdds()  # Equal TPR and FPR across groups

mitigator = ExponentiatedGradient(base_model, constraint)
mitigator.fit(
    X_train, y_train,
    sensitive_features=X_train["gender"]
)

# Fair predictions
fair_predictions = mitigator.predict(X_test)