AI Model Wiki

    Skip to main content
    30 min
    Lesson 2 of 4

    MLOps and Pipelines

    From the course Google Vertex AI Platform

    Build production ML workflows with Vertex AI Pipelines, model monitoring, and MLOps best practices.

    Vertex AI Pipelines

    Pipeline Definition

    from kfp import dsl
    from kfp.v2 import compiler
    
    @dsl.component(
        base_image="python:3.11",
        packages_to_install=["pandas", "scikit-learn"]
    )
    def preprocess_data(
        input_path: str,
        output_path: dsl.Output[dsl.Dataset]
    ):
        import pandas as pd
        df = pd.read_csv(input_path)
        df_clean = df.dropna()
        df_clean.to_csv(output_path.path, index=False)
    
    @dsl.component(
        base_image="python:3.11",
        packages_to_install=["scikit-learn", "joblib"]
    )
    def train_model(
        training_data: dsl.Input[dsl.Dataset],
        model: dsl.Output[dsl.Model],
        accuracy: dsl.Output[dsl.Metrics]
    ):
        import pandas as pd
        from sklearn.ensemble import RandomForestClassifier
        from sklearn.model_selection import train_test_split
        from sklearn.metrics import accuracy_score
        import joblib
        
        df = pd.read_csv(training_data.path)
        X = df.drop('target', axis=1)
        y = df['target']
        
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
        
        clf = RandomForestClassifier(n_estimators=100)
        clf.fit(X_train, y_train)
        
        predictions = clf.predict(X_test)
        acc = accuracy_score(y_test, predictions)
        
        accuracy.log_metric("accuracy", acc)
        joblib.dump(clf, model.path)
    
    @dsl.pipeline(name="ml-training-pipeline")
    def training_pipeline(input_data: str):
        preprocess = preprocess_data(input_path=input_data)
        train = train_model(training_data=preprocess.outputs['output_path'])
    
    # Compile and run
    compiler.Compiler().compile(training_pipeline, "pipeline.json")
    
    from google.cloud import aiplatform
    aiplatform.init(project="my-project", location="us-central1")
    
    job = aiplatform.PipelineJob(
        display_name="training-run",
        template_path="pipeline.json",
        parameter_values={"input_data": "gs://my-bucket/data.csv"}
    )
    job.run()
    

    Model Monitoring

    Skew and Drift Detection

    from google.cloud.aiplatform_v1.types import (
        ModelMonitoringObjectiveConfig,
        ModelMonitoringAlertConfig,
        ThresholdConfig
    )
    
    monitoring_job = aiplatform.ModelDeploymentMonitoringJob.create(
        display_name="production-model-monitor",
        endpoint=endpoint,
        model_deployment_monitoring_objective_configs=[
            ModelMonitoringObjectiveConfig(
                deployed_model_id=deployed_model_id,
                objective_config=ModelMonitoringObjectiveConfig.ObjectiveConfig(
                    training_prediction_skew_detection_config={
                        "default_skew_threshold": ThresholdConfig(value=0.1),
                        "feature_thresholds": {
                            "age": ThresholdConfig(value=0.05),
                            "income": ThresholdConfig(value=0.08)
                        }
                    },
                    prediction_drift_detection_config={
                        "default_drift_threshold": ThresholdConfig(value=0.1)
                    }
                )
            )
        ],
        logging_sampling_strategy={"random_sample_config": {"sample_rate": 0.1}},
        monitor_interval={"seconds": 3600}  # Check every hour
    )
    

    Key Takeaways

    • •KFP pipelines orchestrate data preprocessing, training, and deployment as code
    • •Component-based architecture enables reusable, versioned ML workflow steps
    • •Model monitoring detects training-serving skew and prediction drift automatically
    • •Artifact lineage tracks every version of data, models, and metrics
    0 of 4 lessons marked complete

    Progress is kept in this browser only.