30 min
Lesson 2 of 4
MLOps and Pipelines
From the course Google Vertex AI Platform
Build production ML workflows with Vertex AI Pipelines, model monitoring, and MLOps best practices.
Vertex AI Pipelines
Pipeline Definition
from kfp import dsl
from kfp.v2 import compiler
@dsl.component(
base_image="python:3.11",
packages_to_install=["pandas", "scikit-learn"]
)
def preprocess_data(
input_path: str,
output_path: dsl.Output[dsl.Dataset]
):
import pandas as pd
df = pd.read_csv(input_path)
df_clean = df.dropna()
df_clean.to_csv(output_path.path, index=False)
@dsl.component(
base_image="python:3.11",
packages_to_install=["scikit-learn", "joblib"]
)
def train_model(
training_data: dsl.Input[dsl.Dataset],
model: dsl.Output[dsl.Model],
accuracy: dsl.Output[dsl.Metrics]
):
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import joblib
df = pd.read_csv(training_data.path)
X = df.drop('target', axis=1)
y = df['target']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
clf = RandomForestClassifier(n_estimators=100)
clf.fit(X_train, y_train)
predictions = clf.predict(X_test)
acc = accuracy_score(y_test, predictions)
accuracy.log_metric("accuracy", acc)
joblib.dump(clf, model.path)
@dsl.pipeline(name="ml-training-pipeline")
def training_pipeline(input_data: str):
preprocess = preprocess_data(input_path=input_data)
train = train_model(training_data=preprocess.outputs['output_path'])
# Compile and run
compiler.Compiler().compile(training_pipeline, "pipeline.json")
from google.cloud import aiplatform
aiplatform.init(project="my-project", location="us-central1")
job = aiplatform.PipelineJob(
display_name="training-run",
template_path="pipeline.json",
parameter_values={"input_data": "gs://my-bucket/data.csv"}
)
job.run()
Model Monitoring
Skew and Drift Detection
from google.cloud.aiplatform_v1.types import (
ModelMonitoringObjectiveConfig,
ModelMonitoringAlertConfig,
ThresholdConfig
)
monitoring_job = aiplatform.ModelDeploymentMonitoringJob.create(
display_name="production-model-monitor",
endpoint=endpoint,
model_deployment_monitoring_objective_configs=[
ModelMonitoringObjectiveConfig(
deployed_model_id=deployed_model_id,
objective_config=ModelMonitoringObjectiveConfig.ObjectiveConfig(
training_prediction_skew_detection_config={
"default_skew_threshold": ThresholdConfig(value=0.1),
"feature_thresholds": {
"age": ThresholdConfig(value=0.05),
"income": ThresholdConfig(value=0.08)
}
},
prediction_drift_detection_config={
"default_drift_threshold": ThresholdConfig(value=0.1)
}
)
)
],
logging_sampling_strategy={"random_sample_config": {"sample_rate": 0.1}},
monitor_interval={"seconds": 3600} # Check every hour
)
Key Takeaways
- •KFP pipelines orchestrate data preprocessing, training, and deployment as code
- •Component-based architecture enables reusable, versioned ML workflow steps
- •Model monitoring detects training-serving skew and prediction drift automatically
- •Artifact lineage tracks every version of data, models, and metrics
0 of 4 lessons marked complete
Progress is kept in this browser only.