28 min
Lesson 2 of 4
Real-Time and Streaming Pipelines
From the course AI Data Pipeline Engineering
Build real-time data pipelines that feed online ML models with fresh, processed features.
Streaming Data Processing
Kafka + Spark Structured Streaming
from pyspark.sql.functions import from_json, col, window
from pyspark.sql.types import StructType, StringType
# Define schema for incoming events
schema = StructType() \
.add("user_id", StringType()) \
.add("event_type", StringType()) \
.add("timestamp", StringType()) \
.add("properties", StringType())
# Read from Kafka
events = spark \
.readStream \
.format("kafka") \
.option("kafka.bootstrap.servers", "kafka:9092") \
.option("subscribe", "user-events") \
.load() \
.select(from_json(col("value").cast("string"), schema).alias("data")) \
.select("data.*")
# Compute real-time features (5-minute windows)
user_features = events \
.groupBy(
col("user_id"),
window(col("timestamp"), "5 minutes")
) \
.agg(
count("event_type").alias("event_count_5m"),
countDistinct("event_type").alias("unique_events_5m")
)
# Write to online feature store
user_features.writeStream \
.foreachBatch(update_feature_store) \
.outputMode("update") \
.start()
def update_feature_store(batch_df, epoch_id):
for row in batch_df.collect():
redis_client.hset(
f"user:{row.user_id}:features",
mapping={
"event_count_5m": row.event_count_5m,
"unique_events_5m": row.unique_events_5m,
"updated_at": datetime.utcnow().isoformat()
}
)
Feature Engineering Pipeline
Automated Feature Generation
from feast import FeatureStore, Entity, FeatureView, Field
from feast.types import Int64, Float64, String
# Define entities
user = Entity(name="user_id", join_keys=["user_id"])
# Define feature view
user_activity_fv = FeatureView(
name="user_activity_features",
entities=[user],
ttl=timedelta(days=1),
schema=[
Field(name="purchase_count_7d", dtype=Int64),
Field(name="avg_order_value_30d", dtype=Float64),
Field(name="preferred_category", dtype=String),
Field(name="churn_risk_score", dtype=Float64),
],
source=user_activity_source
)
# Retrieve features for prediction
store = FeatureStore(repo_path="feature_repo/")
feature_vector = store.get_online_features(
features=[
"user_activity_features:purchase_count_7d",
"user_activity_features:avg_order_value_30d",
"user_activity_features:churn_risk_score"
],
entity_rows=[{"user_id": user_id}]
).to_dict()
Pipeline Orchestration with Airflow
from airflow import DAG
from airflow.operators.python import PythonOperator
with DAG("ml_training_pipeline", schedule_interval="@daily") as dag:
extract = PythonOperator(
task_id="extract_data",
python_callable=extract_from_sources
)
transform = PythonOperator(
task_id="transform_features",
python_callable=engineer_features
)
validate = PythonOperator(
task_id="validate_data_quality",
python_callable=run_quality_checks
)
train = PythonOperator(
task_id="train_model",
python_callable=train_ml_model
)
evaluate = PythonOperator(
task_id="evaluate_and_promote",
python_callable=evaluate_model
)
extract >> transform >> validate >> train >> evaluate
Key Takeaways
- •Kafka + Structured Streaming computes real-time features in 5-minute windows
- •Feast Feature Store provides consistent feature serving between training and inference
- •Airflow DAGs orchestrate complex multi-step data and training pipelines
- •Online feature stores in Redis serve low-latency features for real-time inference
0 of 4 lessons marked complete
Progress is kept in this browser only.