AI Model Wiki

    Skip to main content
    28 min
    Lesson 2 of 4

    Real-Time and Streaming Pipelines

    From the course AI Data Pipeline Engineering

    Build real-time data pipelines that feed online ML models with fresh, processed features.

    Streaming Data Processing

    Kafka + Spark Structured Streaming

    from pyspark.sql.functions import from_json, col, window
    from pyspark.sql.types import StructType, StringType
    
    # Define schema for incoming events
    schema = StructType() \
        .add("user_id", StringType()) \
        .add("event_type", StringType()) \
        .add("timestamp", StringType()) \
        .add("properties", StringType())
    
    # Read from Kafka
    events = spark \
        .readStream \
        .format("kafka") \
        .option("kafka.bootstrap.servers", "kafka:9092") \
        .option("subscribe", "user-events") \
        .load() \
        .select(from_json(col("value").cast("string"), schema).alias("data")) \
        .select("data.*")
    
    # Compute real-time features (5-minute windows)
    user_features = events \
        .groupBy(
            col("user_id"),
            window(col("timestamp"), "5 minutes")
        ) \
        .agg(
            count("event_type").alias("event_count_5m"),
            countDistinct("event_type").alias("unique_events_5m")
        )
    
    # Write to online feature store
    user_features.writeStream \
        .foreachBatch(update_feature_store) \
        .outputMode("update") \
        .start()
    
    def update_feature_store(batch_df, epoch_id):
        for row in batch_df.collect():
            redis_client.hset(
                f"user:{row.user_id}:features",
                mapping={
                    "event_count_5m": row.event_count_5m,
                    "unique_events_5m": row.unique_events_5m,
                    "updated_at": datetime.utcnow().isoformat()
                }
            )
    

    Feature Engineering Pipeline

    Automated Feature Generation

    from feast import FeatureStore, Entity, FeatureView, Field
    from feast.types import Int64, Float64, String
    
    # Define entities
    user = Entity(name="user_id", join_keys=["user_id"])
    
    # Define feature view
    user_activity_fv = FeatureView(
        name="user_activity_features",
        entities=[user],
        ttl=timedelta(days=1),
        schema=[
            Field(name="purchase_count_7d", dtype=Int64),
            Field(name="avg_order_value_30d", dtype=Float64),
            Field(name="preferred_category", dtype=String),
            Field(name="churn_risk_score", dtype=Float64),
        ],
        source=user_activity_source
    )
    
    # Retrieve features for prediction
    store = FeatureStore(repo_path="feature_repo/")
    
    feature_vector = store.get_online_features(
        features=[
            "user_activity_features:purchase_count_7d",
            "user_activity_features:avg_order_value_30d",
            "user_activity_features:churn_risk_score"
        ],
        entity_rows=[{"user_id": user_id}]
    ).to_dict()
    

    Pipeline Orchestration with Airflow

    from airflow import DAG
    from airflow.operators.python import PythonOperator
    
    with DAG("ml_training_pipeline", schedule_interval="@daily") as dag:
        
        extract = PythonOperator(
            task_id="extract_data",
            python_callable=extract_from_sources
        )
        
        transform = PythonOperator(
            task_id="transform_features", 
            python_callable=engineer_features
        )
        
        validate = PythonOperator(
            task_id="validate_data_quality",
            python_callable=run_quality_checks
        )
        
        train = PythonOperator(
            task_id="train_model",
            python_callable=train_ml_model
        )
        
        evaluate = PythonOperator(
            task_id="evaluate_and_promote",
            python_callable=evaluate_model
        )
        
        extract >> transform >> validate >> train >> evaluate
    

    Key Takeaways

    • •Kafka + Structured Streaming computes real-time features in 5-minute windows
    • •Feast Feature Store provides consistent feature serving between training and inference
    • •Airflow DAGs orchestrate complex multi-step data and training pipelines
    • •Online feature stores in Redis serve low-latency features for real-time inference
    0 of 4 lessons marked complete

    Progress is kept in this browser only.