AI Model Wiki

    Skip to main content
    28 min
    Lesson 2 of 5

    Distributed Training at Scale

    From the course GPU Cluster Management

    Execute distributed training across GPU clusters with data and model parallelism.

    Multi-Node Training Setup

    PyTorch Distributed

    import torch.distributed as dist
    import torch.multiprocessing as mp
    
    def train(rank, world_size):
        # Initialize distributed process group
        dist.init_process_group(
            backend='nccl',
            init_method='env://',
            world_size=world_size,
            rank=rank
        )
        
        # Model on this GPU
        model = YourModel().to(rank)
        model = torch.nn.parallel.DistributedDataParallel(
            model,
            device_ids=[rank]
        )
        
        # Training loop
        for epoch in range(100):
            for batch in dataloader:
                output = model(batch)
                loss = criterion(output)
                loss.backward()
                optimizer.step()
    
    # Launch on each node
    if __name__ == '__main__':
        world_size = 128  # 16 nodes × 8 GPUs
        mp.spawn(train, args=(world_size,), nprocs=8)
    

    DeepSpeed ZeRO Optimization

    import deepspeed
    
    # Configure ZeRO for massive models
    ds_config = {
        "train_batch_size": 2048,
        "gradient_accumulation_steps": 16,
        "fp16": {"enabled": True},
        "zero_optimization": {
            "stage": 3,  # Partition optimizer, gradients, parameters
            "offload_optimizer": {"device": "cpu"},  # CPU offload
            "offload_param": {"device": "cpu"},
            "overlap_comm": True,  # Overlap communication
        }
    }
    
    model_engine, optimizer, _, _ = deepspeed.initialize(
        model=model,
        config=ds_config
    )
    
    # Train trillion-parameter models
    for batch in dataloader:
        loss = model_engine(batch)
        model_engine.backward(loss)
        model_engine.step()
    

    Fault Tolerance

    Checkpointing Strategy

    # Automatic checkpointing every N steps
    def save_checkpoint(model, optimizer, epoch, step):
        checkpoint = {
            'epoch': epoch,
            'step': step,
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
        }
        torch.save(checkpoint, f'/shared/checkpoints/ckpt_{step}.pt')
    
    # Resume from failure
    def load_checkpoint(model, optimizer):
        checkpoint = torch.load('/shared/checkpoints/latest.pt')
        model.load_state_dict(checkpoint['model_state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
        return checkpoint['epoch'], checkpoint['step']
    

    Key Takeaways

    • •PyTorch DDP enables data-parallel training across hundreds of GPUs
    • •DeepSpeed ZeRO partitions model state for trillion-parameter models
    • •Checkpointing every N steps enables recovery from node failures
    • •NCCL backend optimizes GPU communication over InfiniBand
    0 of 5 lessons marked complete

    Progress is kept in this browser only.