梯度下降变体：SGD、Adam、RMSProp 对比实验

梯度下降变体：SGD、Adam、RMSProp 对比实验

1. 技术分析

1.1 梯度下降算法对比

1.2 算法特性对比

1.3 优化地形可视化

优化地形示意图 全局最小值 ▼ ┌─────────────┐ / \ / \ / \ └───────────────────┘ 鞍点 局部最小值

2. 核心功能实现

2.1 SGD 及其变体

import torch class SGD(torch.optim.Optimizer): def __init__(self, params, lr=0.01, momentum=0, weight_decay=0): defaults = dict(lr=lr, momentum=momentum, weight_decay=weight_decay) super().__init__(params, defaults) @torch.no_grad() def step(self): for group in self.param_groups: lr = group['lr'] momentum = group['momentum'] weight_decay = group['weight_decay'] for p in group['params']: if p.grad is None: continue grad = p.grad.data if weight_decay != 0: grad.add_(p.data, alpha=weight_decay) if momentum != 0: state = self.state[p] if 'momentum_buffer' not in state: buf = state['momentum_buffer'] = grad.clone() else: buf = state['momentum_buffer'] buf.mul_(momentum).add_(grad) grad = buf p.data.add_(grad, alpha=-lr) class NesterovSGD(torch.optim.Optimizer): def __init__(self, params, lr=0.01, momentum=0.9): defaults = dict(lr=lr, momentum=momentum) super().__init__(params, defaults) @torch.no_grad() def step(self): for group in self.param_groups: lr = group['lr'] momentum = group['momentum'] for p in group['params']: if p.grad is None: continue grad = p.grad.data state = self.state[p] if 'momentum_buffer' not in state: buf = state['momentum_buffer'] = torch.zeros_like(p.data) else: buf = state['momentum_buffer'] buf.mul_(momentum).add_(grad) p.data.add_(buf, alpha=-lr)

2.2 RMSProp 实现

class RMSProp(torch.optim.Optimizer): def __init__(self, params, lr=0.01, alpha=0.99, eps=1e-8, weight_decay=0): defaults = dict(lr=lr, alpha=alpha, eps=eps, weight_decay=weight_decay) super().__init__(params, defaults) @torch.no_grad() def step(self): for group in self.param_groups: lr = group['lr'] alpha = group['alpha'] eps = group['eps'] weight_decay = group['weight_decay'] for p in group['params']: if p.grad is None: continue grad = p.grad.data if weight_decay != 0: grad.add_(p.data, alpha=weight_decay) state = self.state[p] if 'square_avg' not in state: square_avg = state['square_avg'] = torch.zeros_like(p.data) square_avg = state['square_avg'] square_avg.mul_(alpha).addcmul_(grad, grad, value=1 - alpha) p.data.addcdiv_(grad, square_avg.sqrt().add_(eps), value=-lr) class Adagrad(torch.optim.Optimizer): def __init__(self, params, lr=0.01, eps=1e-10): defaults = dict(lr=lr, eps=eps) super().__init__(params, defaults) @torch.no_grad() def step(self): for group in self.param_groups: lr = group['lr'] eps = group['eps'] for p in group['params']: if p.grad is None: continue grad = p.grad.data state = self.state[p] if 'sum' not in state: sum_ = state['sum'] = torch.zeros_like(p.data) sum_ = state['sum'] sum_.addcmul_(grad, grad) p.data.addcdiv_(grad, sum_.sqrt().add_(eps), value=-lr)

2.3 Adam 实现

class Adam(torch.optim.Optimizer): def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) super().__init__(params, defaults) @torch.no_grad() def step(self): import math for group in self.param_groups: lr = group['lr'] beta1, beta2 = group['betas'] eps = group['eps'] weight_decay = group['weight_decay'] for p in group['params']: if p.grad is None: continue grad = p.grad.data if weight_decay != 0: grad.add_(p.data, alpha=weight_decay) state = self.state[p] if len(state) == 0: state['step'] = 0 state['exp_avg'] = torch.zeros_like(p.data) state['exp_avg_sq'] = torch.zeros_like(p.data) exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] state['step'] += 1 exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) bias_correction1 = 1 - beta1 ** state['step'] bias_correction2 = 1 - beta2 ** state['step'] denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps) step_size = lr / bias_correction1 p.data.addcdiv_(exp_avg, denom, value=-step_size)

3. 性能对比

3.1 收敛速度对比

3.2 不同学习率下的表现

3.3 参数敏感性对比

4. 最佳实践

4.1 优化器选择指南

def select_optimizer(model, task_type): if task_type == 'computer_vision': return torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) elif task_type == 'nlp': return torch.optim.Adam(model.parameters(), lr=1e-4) elif task_type == 'reinforcement_learning': return torch.optim.RMSprop(model.parameters(), lr=1e-3) else: return torch.optim.Adam(model.parameters(), lr=1e-3) class OptimizerRecommendation: @staticmethod def based_on_data_size(data_size): if data_size < 1000: return {'optimizer': 'adam', 'lr': 1e-3} elif data_size < 10000: return {'optimizer': 'adamw', 'lr': 1e-4} else: return {'optimizer': 'sgd', 'lr': 0.1, 'momentum': 0.9}

4.2 优化器切换策略

class OptimizerSwitcher: def __init__(self, model): self.model = model self.optimizers = { 'sgd': torch.optim.SGD(model.parameters(), lr=0.1), 'adam': torch.optim.Adam(model.parameters(), lr=1e-3), 'rmsprop': torch.optim.RMSprop(model.parameters(), lr=1e-3) } self.current = 'adam' def switch(self, optimizer_name): if optimizer_name in self.optimizers: self.current = optimizer_name else: raise ValueError(f"Unknown optimizer: {optimizer_name}") def step(self): self.optimizers[self.current].step() def zero_grad(self): self.optimizers[self.current].zero_grad()

5. 总结

选择合适的优化器是训练成功的关键：

1. SGD：简单但需要调优，适合大规模数据
2. Momentum：加速收敛，适合非凸优化
3. RMSProp：自适应学习率，适合不稳定目标
4. Adam：综合动量和自适应，通用首选

对比数据如下：

• Adam 在大多数场景下表现最佳
• SGD 在大规模数据上可能更优
• RMSProp 在不稳定目标上表现更好
• 推荐从 Adam 开始，根据结果调整