深度学习正则化：防止过拟合的核心技术

深度学习正则化：防止过拟合的核心技术

1. 技术分析

1.1 正则化方法分类

1.2 过拟合表现

• 训练损失低，测试损失高
• 模型在训练数据上表现很好，但新数据上表现差
• 模型参数值过大或过小

2. 核心功能实现

2.1 L1和L2正则化

import torch import torch.nn as nn from torch.keras.regularizers import l1, l2, l1_l2 class RegularizedModel(nn.Module): def __init__(self): super().__init__() self.fc1 = nn.Linear(784, 256) self.fc2 = nn.Linear(256, 128) self.fc3 = nn.Linear(128, 10) # L2正则化 self.reg_lambda = 0.01 def forward(self, x): x = torch.relu(self.fc1(x)) x = torch.relu(self.fc2(x)) x = self.fc3(x) return x def l2_loss(self): l2_reg = 0 for param in self.parameters(): l2_reg += torch.norm(param, p=2) ** 2 return self.reg_lambda * l2_reg # 使用PyTorch内置正则化 model = nn.Sequential( nn.Linear(784, 256), nn.ReLU(), nn.Linear(256, 10) ) # 方式1：weight_decay（L2正则化） optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=0.01) # 方式2：直接应用正则化 optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

2.2 Dropout实现

import torch import torch.nn as nn class DropoutModel(nn.Module): def __init__(self, input_dim=784, hidden_dims=[256, 128], output_dim=10, dropout_rate=0.5): super().__init__() layers = [] prev_dim = input_dim for hidden_dim in hidden_dims: layers.append(nn.Linear(prev_dim, hidden_dim)) layers.append(nn.ReLU()) layers.append(nn.Dropout(p=dropout_rate)) prev_dim = hidden_dim layers.append(nn.Linear(prev_dim, output_dim)) self.network = nn.Sequential(*layers) def forward(self, x): return self.network(x) # 测试时关闭Dropout model = DropoutModel(dropout_rate=0.5) model.eval() # 评估模式，自动关闭Dropout # 训练时开启Dropout model.train() # 训练模式，自动开启Dropout

2.3 数据增强

import torchvision.transforms as T class DataAugmentation: def __init__(self, img_size=224): self.train_transform = T.Compose([ T.RandomResizedCrop(img_size, scale=(0.8, 1.0)), T.RandomHorizontalFlip(p=0.5), T.RandomRotation(degrees=15), T.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1), T.RandomAffine(degrees=0, translate=(0.1, 0.1)), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) self.val_transform = T.Compose([ T.Resize((img_size, img_size)), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) def get_train_transform(self): return self.train_transform def get_val_transform(self): return self.val_transform # CutMix增强 class CutMix: def __init__(self, alpha=1.0, p=0.5): self.alpha = alpha self.p = p def __call__(self, batch_images, batch_labels): if torch.rand(1).item() > self.p: return batch_images, batch_labels batch_size = batch_images.size(0) indices = torch.randperm(batch_size) lam = np.random.beta(self.alpha, self.alpha) _, _, H, W = batch_images.shape cut_rat = np.sqrt(1.0 - lam) cut_w = int(W * cut_rat) cut_h = int(H * cut_rat) cx = torch.randint(0, W, (1,)).item() cy = torch.randint(0, H, (1,)).item() bbx1 = max(0, cx - cut_w // 2) bby1 = max(0, cy - cut_h // 2) bbx2 = min(W, cx + cut_w // 2) bby2 = min(H, cy + cut_h // 2) batch_images[:, :, bby1:bby2, bbx1:bbx2] = batch_images[indices, :, bby1:bby2, bbx1:bbx2] lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (W * H)) batch_labels = lam * batch_labels + (1 - lam) * batch_labels[indices] return batch_images, batch_labels

2.4 Early Stopping

class EarlyStopping: def __init__(self, patience=10, min_delta=0.001, mode='min'): self.patience = patience self.min_delta = min_delta self.mode = mode self.counter = 0 self.best_score = None self.early_stop = False def step(self, score): if self.best_score is None: self.best_score = score return False if self.mode == 'min': improved = score < self.best_score - self.min_delta else: improved = score > self.best_score + self.min_delta if improved: self.best_score = score self.counter = 0 else: self.counter += 1 if self.counter >= self.patience: self.early_stop = True return self.early_stop # 使用 early_stopping = EarlyStopping(patience=10, mode='min') for epoch in range(100): train_loss = train_one_epoch(model, train_loader) val_loss = evaluate(model, val_loader) if early_stopping.step(val_loss): print(f"早停触发于 epoch {epoch+1}") break

3. 性能对比

3.1 正则化效果对比

3.2 Dropout率对比

4. 最佳实践

4.1 正则化方法选择

4.2 超参数调优

# 学习率查找器 def find_learning_rate(model, train_loader, optimizer_class): lrs = [1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1] losses = [] for lr in lrs: optimizer = optimizer_class(model.parameters(), lr=lr) for inputs, targets in train_loader: optimizer.zero_grad() outputs = model(inputs) loss = torch.nn.functional.cross_entropy(outputs, targets) loss.backward() optimizer.step() losses.append(loss.item()) break return lrs[losses.index(min(losses))]

5. 总结

正则化是防止过拟合的关键技术：

1. L2正则化：最常用，weight_decay参数简单易用
2. Dropout：训练时随机失活，效果显著
3. 数据增强：增加数据多样性，提高泛化能力
4. 早停法：简单有效的正则化策略