Hexo
0%

pytorch-optim

发表于 更新于 分类于

pytorch-optim

optim的学习率设置问题

1.不同的学习率

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# 第一种
optimizer = t.optim.Adam(model.parameters(), lr = 0.1)

# 第二种
optim_group = [{'params':model.net1.parameters(),'lr':0.4},
       {'params':model.net2.parameters(),'lr':0.1}]
optimizer = t.optim.Adam(optim_group,lr=0.04)

# 第三种
ignored_params = list(map(id, model.model.fc.parameters() )) 
               + list(map(id, model.classifier.parameters() ))
base_params = filter(lambda p: id(p) not in ignored_params, model.parameters())
optimizer_ft = optim.SGD([
     {'params': base_params, 'lr': 0.01},
     {'params': model.model.fc.parameters(), 'lr': 0.1},
     {'params': model.classifier.parameters(), 'lr': 0.1}
 ], weight_decay=5e-4, momentum=0.9, nesterov=True)

2. 学习率衰减

1
2
3
4
5
6
7
8
9
exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=40, gamma=0.1)
def train_model(model, criterion, optimizer, scheduler, num_epochs=25):

    for epoch in range(num_epochs):
        scheduler.step()

        for data in dataloaders[phase]:
            optimizer.zero_grad()

3. iterools.chain

1
2
3
4
5
6
# initialize optimizers
self.optimizer_G = torch.optim.Adam(filter(lambda p: p.requires_grad, itertools.chain(self.netG_A.parameters(), self.netG_B.parameters())), lr=opt.lr, betas=(opt.beta1, 0.999))
self.optimizer_D = torch.optim.Adam(filter(lambda p: p.requires_grad, itertools.chain(self.netD_A.parameters(), self.netD_B.parameters())), lr=opt.lr, betas=(opt.beta1, 0.999))
self.optimizers = []
self.optimizers.append(self.optimizer_G)
self.optimizers.append(self.optimizer_D)