pytorch学习率调整

2021-07-02

keras

在keras中，比如动态调整学习率，可以：


import tensorflow as tf


def step_decay(epoch):
        if epoch < 3:
            lr = 1e-5
        else:
            lr = 1e-6
        return lr

tf.keras.callbacks.LearningRateScheduler(step_decay, verbose=2)

lr_scheduler

在pytorch中，提供了torch.optim.lr_scheduler

1. StepLR

# -*- coding: utf8 -*-
#

import torch
from pyecharts import options
from pyecharts.charts import Line
from torch import optim
from torch.nn import Linear
from torch.optim import lr_scheduler


class TestModel(torch.nn.Module):
    def __init__(self):
        super(TestModel, self).__init__()
        self.linear = Linear(100, 2)

    def forward(self, x):
        return self.linear(x)


def line_graph(xs, ys):
    line = Line()
    line.add_xaxis(xs)
    line.add_yaxis(series_name='学习率', y_axis=ys, is_smooth=True)
    line.set_global_opts(
        title_opts=options.TitleOpts(title='学习率调整图'),
        toolbox_opts=options.ToolboxOpts()
    )
    line.set_series_opts(
        label_opts=options.LabelOpts(is_show=False),
        # markline_opts=options.MarkLineOpts(
        #     # 设置平均值的标记线
        #     data=[options.MarkLineItem(name='平均值', type_='average')],
        #     # 设置最大值的标记线
        #     # data = [options.MarkLineItem(name='最大值', type_='max')]
        # )
    )

    line.render('折线图.html')


model = TestModel()

optimizer = optim.Adam(params=model.parameters(), lr=0.05)

# Assuming optimizer uses lr = 0.05 for all groups
# lr = 0.05     if epoch < 30
# lr = 0.005    if 30 <= epoch < 60
# lr = 0.0005   if 60 <= epoch < 90

scheduler = lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)

x = list(range(100))
y = []
for epoch in range(100):
    optimizer.step()
    scheduler.step()
    lr = scheduler.get_lr()
    y.append(scheduler.get_lr()[0])

line_graph(x, y)

2. MultiStepLR


scheduler = lr_scheduler.MultiStepLR(optimizer, [30, 80], 0.1)

这个可以设置区间，在30 ~ 80 为一个学习率

3. ExponentialLR

1	scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=0.9)

指数衰减

transformers库

在transformers库中，也提供了一些，比如：

1. get_linear_schedule_with_warmup

学习率预热

1	num_warmup_steps = 0.05 * len(train_dataloader) * epochs

optimizer = optim.Adam(params=model.parameters(), lr=1e-3)

scheduler = get_linear_schedule_with_warmup(
    optimizer=optimizer,
    num_warmup_steps=10000,
    num_training_steps=100000)

学习率先不断上升，然后再不断减小。

在预热期间，学习率从0线性增加到优化器中的初始lr。随后线性降低到0

理论