研究torch0.4.1 rnn的实现

公式

说明

  1. 所有测试代码以此为准。
  2. 以torch0.4.1的代码为准进行debug rnn代码， 可以pip install 指定版本的torch.

从测试代码可以看到：

  1. num_layers相当初始化多少层rnn layer，比如此例子的10，
  2. input_size表示上一层的hidden_size(例如本例子的embedding)
  3. hidden_size表示rnn的输出hidden_size
  4. batch_first简单来讲就是内存存储的问题，if batch_first=True，那么维度(batch_size, seq_length, hidden_size)，否则seq_length和batch_size对调，另外cudnn底层就是seq_length放在第一位，torch底层也会根据batch_first进行转换

1. 初始化一些参数

以embedding出来为准，输出维度为: (32, 3, 30)
rnn第一步初始化hx(10, 32, 4)，一共初始化了num_layer=10层，batch_size=32，hidden_size=4，白话文就是每一层rnn都有一个(32, 4)的矩阵来保存最后一个时刻(ht)的结果。

这个可以看class RNNBase(Module):。
另外还有2个weight，2个bias的初始化。即公式中的w和b

2. AutogradRNN

初始化参数后，调用此方法，进行rnn计算，如果支持GPU，就会使用cudnn提供好的，此处忽略这里。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
def AutogradRNN(mode, input_size, hidden_size, num_layers=1, batch_first=False,
                dropout=0, train=True, bidirectional=False, variable_length=False,
                dropout_state=None, flat_weight=None):
    # 使用不同的cell进行计算，每一层rnn layer即一个cell。
    if mode == 'RNN_RELU':
        cell = RNNReLUCell
    elif mode == 'RNN_TANH':
        cell = RNNTanhCell
    elif mode == 'LSTM':
        cell = LSTMCell
    elif mode == 'GRU':
        cell = GRUCell
    else:
        raise Exception('Unknown mode: {}'.format(mode))
    # rec_factory即表示Recurrent，就是第四步的函数
    rec_factory = variable_recurrent_factory if variable_length else Recurrent

    if bidirectional:
        # 可以看到哦，反向就是进行reverse一下
        # 对应代码：steps = range(input.size(0) - 1, -1, -1) if reverse else range(input.size(0))
        # 意思就是从最后一个时刻反向进行更新hidden_size（不同时刻的y）。
        layer = (rec_factory(cell), rec_factory(cell, reverse=True))
    else:
        layer = (rec_factory(cell),)
    # 这地方疯狂使用闭包...
    func = StackedRNN(layer,
                      num_layers,
                      (mode == 'LSTM'),
                      dropout=dropout,
                      train=train)

    def forward(input, weight, hidden, batch_sizes):
        if batch_first and not variable_length:
            # 囔，这里就会batch_size 和  sequence_length对调
            input = input.transpose(0, 1)

        nexth, output = func(input, hidden, weight, batch_sizes)

        if batch_first and not variable_length:
            output = output.transpose(0, 1)

        return output, nexth

    return forward

3. StackedRNN

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

def StackedRNN(inners, num_layers, lstm=False, dropout=0, train=True):
    
    """
     拿到每一层Recurrent layer计算出来的结果，进行concat到一起进行返回
     
     inner表示 Recurrent，代码如下

    """

    num_directions = len(inners)
    total_layers = num_layers * num_directions

    def forward(input, hidden, weight, batch_sizes):
        """
        input: 3, 32, 30 (sequence_length, batch_size, embedding input hidden_size),
        hidden: 10, 32, 4 (num_layers, batch_size, output hidden_size),
        """
        assert(len(weight) == total_layers)
        next_hidden = []

        if lstm:
            hidden = list(zip(*hidden))
        # 遍历rnn中num_layers
        for i in range(num_layers):
            all_output = []
            for j, inner in enumerate(inners): # inner表示 Recurrent，代码如下
                l = i * num_directions + j

                hy, output = inner(input, hidden[l], weight[l], batch_sizes)
                next_hidden.append(hy)
                all_output.append(output)

            input = torch.cat(all_output, input.dim() - 1)

            if dropout != 0 and i < num_layers - 1:
                input = F.dropout(input, p=dropout, training=train, inplace=False)

        if lstm:
            next_h, next_c = zip(*next_hidden)
            next_hidden = (
                torch.cat(next_h, 0).view(total_layers, *next_h[0].size()),
                torch.cat(next_c, 0).view(total_layers, *next_c[0].size())
            )
        else:
            next_hidden = torch.cat(next_hidden, 0).view(
                total_layers, *next_hidden[0].size())

        return next_hidden, input

    return forward

4. Recurrent

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
def Recurrent(inner, reverse=False):
    """
    表示单层layer的计算
    """
    def forward(input, hidden, weight, batch_sizes):
        """
        input: 3, 32, 30
        hidden: 32, 4 (表示遍历每一个单独的rnn进来，一共10次)
        weight:List[  (4, 30), (4,4), (4, ), (4, ) ]
        """
        output = []
        steps = range(input.size(0) - 1, -1, -1) if reverse else range(input.size(0))
        for i in steps: # steps为3,因为sequence_length为3
            hidden = inner(input[i], hidden, *weight) # hidden就代表上一个时刻的输出， inner为RNNTanhCell，代码如下
            # hack to handle LSTM
            output.append(hidden[0] if isinstance(hidden, tuple) else hidden)

        if reverse:
            output.reverse()
        output = torch.cat(output, 0).view(input.size(0), *output[0].size())
        # 此处hidden就代表最后一个时刻的输出(32, 4)，output表示每一个时刻的输出(3, 32, 4)
        # torch.any(output[-1,:,:] == hidden)
        return hidden, output

    return forward

5. Recurrent中的inner：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# 针对rnn
def RNNTanhCell(input, hidden, w_ih, w_hh, b_ih=None, b_hh=None):
    """
    input: 32, 30
    hidden: 32, 4
    w_ih: 4,30
    w_hh: 4,4
    b_ih: 4
    b_hh: 4
    """
    hy = torch.tanh(F.linear(input, w_ih, b_ih) + F.linear(hidden, w_hh, b_hh))
    return hy

# 针对lstm cell.
def LSTMCell(input, hidden, w_ih, w_hh, b_ih=None, b_hh=None):
    """
    input: (32, 30)
    hidden: ( (32, 4), (32, 4) )
    w_ih: (16, 30)
    w_hh: (16, 4)
    b_ih: (16, )
    b_hh: (16, )
    """
    if input.is_cuda: # 自行忽略此处代码
        igates = F.linear(input, w_ih)
        hgates = F.linear(hidden[0], w_hh)
        state = fusedBackend.LSTMFused.apply
        return state(igates, hgates, hidden[1]) if b_ih is None else state(igates, hgates, hidden[1], b_ih, b_hh)

    hx, cx = hidden
    # 看到么，也是rnn公式哦
    gates = F.linear(input, w_ih, b_ih) + F.linear(hx, w_hh, b_hh)
    # 但是此处不一样了，在第一维分割4份出来
    ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)
    # 看到下面这几行公式么，为啥子就能解决梯度消失呢？
    # 神不神奇，意不意外
    ingate = torch.sigmoid(ingate)
    forgetgate = torch.sigmoid(forgetgate)
    cellgate = torch.tanh(cellgate)
    outgate = torch.sigmoid(outgate)

    cy = (forgetgate * cx) + (ingate * cellgate)
    hy = outgate * torch.tanh(cy)
    # hy: (32,4), (32,4)
    return hy, cy

# 针对gru
def GRUCell(input, hidden, w_ih, w_hh, b_ih=None, b_hh=None):

    if input.is_cuda:
        gi = F.linear(input, w_ih)
        gh = F.linear(hidden, w_hh)
        state = fusedBackend.GRUFused.apply
        return state(gi, gh, hidden) if b_ih is None else state(gi, gh, hidden, b_ih, b_hh)

    # 这里就变了哦
    gi = F.linear(input, w_ih, b_ih)
    gh = F.linear(hidden, w_hh, b_hh)

    i_r, i_i, i_n = gi.chunk(3, 1)
    h_r, h_i, h_n = gh.chunk(3, 1)

    resetgate = torch.sigmoid(i_r + h_r)
    inputgate = torch.sigmoid(i_i + h_i)
    newgate = torch.tanh(i_n + resetgate * h_n)
    hy = newgate + inputgate * (hidden - newgate)

    return hy

6. 测试代码完整如下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
# -*- coding: utf8 -*-
#
from torch.nn import Embedding, Sigmoid, Linear
from torch.nn.modules.rnn import RNN

import torch
from torch.nn import BCELoss
from torch.optim import RMSprop


class RNNModel(torch.nn.Module):
    def __init__(self):
        super(RNNModel, self).__init__()
        self.embed = Embedding(6, 30)
        self.rnn = RNN(input_size=30, hidden_size=4, batch_first=True, num_layers=10)
        self.linear = Linear(4, 1)

    def forward(self, input):
        input = self.embed(input)
        # rnn_out: (32, 3, 4) 返回batch_size下每个时刻(sequence_length)的最终输出
        # _ : (10, 32, 4) 表示batch_size下每个rnn layer最后一个时刻的输出
        rnn_out, _ = self.rnn(input)
        y_pred = self.linear(_.sum(0))
        return y_pred


class RNNDemo(object):
    def __init__(self):
        self.model = RNNModel()

        self.optim = RMSprop(self.model.parameters())

    def loss(self, y_pred, y_true):
        y_pred = Sigmoid()(y_pred)
        return BCELoss()(y_pred, y_true.float())

    def train(self, x, y):
        for i in range(100):
            for _x, _y in zip(x, y):
                _x = _x.repeat(32, 1)
                _y = _y.repeat(32, 1)
                y_pred = self.model(_x)
                loss = self.loss(y_pred, _y)
                loss.backward()
                print(loss.item())
                self.optim.step()
                self.optim.zero_grad()


if __name__ == '__main__':
    train_data = [
        (['A', 'B', 'C'], 'right'),
        (['C', 'D', 'E'], 'false')
    ]
    id_map = {
        'A': 1,
        'B': 2,
        'C': 3,
        'D': 4,
        'E': 5,
        'right': 6,
        'false': 7

    }
    rnn = RNNDemo()
    rnn.train(
        [torch.tensor([id_map[i] for i in sample[0]]).reshape(1, 3) for sample in train_data],
        [torch.tensor(id_map[sample[1]]).reshape(1, 1) for sample in train_data]
    )


至此，前向过程完结。

赏

谢谢你请我吃糖果

支付宝

扫一扫，分享到微信

微信分享二维码

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

探索世界美好的存在。