python bdist_wheel

利用cython进行编译成库.

1. setup.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
# coding: utf-8
import os
import sysconfig


from setuptools import setup, find_packages

# https://stackoverflow.com/questions/21594925/error-each-element-of-ext-modules-option-must-be-an-extension-instance-or-2-t
# 注意，这个顺序必须在这里，要比setup低.
from Cython.Build import cythonize

from setuptools.command.build_py import build_py as _build_py


def read():
    with open("./requirements.txt", "r") as f:
        return f.readlines()


# Notice1: your modules.
packages = set(find_packages()) | {'your modules.'}

# Notice2: exclude files.
EXCLUDE_FILES = [
    # 'your package/ignore python files.'
    'package/file1.py'

]


def get_ext_paths(root_dir, exclude_files):
    """get filepaths for compilation"""
    paths = []

    for root, dirs, files in os.walk(root_dir):
        for filename in files:
            if os.path.splitext(filename)[1] != '.py':
                continue

            file_path = os.path.join(root, filename)
            if file_path in exclude_files:
                continue

            paths.append(file_path)
    return paths


# noinspection PyPep8Naming
class build_py(_build_py):

    def find_package_modules(self, package, package_dir):
        ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
        modules = super().find_package_modules(package, package_dir)
        filtered_modules = []
        for (pkg, mod, filepath) in modules:
            if os.path.exists(filepath.replace('.py', ext_suffix)):
                continue
            filtered_modules.append((pkg, mod, filepath,))
        return filtered_modules


setup(
    name='package name', # Notice3
    version='0.3.0',

    ext_modules=cythonize(
        get_ext_paths('project root path', EXCLUDE_FILES), # Notice4
        compiler_directives={'language_level': 3.6}
    ),
    cmdclass={
        'build_py': build_py
    },
    packages=packages,
    include_package_data=True,
    install_requires=read(),
)

2. build

1
2
3
python setup.py build_ext --inplace
python setup.py bdist_wheel

3. 结论

通过这种方式编译成一个库，正常情况下无法查看源码。
在开发过程中，IDE无法自动导入，就正常打包方便开发同学调试、使用，而在生产环境使用2方式进行编译成一个安装包，开发替换一下安装包路径即可。

展开全文 >>

pytorch学习率调整

keras

在keras中，比如动态调整学习率，可以：

1
2
3
4
5
6
7
8
9
10
11
12
13
14

import tensorflow as tf


def step_decay(epoch):
        if epoch < 3:
            lr = 1e-5
        else:
            lr = 1e-6
        return lr

tf.keras.callbacks.LearningRateScheduler(step_decay, verbose=2)


lr_scheduler

在pytorch中，提供了torch.optim.lr_scheduler

1. StepLR

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
# -*- coding: utf8 -*-
#

import torch
from pyecharts import options
from pyecharts.charts import Line
from torch import optim
from torch.nn import Linear
from torch.optim import lr_scheduler


class TestModel(torch.nn.Module):
    def __init__(self):
        super(TestModel, self).__init__()
        self.linear = Linear(100, 2)

    def forward(self, x):
        return self.linear(x)


def line_graph(xs, ys):
    line = Line()
    line.add_xaxis(xs)
    line.add_yaxis(series_name='学习率', y_axis=ys, is_smooth=True)
    line.set_global_opts(
        title_opts=options.TitleOpts(title='学习率调整图'),
        toolbox_opts=options.ToolboxOpts()
    )
    line.set_series_opts(
        label_opts=options.LabelOpts(is_show=False),
        # markline_opts=options.MarkLineOpts(
        #     # 设置平均值的标记线
        #     data=[options.MarkLineItem(name='平均值', type_='average')],
        #     # 设置最大值的标记线
        #     # data = [options.MarkLineItem(name='最大值', type_='max')]
        # )
    )

    line.render('折线图.html')


model = TestModel()

optimizer = optim.Adam(params=model.parameters(), lr=0.05)

# Assuming optimizer uses lr = 0.05 for all groups
# lr = 0.05     if epoch < 30
# lr = 0.005    if 30 <= epoch < 60
# lr = 0.0005   if 60 <= epoch < 90

scheduler = lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)

x = list(range(100))
y = []
for epoch in range(100):
    optimizer.step()
    scheduler.step()
    lr = scheduler.get_lr()
    y.append(scheduler.get_lr()[0])

line_graph(x, y)


2. MultiStepLR

1
2
3
4

scheduler = lr_scheduler.MultiStepLR(optimizer, [30, 80], 0.1)


这个可以设置区间，在30 ~ 80 为一个学习率

3. ExponentialLR

1
scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=0.9)

指数衰减

transformers库

在transformers库中，也提供了一些，比如：

1. get_linear_schedule_with_warmup

学习率预热

1
num_warmup_steps = 0.05 * len(train_dataloader) * epochs
1
2
3
4
5
6
optimizer = optim.Adam(params=model.parameters(), lr=1e-3)

scheduler = get_linear_schedule_with_warmup(
    optimizer=optimizer,
    num_warmup_steps=10000,
    num_training_steps=100000)

学习率先不断上升，然后再不断减小。

在预热期间，学习率从0线性增加到优化器中的初始lr。随后线性降低到0

理论

展开全文 >>

批次数据处理

在数据处理里，常使用DataSet和DataLoader，关于具体使用此处不介绍，对于每一个batch_size里的数据来讲，
一般数据是shuffle=True，即表示打乱顺序，能够使数据更无规律和更为随机。但是如果对于数据样本不定长的情况或者说分布不均匀的情况下，
要使其定长，做法就是pad到一个固定长度，如果长短分布差距大呢？？

比如：

1
2
example1: 我有一个小摩托，我从来也不急，骑着我的小摩托，从此把它骑。
example2: 我爱南京。

如果将example2和example1pad到一个固定长度，pad太多。（虽然你可以扯进来mask，但是要在model的每一layer都要设置mask,关于mask此处忽略。）

如何在一个batch_size样本里面使数据分布更为接近？

kmean algo for clustering the feature by length.

kmeans等聚类算法就可以辅助解决，以每条数据的长度为依据。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75


def kmeans(x, k, max_it=32):
    r"""
    KMeans algorithm for clustering the sentences by length.

    Args:
        x (list[int]):
            The list of sentence lengths.
        k (int):
            The number of clusters.
            This is an approximate value. The final number of clusters can be less or equal to `k`.
        max_it (int):
            Maximum number of iterations.
            If centroids does not converge after several iterations, the algorithm will be early stopped.

    Returns:
        list[float], list[list[int]]:
            The first list contains average lengths of sentences in each cluster.
            The second is the list of clusters holding indices of data points.

    Examples:
        >>> x = torch.randint(10,20,(10,)).tolist()
        >>> x
        [15, 10, 17, 11, 18, 13, 17, 19, 18, 14]
        >>> centroids, clusters = kmeans(x, 3)
        >>> centroids
        [10.5, 14.0, 17.799999237060547]
        >>> clusters
        [[1, 3], [0, 5, 9], [2, 4, 6, 7, 8]]
    """

    # the number of clusters must not be greater than the number of datapoints
    x, k = torch.tensor(x, dtype=torch.float), min(len(x), k)
    # collect unique datapoints
    d = x.unique()
    # initialize k centroids randomly
    c = d[torch.randperm(len(d))[:k]]
    # assign each datapoint to the cluster with the closest centroid
    dists, y = torch.abs_(x.unsqueeze(-1) - c).min(-1)

    for _ in range(max_it):
        # if an empty cluster is encountered,
        # choose the farthest datapoint from the biggest cluster and move that the empty one
        mask = torch.arange(k).unsqueeze(-1).eq(y)
        none = torch.where(~mask.any(-1))[0].tolist()
        while len(none) > 0:
            for i in none:
                # the biggest cluster
                b = torch.where(mask[mask.sum(-1).argmax()])[0]
                # the datapoint farthest from the centroid of cluster b
                f = dists[b].argmax()
                # update the assigned cluster of f
                y[b[f]] = i
                # re-calculate the mask
                mask = torch.arange(k).unsqueeze(-1).eq(y)
            none = torch.where(~mask.any(-1))[0].tolist()
        # update the centroids
        c, old = (x * mask).sum(-1) / mask.sum(-1), c
        # re-assign all datapoints to clusters
        dists, y = torch.abs_(x.unsqueeze(-1) - c).min(-1)
        # stop iteration early if the centroids converge
        if c.equal(old):
            break
    # assign all datapoints to the new-generated clusters
    # the empty ones are discarded
    assigned = y.unique().tolist()
    # get the centroids of the assigned clusters
    centroids = c[assigned].tolist()
    # map all values of datapoints to buckets
    clusters = [torch.where(y.eq(i))[0].tolist() for i in assigned]

    return centroids, clusters


这样使用的话，那么batch_size就木有啥子用处了，有可能一个batch样本量为1,也有可能为设置的上限。

但是转念一想，其实也可以不用如此复杂，直接sorted by length也是可以的嘛，具体解释就忽略了。

展开全文 >>

数据增强

图像常见数据增强有翻转，旋转，缩放比例等不同的transforms。
对于文本，可以增加噪声干扰，也可以通过添加slide window（此处重点，嘿嘿。）

举个简单例子描述下滑动窗口。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
a = [1,2,3,4,5]

[a[i: i + 1] for i in range(0, len(a), 1)]
Out[36]: [[1], [2], [3], [4], [5]]

[a[i: i + 2] for i in range(0, len(a), 2)]
Out[37]: [[1, 2], [3, 4], [5]]

[a[i: i + 3] for i in range(0, len(a), 3)]
Out[38]: [[1, 2, 3], [4, 5]]

[a[i: i + 4] for i in range(0, len(a), 4)]
Out[39]: [[1, 2, 3, 4], [5]]

[a[i: i + 5] for i in range(0, len(a), 5)]
Out[40]: [[1, 2, 3, 4, 5]]


但是在某些训练结果下，其作用貌似并不大，反正可以尝试。

展开全文 >>

bert取平均向量方式2

这个方式来自HanLP，具体就不细讲了，大家看代码即可。

1、如果后续处理需要[CLS]和[SEP]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159

# -*- coding: utf8 -*-
#
from typing import List

import torch
from torch.nn.utils.rnn import pad_sequence
from transformers import AutoTokenizer, PreTrainedTokenizerBase, AutoModel


def tokenize(form: List[List[str]], tokenizer: PreTrainedTokenizerBase, max_length: int = 128, char_base: bool = False):
    """
    Args:
        form:
        tokenizer:
        max_length:
        char_base: 这里指的是form[即 word]是否是字级别的
    Returns:
    """
    res = tokenizer.batch_encode_plus(
        form,
        is_split_into_words=True,
        max_length=max_length,
        truncation=True,
    )
    result = res.data
    # 可用于长度大于指定长度过滤, overflow指字长度大于指定max_length，如果有cls,sep，那么就算上这个
    result['overflow'] = [len(encoding.overflowing) > 0 for encoding in res.encodings]
    if not char_base:
        word_index = []
        for encoding in res.encodings:
            word_index.append([])

            last_word_idx = -1
            current_length = 0
            for word_idx in encoding.word_ids[1:-1]:
                if word_idx != last_word_idx:
                    word_index[-1].append(current_length)

                current_length += 1
                last_word_idx = word_idx
        result['word_index'] = word_index
        result['word_attention_mask'] = [[True] * len(index) for index in word_index]

    # 加上CLS和SEP
    for index, word_index in enumerate(result['word_index']):
        add_1 = [i + 1 for i in word_index]
        result['word_index'][index] = [0, *add_1, add_1[-1] + 1]

    return result


def pick_tensor_for_each_token(h, token_span, average_subwords):
    if token_span is None:
        return h
    if average_subwords and token_span.size(-1) > 1:
        batch_size = h.size(0)
        h_span = h.gather(1, token_span.view(batch_size, -1).unsqueeze(-1).expand(-1, -1, h.shape[-1]))
        h_span = h_span.view(batch_size, *token_span.shape[1:], -1)
        n_sub_tokens = token_span.ne(0)
        n_sub_tokens[:, 0, 0] = True
        h_span = (h_span * n_sub_tokens.unsqueeze(-1)).sum(2)
        n_sub_tokens = n_sub_tokens.sum(-1).unsqueeze(-1)
        zero_mask = n_sub_tokens == 0
        if torch.any(zero_mask):
            n_sub_tokens[zero_mask] = 1  # avoid dividing by zero
        embed = h_span / n_sub_tokens
    else:
        embed = h.gather(1, token_span[:, :, 0].unsqueeze(-1).expand(-1, -1, h.size(-1)))
    return embed


def get_span(indices):
    """

    :param indices:
    :return:
    """
    span = []
    for index, item in enumerate(indices):
        try:
            next_item = indices[index + 1]
            if next_item > item:  # padding 后面都是0
                s = torch.arange(item, next_item, dtype=torch.long)
            else:
                s = torch.tensor([item], dtype=torch.long)
            span.append(s)
        except IndexError:
            span.append(torch.tensor([item], dtype=torch.long))
    return pad_sequence(span).T


def get_spans(indices: List[List[int]]):
    l = None

    batch = []
    for index_item in indices:
        if l is None:
            l = len(index_item)
        else:
            if len(index_item) != l:
                raise ValueError('padding first.')
        batch.append(get_span(indices=index_item))
    span = torch.zeros(size=(len(batch), batch[0].size(0), max([i.size(1) for i in batch])), dtype=torch.long)
    for index, item in enumerate(batch):
        b, l = item.shape
        span[index][:b, :l] = item
    return span


if __name__ == '__main__':
    tokens = [
        ['我', '爱', '北京', '。'],
        ['我', '爱', '北京', '和', '你们', '。'],
    ]

    tokenizer = AutoTokenizer.from_pretrained('hfl/chinese-electra-180g-small-discriminator', use_fast=True)

    output = tokenize(form=tokens, tokenizer=tokenizer)
    input_ids = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['input_ids']],
                             padding_value=tokenizer.pad_token_id, batch_first=True)
    token_type_ids = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['token_type_ids']],
                                  padding_value=tokenizer.pad_token_id, batch_first=True)
    attention_mask = pad_sequence([torch.tensor(_, dtype=torch.bool) for _ in output['attention_mask']],
                                  padding_value=tokenizer.pad_token_id, batch_first=True)

    word_index = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['word_index']],
                              padding_value=tokenizer.pad_token_id, batch_first=True)
    word_attention_mask = pad_sequence([torch.tensor(_, dtype=torch.bool) for _ in output['word_attention_mask']],
                                       padding_value=tokenizer.pad_token_id, batch_first=True)

    # model
    encoder = AutoModel.from_pretrained('hfl/chinese-electra-180g-small-discriminator')
    bert_output = encoder(input_ids, attention_mask, token_type_ids)[0]
    token_span = get_spans(indices=word_index)
    output = pick_tensor_for_each_token(h=bert_output, token_span=token_span, average_subwords=True) # 只有这样才是取平均，否则就是首字了

    print(output.shape)  # torch.Size([2, 8, 256])

    # ['我', '爱', '北京', '。'],
    assert (output[0][0] == bert_output[0][0]).all()  # [CLS]
    assert (output[0][1] == bert_output[0][1]).all()  # 我
    assert (output[0][2] == bert_output[0][2]).all()  # 爱
    assert (output[0][3] == torch.mean(bert_output[0][3:5], dim=0)).all()  # 北京
    assert (output[0][4] == bert_output[0][5]).all()  # 。
    assert (output[0][5] == bert_output[0][6]).all()  # [SEP]
    assert (output[0][6] == torch.zeros(output[0][6].shape)).all()  # padding
    assert (output[0][7] == torch.zeros(output[0][7].shape)).all()  # padding

    # ['我', '爱', '北京', '和', '你们', '。'],
    assert (output[1][0] == bert_output[1][0]).all()  # [CLS]
    assert (output[1][1] == bert_output[1][1]).all()  # 我
    assert (output[1][2] == bert_output[1][2]).all()  # 爱
    assert (output[1][3] == torch.mean(bert_output[1][3:5], dim=0)).all()  # 北京
    assert (output[1][4] == bert_output[1][5]).all()  # 和
    assert (output[1][5] == torch.mean(bert_output[1][6:8], dim=0)).all()  # 你们
    assert (output[1][6] == bert_output[1][8]).all()  # 。
    assert (output[1][7] == bert_output[1][9]).all()  # [SEP]

2、后续不需要[CLS]和[SEP]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
# -*- coding: utf8 -*-
#
from typing import List

import torch
from torch.nn.utils.rnn import pad_sequence
from transformers import AutoTokenizer, PreTrainedTokenizerBase, AutoModel


def tokenize(form: List[List[str]], tokenizer: PreTrainedTokenizerBase, max_length: int = 128, char_base: bool = False):
    """
    Args:
        form:
        tokenizer:
        max_length:
        char_base: 这里指的是form[即 word]是否是字级别的
    Returns:
    """
    res = tokenizer.batch_encode_plus(
        form,
        is_split_into_words=True,
        max_length=max_length,
        truncation=True,
    )
    result = res.data
    # 可用于长度大于指定长度过滤, overflow指字长度大于指定max_length，如果有cls,sep，那么就算上这个
    result['overflow'] = [len(encoding.overflowing) > 0 for encoding in res.encodings]
    if not char_base:
        word_index = []
        for encoding in res.encodings:
            word_index.append([])

            last_word_idx = -1
            current_length = 0
            for word_idx in encoding.word_ids[1:-1]:
                if word_idx != last_word_idx:
                    word_index[-1].append(current_length)

                current_length += 1
                last_word_idx = word_idx
        result['word_index'] = word_index
        result['word_attention_mask'] = [[True] * len(index) for index in word_index]

    # NOTE:区别点在这里，只有这里不同哦，就导致结果不同
    # # 加上CLS和SEP
    # for index, word_index in enumerate(result['word_index']):
    #     add_1 = [i + 1 for i in word_index]
    #     result['word_index'][index] = [0, *add_1, add_1[-1] + 1]

    return result


def pick_tensor_for_each_token(h, token_span, average_subwords):
    if token_span is None:
        return h
    if average_subwords and token_span.size(-1) > 1:
        batch_size = h.size(0)
        h_span = h.gather(1, token_span.view(batch_size, -1).unsqueeze(-1).expand(-1, -1, h.shape[-1]))
        h_span = h_span.view(batch_size, *token_span.shape[1:], -1)
        n_sub_tokens = token_span.ne(0)
        n_sub_tokens[:, 0, 0] = True
        h_span = (h_span * n_sub_tokens.unsqueeze(-1)).sum(2)
        n_sub_tokens = n_sub_tokens.sum(-1).unsqueeze(-1)
        zero_mask = n_sub_tokens == 0
        if torch.any(zero_mask):
            n_sub_tokens[zero_mask] = 1  # avoid dividing by zero
        embed = h_span / n_sub_tokens
    else:
        embed = h.gather(1, token_span[:, :, 0].unsqueeze(-1).expand(-1, -1, h.size(-1)))
    return embed


def get_span(indices):
    """

    :param indices:
    :return:
    """
    span = []
    for index, item in enumerate(indices):
        try:
            next_item = indices[index + 1]
            if next_item > item:  # padding 后面都是0
                s = torch.arange(item, next_item, dtype=torch.long)
            else:
                s = torch.tensor([item], dtype=torch.long)
            span.append(s)
        except IndexError:
            span.append(torch.tensor([item], dtype=torch.long))
    return pad_sequence(span).T


def get_spans(indices: List[List[int]]):
    l = None

    batch = []
    for index_item in indices:
        if l is None:
            l = len(index_item)
        else:
            if len(index_item) != l:
                raise ValueError('padding first.')
        batch.append(get_span(indices=index_item))
    span = torch.zeros(size=(len(batch), batch[0].size(0), max([i.size(1) for i in batch])), dtype=torch.long)
    for index, item in enumerate(batch):
        b, l = item.shape
        span[index][:b, :l] = item
    return span


if __name__ == '__main__':
    tokens = [
        ['我', '爱', '北京', '。'],
        ['我', '爱', '北京', '和', '你们', '。'],
    ]

    tokenizer = AutoTokenizer.from_pretrained('hfl/chinese-electra-180g-small-discriminator', use_fast=True)

    output = tokenize(form=tokens, tokenizer=tokenizer)
    input_ids = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['input_ids']],
                             padding_value=tokenizer.pad_token_id, batch_first=True)
    token_type_ids = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['token_type_ids']],
                                  padding_value=tokenizer.pad_token_id, batch_first=True)
    attention_mask = pad_sequence([torch.tensor(_, dtype=torch.bool) for _ in output['attention_mask']],
                                  padding_value=tokenizer.pad_token_id, batch_first=True)

    word_index = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['word_index']],
                              padding_value=tokenizer.pad_token_id, batch_first=True)
    word_attention_mask = pad_sequence([torch.tensor(_, dtype=torch.bool) for _ in output['word_attention_mask']],
                                       padding_value=tokenizer.pad_token_id, batch_first=True)

    # model
    encoder = AutoModel.from_pretrained('hfl/chinese-electra-180g-small-discriminator')
    bert_output = encoder(input_ids, attention_mask, token_type_ids)[0]
    token_span = get_spans(indices=word_index)
    output = pick_tensor_for_each_token(h=bert_output, token_span=token_span, average_subwords=True)

    print(output.shape)  # torch.Size([2, 6, 256])

    # ['我', '爱', '北京', '。'],
    assert (output[0][0] == bert_output[0][0]).all()  # 我
    assert (output[0][1] == bert_output[0][1]).all()  # 爱
    assert (output[0][2] == torch.mean(bert_output[0][2:4], dim=0)).all()  # 北京
    assert (output[0][3] == bert_output[0][4]).all()  # 。
    assert (output[0][4] == torch.zeros(output[0][4].shape)).all()  # padding
    assert (output[0][5] == torch.zeros(output[0][5].shape)).all()  # padding

    # ['我', '爱', '北京', '和', '你们', '。'],
    assert (output[1][0] == bert_output[1][0]).all()  # 我
    assert (output[1][1] == bert_output[1][1]).all()  # 爱
    assert (output[1][2] == torch.mean(bert_output[1][2:4], dim=0)).all()  # 北京
    assert (output[1][3] == bert_output[1][4]).all()  # 和
    assert (output[1][4] == torch.mean(bert_output[1][5:7], dim=0)).all()  # 你们
    assert (output[1][5] == bert_output[1][7]).all()  # 。

展开全文 >>

bert以首字表示词向量(2)

第二篇文章，通过一种新的方式来实现以首字表示词向量
也可以看这里：一些mask的操作理解 gather部分。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

# -*- coding: utf8 -*-
#
from typing import List
from unittest import TestCase

import torch
from torch.nn.utils.rnn import pad_sequence
from transformers import AutoTokenizer, AutoModel, PreTrainedTokenizerBase


def tokenize(form: List[List[str]], tokenizer: PreTrainedTokenizerBase, max_length: int, char_base: bool = False):
    """

    Args:
        form:
        tokenizer:
        max_length:
        char_base: 这里指的是form[即 word]是否是字级别的

    Returns:

    """
    res = tokenizer.batch_encode_plus(
        form,
        is_split_into_words=True,
        max_length=max_length,
        truncation=True,
    )
    result = res.data
    # 可用于长度大于指定长度过滤, overflow指字长度大于指定max_length，如果有cls,sep，那么就算上这个
    result['overflow'] = [len(encoding.overflowing) > 0 for encoding in res.encodings]
    if not char_base:
        word_index = []
        for encoding in res.encodings:
            word_index.append([])

            last_word_idx = -1
            current_length = 0
            for word_idx in encoding.word_ids[1:-1]:
                if word_idx != last_word_idx:
                    word_index[-1].append(current_length)

                current_length += 1
                last_word_idx = word_idx
        result['word_index'] = word_index
        result['word_attention_mask'] = [[True] * len(index) for index in word_index]
    return result


class TestSample(TestCase):
    def test_max_length(self):
        """
        测试max_length overflow情况
        :return:
        """
        pass

    def test_sample(self):
        form = [
            ['我', '呀'],
            ['我', '小明', '呀']
        ]

        tokenizer = AutoTokenizer.from_pretrained('hfl/chinese-electra-180g-small-discriminator')
        result = tokenize(form, tokenizer, 6)
        model = AutoModel.from_pretrained('hfl/chinese-electra-180g-small-discriminator')

        input_ids = pad_sequence([torch.tensor(input_ids) for input_ids in result['input_ids']], batch_first=True)
        token_type_ids = pad_sequence([torch.tensor(token_type_ids) for token_type_ids in result['token_type_ids']],
                                      batch_first=True)
        attention_mask = pad_sequence([torch.tensor(attention_mask) for attention_mask in result['attention_mask']],
                                      batch_first=True)

        # tensor([[ 101, 2769, 1435,  102,    0,    0],
        #         [ 101, 2769, 2207, 3209, 1435,  102]])

        # 1. 获取bert output.
        bert_out = model(input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)
        seq_out = bert_out[0]

        word_index = pad_sequence([torch.tensor(word_index) for word_index in result['word_index']], batch_first=True)

        # 2. 获取词首字向量，包括cls开头
        word_out = torch.cat([seq_out[:, :1, :], torch.gather(
            seq_out[:, 1:, :], dim=1, index=word_index.unsqueeze(-1).expand(-1, -1, seq_out.size(-1))
        )], dim=1)

        word_attention_mask = pad_sequence(
            [torch.tensor(word_attention_mask) for word_attention_mask in result['word_attention_mask']],
            batch_first=True)

        # 这里方便view
        # 1. ['我', '呀']
        self.assertTrue((seq_out[0][0] == word_out[0][0]).all())  # cls
        self.assertTrue((seq_out[0][1] == word_out[0][1]).all())  # 我
        self.assertTrue((seq_out[0][2] == word_out[0][2]).all())  # 呀

        self.assertTrue((word_out[0][1] == word_out[0][3]).all())  # 填充位

        # 2. ['我', '小明', '呀']
        self.assertTrue((seq_out[1][0] == word_out[1][0]).all())  # cls
        self.assertTrue((seq_out[1][1] == word_out[1][1]).all())  # 我
        self.assertTrue((seq_out[1][2] == word_out[1][2]).all())  # 小明
        self.assertTrue((seq_out[1][4] == word_out[1][3]).all())  # 呀

        # 3. Note: word_out的时候concat了seq_out[:, :1, :](cls)，所以word_out的长度比word_attention_mask大1
        self.assertEqual(word_out.size(1), word_attention_mask.size(1) + 1)

        # 4. 获取每个词对应的向量
        result = word_out[:, 1:, :][word_attention_mask]
        result2 = result.split(word_attention_mask.sum(1).tolist())
        self.assertEqual(len(result2[0]), 2)
        self.assertEqual(len(result2[1]), 3)


展开全文 >>

bert以平均、首字、尾字表示词向量(1)

1. word preprocess.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37


import torch
from transformers import AutoTokenizer


def encoder_texts(texts: List[List[str]], tokenizer):
    # 统计句子中最大的词长度
    fix_len = max([max([len(word) for word in text]) for text in texts])

    matrix = []
    for text in texts:
        vector = []

        text = [tokenizer.cls_token, *text, tokenizer.sep_token]
        input_ids = tokenizer.batch_encode_plus(
            text,
            add_special_tokens=False,
        )['input_ids']

        for _input_ids in input_ids:
            # 修复例如: texts = [['\ue5f1\ue5f1\ue5f1\ue5f1']] 这种情况
            _input_ids = _input_ids or [tokenizer.unk_token_id]
            vector.append(_input_ids + (fix_len - len(_input_ids)) * [tokenizer.pad_token_id])
        matrix.append(torch.tensor(vector, dtype=torch.long))
    return pad_sequence(matrix, batch_first=True)


if __name__ == '__main__':
    texts = [
        ['我', '爱中国'],
        ['我', '爱', '中国']
    ]
    tokenizer = AutoTokenizer.from_pretrained('ckiplab/albert-tiny-chinese')
    print(encoder_texts(texts, tokenizer))


output：

1
2
3
4
5
6
7
8
9
tensor([[[ 101,    0,    0],
         [2769,    0,    0],
         [4263,  704, 1744],
         [   0,    0,    0]],

        [[ 101,    0,    0],
         [2769,    0,    0],
         [4263,    0,    0],
         [ 704, 1744,    0]]])

2. transformer layer.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

# -*- coding: utf-8 -*-

import torch
import torch.nn as nn
from supar.modules.scalar_mix import ScalarMix
from supar.utils.fn import pad

# pad 等价于 pad_sequence。

# from torch.nn.utils.rnn import pad_sequence

class TransformerEmbedding(nn.Module):
    r"""
    A module that directly utilizes the pretrained models in `transformers`_ to produce BERT representations.
    While mainly tailored to provide input preparation and post-processing for the BERT model,
    it is also compatible with other pretrained language models like XLNet, RoBERTa and ELECTRA, etc.

    Args:
        model (str):
            Path or name of the pretrained models registered in `transformers`_, e.g., ``'bert-base-cased'``.
        n_layers (int):
            The number of BERT layers to use. If 0, uses all layers.
        n_out (int):
            The requested size of the embeddings. If 0, uses the size of the pretrained embedding model. Default: 0.
        stride (int):
            A sequence longer than max length will be splitted into several small pieces
            with a window size of ``stride``. Default: 10.
        pooling (str):
            Pooling way to get from token piece embeddings to token embedding.
            ``first``: take the first subtoken. ``last``: take the last subtoken. ``mean``: take a mean over all.
            Default: ``mean``.
        pad_index (int):
            The index of the padding token in BERT vocabulary. Default: 0.
        dropout (float):
            The dropout ratio of BERT layers. Default: 0. This value will be passed into the :class:`ScalarMix` layer.
        requires_grad (bool):
            If ``True``, the model parameters will be updated together with the downstream task. Default: ``False``.

    .. _transformers:
        https://github.com/huggingface/transformers
    """

    def __init__(self, model, n_layers=4, n_out=0, stride=256, pooling='mean', pad_index=0, dropout=0, requires_grad=True):
        super().__init__()

        from transformers import AutoConfig, AutoModel, AutoTokenizer
        self.bert = AutoModel.from_pretrained(model, config=AutoConfig.from_pretrained(model, output_hidden_states=True))
        self.bert = self.bert.requires_grad_(requires_grad)

        self.model = model
        self.n_layers = n_layers or self.bert.config.num_hidden_layers
        self.hidden_size = self.bert.config.hidden_size
        self.n_out = n_out or self.hidden_size
        self.stride = stride
        self.pooling = pooling
        self.pad_index = pad_index
        self.dropout = dropout
        self.requires_grad = requires_grad
        self.max_len = int(max(0, self.bert.config.max_position_embeddings) or 1e12) - 2

        self.tokenizer = AutoTokenizer.from_pretrained(model)

        self.scalar_mix = ScalarMix(self.n_layers, dropout)
        self.projection = nn.Linear(self.hidden_size, self.n_out, False) if self.hidden_size != n_out else nn.Identity()

    def __repr__(self):
        s = f"{self.model}, n_layers={self.n_layers}, n_out={self.n_out}, "
        s += f"stride={self.stride}, pooling={self.pooling}, pad_index={self.pad_index}"
        if self.dropout > 0:
            s += f", dropout={self.dropout}"
        if self.requires_grad:
            s += f", requires_grad={self.requires_grad}"

        return f"{self.__class__.__name__}({s})"

    def forward(self, subwords):
        r"""
        Args:
            subwords (~torch.Tensor): ``[batch_size, seq_len, fix_len]``.
        Returns:
            ~torch.Tensor:
                BERT embeddings of shape ``[batch_size, seq_len, n_out]``.
        """

        mask = subwords.ne(self.pad_index)
        lens = mask.sum((1, 2))
        # [batch_size, n_subwords]
        subwords = pad(subwords[mask].split(lens.tolist()), self.pad_index, padding_side=self.tokenizer.padding_side)
        bert_mask = pad(mask[mask].split(lens.tolist()), 0, padding_side=self.tokenizer.padding_side)

        # return the hidden states of all layers
        bert = self.bert(subwords[:, :self.max_len], attention_mask=bert_mask[:, :self.max_len].float())[-1]
        # [n_layers, batch_size, max_len, hidden_size]
        bert = bert[-self.n_layers:]
        # [batch_size, max_len, hidden_size]
        bert = self.scalar_mix(bert)
        # [batch_size, n_subwords, hidden_size]
        for i in range(self.stride, (subwords.shape[1]-self.max_len+self.stride-1)//self.stride*self.stride+1, self.stride):
            part = self.bert(subwords[:, i:i+self.max_len], attention_mask=bert_mask[:, i:i+self.max_len].float())[-1]
            bert = torch.cat((bert, self.scalar_mix(part[-self.n_layers:])[:, self.max_len-self.stride:]), 1)

        # [batch_size, seq_len]
        bert_lens = mask.sum(-1)
        bert_lens = bert_lens.masked_fill_(bert_lens.eq(0), 1)
        # [batch_size, seq_len, fix_len, hidden_size]
        embed = bert.new_zeros(*mask.shape, self.hidden_size).masked_scatter_(mask.unsqueeze(-1), bert[bert_mask])
        # [batch_size, seq_len, hidden_size]
        if self.pooling == 'first':
            embed = embed[:, :, 0]
        elif self.pooling == 'last':
            embed = embed.gather(2, (bert_lens-1).unsqueeze(-1).repeat(1, 1, self.hidden_size).unsqueeze(2)).squeeze(2)
        else:
            embed = embed.sum(2) / bert_lens.unsqueeze(-1)
        embed = self.projection(embed)

        return embed


到此，能够拿到以词为级别的output，embed.shape查看一下即可，后续就可以concat其他input做多输入。

展开全文 >>

bert网络结构

这里以BertForMaskedLM为例，记录下BertModel的网络结构和一些思考，cls那部分网络结构不涉及，即BertOnlyMLMHead那部分。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
BertForMaskedLM(
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(21128, 768, padding_idx=0)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
      (dropout): Dropout(p=0.1, inplace=False)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )

        # 这里重复11次，一共12层结构。
      )
    )
  )
  (cls): BertOnlyMLMHead(
    (predictions): BertLMPredictionHead(
      (transform): BertPredictionHeadTransform(
        (dense): Linear(in_features=768, out_features=768, bias=True)
        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
      )
      (decoder): Linear(in_features=768, out_features=21128, bias=True)
    )
  )
)

1. position_embedding和transformer里面的位置编码不一样

transformer里面的positional embedding，是正余弦绝对位置编码，对于sequence分配不同的position id。而bert是学习出来的，分配一个(512,768)embedding。

随后word_embedding，positional embedding, token_type_embedding进行相加，然后经过LN、dropout获得最终embedding。

2. BertIntermediate是用来干嘛的

在transformer结构里，是没有这个中间层的，细看下，是将768扩大4倍，随后在BertOutput那里又降维到768。那bert加这一个linear的作用是干嘛的呢？

搜了下网上的看法，很少关注或者没有相关思考。但是这这篇论文：Undivided Attention: Are Intermediate Layers Necessary for BERT? 给出了自己的解释。作者观点如下：

1
In recent times, BERT-based models have been extremely successful in solving a variety of natural language processing (NLP) tasks such as reading comprehension, natural language inference, sentiment analysis, etc. All BERT-based architectures have a self-attention block followed by a block of intermediate layers as the basic building component. However, a strong justification for the inclusion of these intermediate layers remains missing in the literature. In this work we investigate the importance of intermediate layers on the overall network performance of downstream tasks. We show that reducing the number of intermediate layers and modifying the architecture for BERT-Base results in minimal loss in fine-tuning accuracy for downstream tasks while decreasing the number of parameters and training time of the model. Additionally, we use the central kernel alignment (CKA) similarity metric and probing classifiers to demonstrate that removing intermediate layers has little impact on the learned self-attention representations.

结论就是实验后发现降低网络复杂度后，同时也能保持微调任务的准确性。

展开全文 >>

mlm数据处理实现解读

之前在看mask language model时，如何做数据处理那里一直没有太本质理解，比如15%做mask,然后又80%做mask,10%不变，10%随机选择，所以这里将mlm数据处理部分的代码列出来，方便需要者可以看到bert是怎么实现的。

另外关于mlm分词wordpience还是整词甚者ngram，这里不做探讨，本质来讲就是数据处理的方式不同，其他对于训练和上游使用预训练模型来讲并不影响。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
import collections
import random


MaskedLmInstance = collections.namedtuple("MaskedLmInstance", ["index", "label"])


def create_masked_lm_predictions(
        tokens,
        masked_lm_prob,
        max_predictions_per_seq,
        vocab_words,
        rng
):
    """
    Creates the predictions for the masked LM objective.
    :param tokens: 输入文本
    :param masked_lm_prob: 掩码语言模型的掩码概率
    :param max_predictions_per_seq: 每个序列的最大预测数目
    :param vocab_words: 词表列表
    :param rng: 随机数生成器
    """

    cand_indexes = []
    for (i, token) in enumerate(tokens):
        if token == "[CLS]" or token == "[SEP]":
            continue
        cand_indexes.append(i)

    rng.shuffle(cand_indexes)

    output_tokens = list(tokens)

    num_to_predict = min(max_predictions_per_seq, max(1, int(round(len(tokens) * masked_lm_prob))))

    masked_lms = []
    covered_indexes = set()
    for index in cand_indexes:
        if len(masked_lms) >= num_to_predict:
            break
        if index in covered_indexes:
            continue
        covered_indexes.add(index)

        masked_token = None
        # 80% of the time, replace with [MASK]
        if rng.random() < 0.8:
            masked_token = "[MASK]"
        else:
            # 10% of the time, keep original
            if rng.random() < 0.5:
                masked_token = tokens[index]
            # 10% of the time, replace with random word
            else:
                masked_token = vocab_words[rng.randint(0, len(vocab_words) - 1)]

        output_tokens[index] = masked_token

        masked_lms.append(MaskedLmInstance(index=index, label=tokens[index]))

    masked_lms = sorted(masked_lms, key=lambda x: x.index)

    masked_lm_positions = []
    masked_lm_labels = []
    for p in masked_lms:
        masked_lm_positions.append(p.index)
        masked_lm_labels.append(p.label)

    return output_tokens, masked_lm_positions, masked_lm_labels


if __name__ == '__main__':
    test_input = ['i', 'love', 'you', 'do', 'you', 'like', 'me']
    test_vocab = ['i', 'love', 'you', 'do', 'like', 'me']

    out1, out2, out3 = create_masked_lm_predictions(
        tokens=test_input,
        masked_lm_prob=0.5,
        max_predictions_per_seq=512,
        vocab_words=test_vocab,
        rng=random.Random()

    )

展开全文 >>

研究torch0.4.1 rnn的实现

公式

说明

  1. 所有测试代码以此为准。
  2. 以torch0.4.1的代码为准进行debug rnn代码， 可以pip install 指定版本的torch.

从测试代码可以看到：

  1. num_layers相当初始化多少层rnn layer，比如此例子的10，
  2. input_size表示上一层的hidden_size(例如本例子的embedding)
  3. hidden_size表示rnn的输出hidden_size
  4. batch_first简单来讲就是内存存储的问题，if batch_first=True，那么维度(batch_size, seq_length, hidden_size)，否则seq_length和batch_size对调，另外cudnn底层就是seq_length放在第一位，torch底层也会根据batch_first进行转换

1. 初始化一些参数

以embedding出来为准，输出维度为: (32, 3, 30)
rnn第一步初始化hx(10, 32, 4)，一共初始化了num_layer=10层，batch_size=32，hidden_size=4，白话文就是每一层rnn都有一个(32, 4)的矩阵来保存最后一个时刻(ht)的结果。

这个可以看class RNNBase(Module):。
另外还有2个weight，2个bias的初始化。即公式中的w和b

2. AutogradRNN

初始化参数后，调用此方法，进行rnn计算，如果支持GPU，就会使用cudnn提供好的，此处忽略这里。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
def AutogradRNN(mode, input_size, hidden_size, num_layers=1, batch_first=False,
                dropout=0, train=True, bidirectional=False, variable_length=False,
                dropout_state=None, flat_weight=None):
    # 使用不同的cell进行计算，每一层rnn layer即一个cell。
    if mode == 'RNN_RELU':
        cell = RNNReLUCell
    elif mode == 'RNN_TANH':
        cell = RNNTanhCell
    elif mode == 'LSTM':
        cell = LSTMCell
    elif mode == 'GRU':
        cell = GRUCell
    else:
        raise Exception('Unknown mode: {}'.format(mode))
    # rec_factory即表示Recurrent，就是第四步的函数
    rec_factory = variable_recurrent_factory if variable_length else Recurrent

    if bidirectional:
        # 可以看到哦，反向就是进行reverse一下
        # 对应代码：steps = range(input.size(0) - 1, -1, -1) if reverse else range(input.size(0))
        # 意思就是从最后一个时刻反向进行更新hidden_size（不同时刻的y）。
        layer = (rec_factory(cell), rec_factory(cell, reverse=True))
    else:
        layer = (rec_factory(cell),)
    # 这地方疯狂使用闭包...
    func = StackedRNN(layer,
                      num_layers,
                      (mode == 'LSTM'),
                      dropout=dropout,
                      train=train)

    def forward(input, weight, hidden, batch_sizes):
        if batch_first and not variable_length:
            # 囔，这里就会batch_size 和  sequence_length对调
            input = input.transpose(0, 1)

        nexth, output = func(input, hidden, weight, batch_sizes)

        if batch_first and not variable_length:
            output = output.transpose(0, 1)

        return output, nexth

    return forward

3. StackedRNN

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

def StackedRNN(inners, num_layers, lstm=False, dropout=0, train=True):
    
    """
     拿到每一层Recurrent layer计算出来的结果，进行concat到一起进行返回
     
     inner表示 Recurrent，代码如下

    """

    num_directions = len(inners)
    total_layers = num_layers * num_directions

    def forward(input, hidden, weight, batch_sizes):
        """
        input: 3, 32, 30 (sequence_length, batch_size, embedding input hidden_size),
        hidden: 10, 32, 4 (num_layers, batch_size, output hidden_size),
        """
        assert(len(weight) == total_layers)
        next_hidden = []

        if lstm:
            hidden = list(zip(*hidden))
        # 遍历rnn中num_layers
        for i in range(num_layers):
            all_output = []
            for j, inner in enumerate(inners): # inner表示 Recurrent，代码如下
                l = i * num_directions + j

                hy, output = inner(input, hidden[l], weight[l], batch_sizes)
                next_hidden.append(hy)
                all_output.append(output)

            input = torch.cat(all_output, input.dim() - 1)

            if dropout != 0 and i < num_layers - 1:
                input = F.dropout(input, p=dropout, training=train, inplace=False)

        if lstm:
            next_h, next_c = zip(*next_hidden)
            next_hidden = (
                torch.cat(next_h, 0).view(total_layers, *next_h[0].size()),
                torch.cat(next_c, 0).view(total_layers, *next_c[0].size())
            )
        else:
            next_hidden = torch.cat(next_hidden, 0).view(
                total_layers, *next_hidden[0].size())

        return next_hidden, input

    return forward

4. Recurrent

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
def Recurrent(inner, reverse=False):
    """
    表示单层layer的计算
    """
    def forward(input, hidden, weight, batch_sizes):
        """
        input: 3, 32, 30
        hidden: 32, 4 (表示遍历每一个单独的rnn进来，一共10次)
        weight:List[  (4, 30), (4,4), (4, ), (4, ) ]
        """
        output = []
        steps = range(input.size(0) - 1, -1, -1) if reverse else range(input.size(0))
        for i in steps: # steps为3,因为sequence_length为3
            hidden = inner(input[i], hidden, *weight) # hidden就代表上一个时刻的输出， inner为RNNTanhCell，代码如下
            # hack to handle LSTM
            output.append(hidden[0] if isinstance(hidden, tuple) else hidden)

        if reverse:
            output.reverse()
        output = torch.cat(output, 0).view(input.size(0), *output[0].size())
        # 此处hidden就代表最后一个时刻的输出(32, 4)，output表示每一个时刻的输出(3, 32, 4)
        # torch.any(output[-1,:,:] == hidden)
        return hidden, output

    return forward

5. Recurrent中的inner：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# 针对rnn
def RNNTanhCell(input, hidden, w_ih, w_hh, b_ih=None, b_hh=None):
    """
    input: 32, 30
    hidden: 32, 4
    w_ih: 4,30
    w_hh: 4,4
    b_ih: 4
    b_hh: 4
    """
    hy = torch.tanh(F.linear(input, w_ih, b_ih) + F.linear(hidden, w_hh, b_hh))
    return hy

# 针对lstm cell.
def LSTMCell(input, hidden, w_ih, w_hh, b_ih=None, b_hh=None):
    """
    input: (32, 30)
    hidden: ( (32, 4), (32, 4) )
    w_ih: (16, 30)
    w_hh: (16, 4)
    b_ih: (16, )
    b_hh: (16, )
    """
    if input.is_cuda: # 自行忽略此处代码
        igates = F.linear(input, w_ih)
        hgates = F.linear(hidden[0], w_hh)
        state = fusedBackend.LSTMFused.apply
        return state(igates, hgates, hidden[1]) if b_ih is None else state(igates, hgates, hidden[1], b_ih, b_hh)

    hx, cx = hidden
    # 看到么，也是rnn公式哦
    gates = F.linear(input, w_ih, b_ih) + F.linear(hx, w_hh, b_hh)
    # 但是此处不一样了，在第一维分割4份出来
    ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)
    # 看到下面这几行公式么，为啥子就能解决梯度消失呢？
    # 神不神奇，意不意外
    ingate = torch.sigmoid(ingate)
    forgetgate = torch.sigmoid(forgetgate)
    cellgate = torch.tanh(cellgate)
    outgate = torch.sigmoid(outgate)

    cy = (forgetgate * cx) + (ingate * cellgate)
    hy = outgate * torch.tanh(cy)
    # hy: (32,4), (32,4)
    return hy, cy

# 针对gru
def GRUCell(input, hidden, w_ih, w_hh, b_ih=None, b_hh=None):

    if input.is_cuda:
        gi = F.linear(input, w_ih)
        gh = F.linear(hidden, w_hh)
        state = fusedBackend.GRUFused.apply
        return state(gi, gh, hidden) if b_ih is None else state(gi, gh, hidden, b_ih, b_hh)

    # 这里就变了哦
    gi = F.linear(input, w_ih, b_ih)
    gh = F.linear(hidden, w_hh, b_hh)

    i_r, i_i, i_n = gi.chunk(3, 1)
    h_r, h_i, h_n = gh.chunk(3, 1)

    resetgate = torch.sigmoid(i_r + h_r)
    inputgate = torch.sigmoid(i_i + h_i)
    newgate = torch.tanh(i_n + resetgate * h_n)
    hy = newgate + inputgate * (hidden - newgate)

    return hy

6. 测试代码完整如下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
# -*- coding: utf8 -*-
#
from torch.nn import Embedding, Sigmoid, Linear
from torch.nn.modules.rnn import RNN

import torch
from torch.nn import BCELoss
from torch.optim import RMSprop


class RNNModel(torch.nn.Module):
    def __init__(self):
        super(RNNModel, self).__init__()
        self.embed = Embedding(6, 30)
        self.rnn = RNN(input_size=30, hidden_size=4, batch_first=True, num_layers=10)
        self.linear = Linear(4, 1)

    def forward(self, input):
        input = self.embed(input)
        # rnn_out: (32, 3, 4) 返回batch_size下每个时刻(sequence_length)的最终输出
        # _ : (10, 32, 4) 表示batch_size下每个rnn layer最后一个时刻的输出
        rnn_out, _ = self.rnn(input)
        y_pred = self.linear(_.sum(0))
        return y_pred


class RNNDemo(object):
    def __init__(self):
        self.model = RNNModel()

        self.optim = RMSprop(self.model.parameters())

    def loss(self, y_pred, y_true):
        y_pred = Sigmoid()(y_pred)
        return BCELoss()(y_pred, y_true.float())

    def train(self, x, y):
        for i in range(100):
            for _x, _y in zip(x, y):
                _x = _x.repeat(32, 1)
                _y = _y.repeat(32, 1)
                y_pred = self.model(_x)
                loss = self.loss(y_pred, _y)
                loss.backward()
                print(loss.item())
                self.optim.step()
                self.optim.zero_grad()


if __name__ == '__main__':
    train_data = [
        (['A', 'B', 'C'], 'right'),
        (['C', 'D', 'E'], 'false')
    ]
    id_map = {
        'A': 1,
        'B': 2,
        'C': 3,
        'D': 4,
        'E': 5,
        'right': 6,
        'false': 7

    }
    rnn = RNNDemo()
    rnn.train(
        [torch.tensor([id_map[i] for i in sample[0]]).reshape(1, 3) for sample in train_data],
        [torch.tensor(id_map[sample[1]]).reshape(1, 1) for sample in train_data]
    )


至此，前向过程完结。

展开全文 >>