bert取平均向量方式2

这个方式来自HanLP，具体就不细讲了，大家看代码即可。

1、如果后续处理需要[CLS]和[SEP]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159

# -*- coding: utf8 -*-
#
from typing import List

import torch
from torch.nn.utils.rnn import pad_sequence
from transformers import AutoTokenizer, PreTrainedTokenizerBase, AutoModel


def tokenize(form: List[List[str]], tokenizer: PreTrainedTokenizerBase, max_length: int = 128, char_base: bool = False):
    """
    Args:
        form:
        tokenizer:
        max_length:
        char_base: 这里指的是form[即 word]是否是字级别的
    Returns:
    """
    res = tokenizer.batch_encode_plus(
        form,
        is_split_into_words=True,
        max_length=max_length,
        truncation=True,
    )
    result = res.data
    # 可用于长度大于指定长度过滤, overflow指字长度大于指定max_length，如果有cls,sep，那么就算上这个
    result['overflow'] = [len(encoding.overflowing) > 0 for encoding in res.encodings]
    if not char_base:
        word_index = []
        for encoding in res.encodings:
            word_index.append([])

            last_word_idx = -1
            current_length = 0
            for word_idx in encoding.word_ids[1:-1]:
                if word_idx != last_word_idx:
                    word_index[-1].append(current_length)

                current_length += 1
                last_word_idx = word_idx
        result['word_index'] = word_index
        result['word_attention_mask'] = [[True] * len(index) for index in word_index]

    # 加上CLS和SEP
    for index, word_index in enumerate(result['word_index']):
        add_1 = [i + 1 for i in word_index]
        result['word_index'][index] = [0, *add_1, add_1[-1] + 1]

    return result


def pick_tensor_for_each_token(h, token_span, average_subwords):
    if token_span is None:
        return h
    if average_subwords and token_span.size(-1) > 1:
        batch_size = h.size(0)
        h_span = h.gather(1, token_span.view(batch_size, -1).unsqueeze(-1).expand(-1, -1, h.shape[-1]))
        h_span = h_span.view(batch_size, *token_span.shape[1:], -1)
        n_sub_tokens = token_span.ne(0)
        n_sub_tokens[:, 0, 0] = True
        h_span = (h_span * n_sub_tokens.unsqueeze(-1)).sum(2)
        n_sub_tokens = n_sub_tokens.sum(-1).unsqueeze(-1)
        zero_mask = n_sub_tokens == 0
        if torch.any(zero_mask):
            n_sub_tokens[zero_mask] = 1  # avoid dividing by zero
        embed = h_span / n_sub_tokens
    else:
        embed = h.gather(1, token_span[:, :, 0].unsqueeze(-1).expand(-1, -1, h.size(-1)))
    return embed


def get_span(indices):
    """

    :param indices:
    :return:
    """
    span = []
    for index, item in enumerate(indices):
        try:
            next_item = indices[index + 1]
            if next_item > item:  # padding 后面都是0
                s = torch.arange(item, next_item, dtype=torch.long)
            else:
                s = torch.tensor([item], dtype=torch.long)
            span.append(s)
        except IndexError:
            span.append(torch.tensor([item], dtype=torch.long))
    return pad_sequence(span).T


def get_spans(indices: List[List[int]]):
    l = None

    batch = []
    for index_item in indices:
        if l is None:
            l = len(index_item)
        else:
            if len(index_item) != l:
                raise ValueError('padding first.')
        batch.append(get_span(indices=index_item))
    span = torch.zeros(size=(len(batch), batch[0].size(0), max([i.size(1) for i in batch])), dtype=torch.long)
    for index, item in enumerate(batch):
        b, l = item.shape
        span[index][:b, :l] = item
    return span


if __name__ == '__main__':
    tokens = [
        ['我', '爱', '北京', '。'],
        ['我', '爱', '北京', '和', '你们', '。'],
    ]

    tokenizer = AutoTokenizer.from_pretrained('hfl/chinese-electra-180g-small-discriminator', use_fast=True)

    output = tokenize(form=tokens, tokenizer=tokenizer)
    input_ids = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['input_ids']],
                             padding_value=tokenizer.pad_token_id, batch_first=True)
    token_type_ids = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['token_type_ids']],
                                  padding_value=tokenizer.pad_token_id, batch_first=True)
    attention_mask = pad_sequence([torch.tensor(_, dtype=torch.bool) for _ in output['attention_mask']],
                                  padding_value=tokenizer.pad_token_id, batch_first=True)

    word_index = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['word_index']],
                              padding_value=tokenizer.pad_token_id, batch_first=True)
    word_attention_mask = pad_sequence([torch.tensor(_, dtype=torch.bool) for _ in output['word_attention_mask']],
                                       padding_value=tokenizer.pad_token_id, batch_first=True)

    # model
    encoder = AutoModel.from_pretrained('hfl/chinese-electra-180g-small-discriminator')
    bert_output = encoder(input_ids, attention_mask, token_type_ids)[0]
    token_span = get_spans(indices=word_index)
    output = pick_tensor_for_each_token(h=bert_output, token_span=token_span, average_subwords=True) # 只有这样才是取平均，否则就是首字了

    print(output.shape)  # torch.Size([2, 8, 256])

    # ['我', '爱', '北京', '。'],
    assert (output[0][0] == bert_output[0][0]).all()  # [CLS]
    assert (output[0][1] == bert_output[0][1]).all()  # 我
    assert (output[0][2] == bert_output[0][2]).all()  # 爱
    assert (output[0][3] == torch.mean(bert_output[0][3:5], dim=0)).all()  # 北京
    assert (output[0][4] == bert_output[0][5]).all()  # 。
    assert (output[0][5] == bert_output[0][6]).all()  # [SEP]
    assert (output[0][6] == torch.zeros(output[0][6].shape)).all()  # padding
    assert (output[0][7] == torch.zeros(output[0][7].shape)).all()  # padding

    # ['我', '爱', '北京', '和', '你们', '。'],
    assert (output[1][0] == bert_output[1][0]).all()  # [CLS]
    assert (output[1][1] == bert_output[1][1]).all()  # 我
    assert (output[1][2] == bert_output[1][2]).all()  # 爱
    assert (output[1][3] == torch.mean(bert_output[1][3:5], dim=0)).all()  # 北京
    assert (output[1][4] == bert_output[1][5]).all()  # 和
    assert (output[1][5] == torch.mean(bert_output[1][6:8], dim=0)).all()  # 你们
    assert (output[1][6] == bert_output[1][8]).all()  # 。
    assert (output[1][7] == bert_output[1][9]).all()  # [SEP]

2、后续不需要[CLS]和[SEP]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
# -*- coding: utf8 -*-
#
from typing import List

import torch
from torch.nn.utils.rnn import pad_sequence
from transformers import AutoTokenizer, PreTrainedTokenizerBase, AutoModel


def tokenize(form: List[List[str]], tokenizer: PreTrainedTokenizerBase, max_length: int = 128, char_base: bool = False):
    """
    Args:
        form:
        tokenizer:
        max_length:
        char_base: 这里指的是form[即 word]是否是字级别的
    Returns:
    """
    res = tokenizer.batch_encode_plus(
        form,
        is_split_into_words=True,
        max_length=max_length,
        truncation=True,
    )
    result = res.data
    # 可用于长度大于指定长度过滤, overflow指字长度大于指定max_length，如果有cls,sep，那么就算上这个
    result['overflow'] = [len(encoding.overflowing) > 0 for encoding in res.encodings]
    if not char_base:
        word_index = []
        for encoding in res.encodings:
            word_index.append([])

            last_word_idx = -1
            current_length = 0
            for word_idx in encoding.word_ids[1:-1]:
                if word_idx != last_word_idx:
                    word_index[-1].append(current_length)

                current_length += 1
                last_word_idx = word_idx
        result['word_index'] = word_index
        result['word_attention_mask'] = [[True] * len(index) for index in word_index]

    # NOTE:区别点在这里，只有这里不同哦，就导致结果不同
    # # 加上CLS和SEP
    # for index, word_index in enumerate(result['word_index']):
    #     add_1 = [i + 1 for i in word_index]
    #     result['word_index'][index] = [0, *add_1, add_1[-1] + 1]

    return result


def pick_tensor_for_each_token(h, token_span, average_subwords):
    if token_span is None:
        return h
    if average_subwords and token_span.size(-1) > 1:
        batch_size = h.size(0)
        h_span = h.gather(1, token_span.view(batch_size, -1).unsqueeze(-1).expand(-1, -1, h.shape[-1]))
        h_span = h_span.view(batch_size, *token_span.shape[1:], -1)
        n_sub_tokens = token_span.ne(0)
        n_sub_tokens[:, 0, 0] = True
        h_span = (h_span * n_sub_tokens.unsqueeze(-1)).sum(2)
        n_sub_tokens = n_sub_tokens.sum(-1).unsqueeze(-1)
        zero_mask = n_sub_tokens == 0
        if torch.any(zero_mask):
            n_sub_tokens[zero_mask] = 1  # avoid dividing by zero
        embed = h_span / n_sub_tokens
    else:
        embed = h.gather(1, token_span[:, :, 0].unsqueeze(-1).expand(-1, -1, h.size(-1)))
    return embed


def get_span(indices):
    """

    :param indices:
    :return:
    """
    span = []
    for index, item in enumerate(indices):
        try:
            next_item = indices[index + 1]
            if next_item > item:  # padding 后面都是0
                s = torch.arange(item, next_item, dtype=torch.long)
            else:
                s = torch.tensor([item], dtype=torch.long)
            span.append(s)
        except IndexError:
            span.append(torch.tensor([item], dtype=torch.long))
    return pad_sequence(span).T


def get_spans(indices: List[List[int]]):
    l = None

    batch = []
    for index_item in indices:
        if l is None:
            l = len(index_item)
        else:
            if len(index_item) != l:
                raise ValueError('padding first.')
        batch.append(get_span(indices=index_item))
    span = torch.zeros(size=(len(batch), batch[0].size(0), max([i.size(1) for i in batch])), dtype=torch.long)
    for index, item in enumerate(batch):
        b, l = item.shape
        span[index][:b, :l] = item
    return span


if __name__ == '__main__':
    tokens = [
        ['我', '爱', '北京', '。'],
        ['我', '爱', '北京', '和', '你们', '。'],
    ]

    tokenizer = AutoTokenizer.from_pretrained('hfl/chinese-electra-180g-small-discriminator', use_fast=True)

    output = tokenize(form=tokens, tokenizer=tokenizer)
    input_ids = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['input_ids']],
                             padding_value=tokenizer.pad_token_id, batch_first=True)
    token_type_ids = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['token_type_ids']],
                                  padding_value=tokenizer.pad_token_id, batch_first=True)
    attention_mask = pad_sequence([torch.tensor(_, dtype=torch.bool) for _ in output['attention_mask']],
                                  padding_value=tokenizer.pad_token_id, batch_first=True)

    word_index = pad_sequence([torch.tensor(_, dtype=torch.long) for _ in output['word_index']],
                              padding_value=tokenizer.pad_token_id, batch_first=True)
    word_attention_mask = pad_sequence([torch.tensor(_, dtype=torch.bool) for _ in output['word_attention_mask']],
                                       padding_value=tokenizer.pad_token_id, batch_first=True)

    # model
    encoder = AutoModel.from_pretrained('hfl/chinese-electra-180g-small-discriminator')
    bert_output = encoder(input_ids, attention_mask, token_type_ids)[0]
    token_span = get_spans(indices=word_index)
    output = pick_tensor_for_each_token(h=bert_output, token_span=token_span, average_subwords=True)

    print(output.shape)  # torch.Size([2, 6, 256])

    # ['我', '爱', '北京', '。'],
    assert (output[0][0] == bert_output[0][0]).all()  # 我
    assert (output[0][1] == bert_output[0][1]).all()  # 爱
    assert (output[0][2] == torch.mean(bert_output[0][2:4], dim=0)).all()  # 北京
    assert (output[0][3] == bert_output[0][4]).all()  # 。
    assert (output[0][4] == torch.zeros(output[0][4].shape)).all()  # padding
    assert (output[0][5] == torch.zeros(output[0][5].shape)).all()  # padding

    # ['我', '爱', '北京', '和', '你们', '。'],
    assert (output[1][0] == bert_output[1][0]).all()  # 我
    assert (output[1][1] == bert_output[1][1]).all()  # 爱
    assert (output[1][2] == torch.mean(bert_output[1][2:4], dim=0)).all()  # 北京
    assert (output[1][3] == bert_output[1][4]).all()  # 和
    assert (output[1][4] == torch.mean(bert_output[1][5:7], dim=0)).all()  # 你们
    assert (output[1][5] == bert_output[1][7]).all()  # 。

赏

谢谢你请我吃糖果

支付宝

扫一扫，分享到微信

微信分享二维码

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

探索世界美好的存在。