s计算score的方式有以下几种import torch class Attn(nn.module): def __init__(self, method ,hidden_size): super(Attn, self).__init__() self.method method self.hidden_size hidden_size if self.method not in [dot, general, concat]: raise ValueError(self.method, is not an appropriate attention method.) self.hidden_size hidden_size if self.method general: self.attn nn.Linear(self.hidden_size, hidden_size) elif self.method concat: self.attn nn.Linear(self.hidden_size * 2, hidden_size) self.v nn.Parameter(torch.FloatTensor(hidden_size)) def dot_score(self, hidden, encoder_outputs): return torch.sum(hidden*encoder_outputs, dim2) def general_score(self, hidden, encoder_outputs): return torch.sum(hidden*self.attn(encoder_outputs), dim2) def concat_score(self, hidden, encoder_outputs): energy self.attn(torc.cat(hidden.expand(encoder_outputs.size(0),-1,-1), encoder_outputs)).tanh() return torch.sum(self.v * energy, dim2) def forward(self, hidden, encoder_outputs): if self.method general: attn_energies self.general_score(hidden, encoder_outputs) elif self.method concat: attn_energies self.concat_score(hidden, encoder_outputs) elif self.method dot: attn_energies self.dot_score(hidden, encoder_outputs) # Transpose max_length and batch_size dimensions attn_energies attn_energies.t() # Return the softmax normalized probability scores (with added dimension) return F.softmax(attn_energies, dim1).unsqueeze(1) #在LuongAttnDecoderRNN中使用attn: rnn_output, hidden self.gru(embedded, last_hidden) attn_weights self.attn(rnn_output, encoder_outputs) context attn_weights.bmm(encoder_outputs.transpose(0, 1)) #[batch, 1, hidden] context context.squeeze(1)#去除size为1的维度 #[batch, hidden] concat_input torch.cat((rnn_output, context), 1) concat_output torch.tanh(self.concat(concat_input)) output self.out(concat_output) output F.softmax(output, dim1)bert中的attn借鉴别人博客明白的class BertSelfAttention(nn.Module): def __init__(self, config): super(BertSelfAttention, self).__init__() if config.hidden_size % config.num_attention_heads ! 0: raise ValueError( The hidden size (%d) is not a multiple of the number of attention heads (%d) % (config.hidden_size, config.num_attention_heads)) self.output_attentions config.output_attentions self.num_attention_heads config.num_attention_heads self.attention_head_size int(config.hidden_size / config.num_attention_heads) self.all_head_size self.num_attention_heads * self.attention_head_size self.query nn.Linear(config.hidden_size, self.all_head_size) self.key nn.Linear(config.hidden_size, self.all_head_size) self.value nn.Linear(config.hidden_size, self.all_head_size) self.dropout nn.Dropout(config.attention_probs_dropout_prob) def transpose_for_scores(self, x): new_x_shape x.size()[:-1] (self.num_attention_heads, self.attention_head_size) x x.view(*new_x_shape) return x.permute(0, 2, 1, 3) def forward(self, hidden_states, attention_mask, head_maskNone): mixed_query_layer self.query(hidden_states) mixed_key_layer self.key(hidden_states) mixed_value_layer self.value(hidden_states) query_layer self.transpose_for_scores(mixed_query_layer) key_layer self.transpose_for_scores(mixed_key_layer) value_layer self.transpose_for_scores(mixed_value_layer) # Take the dot product between query and key to get the raw attention scores. attention_scores torch.matmul(query_layer, key_layer.transpose(-1, -2)) attention_scores attention_scores / math.sqrt(self.attention_head_size) # Apply the attention mask is (precomputed for all layers in BertModel forward() function) attention_scores attention_scores attention_mask # Normalize the attention scores to probabilities. attention_probs nn.Softmax(dim-1)(attention_scores) # This is actually dropping out entire tokens to attend to, which might # seem a bit unusual, but is taken from the original Transformer paper. attention_probs self.dropout(attention_probs) # Mask heads if we want to if head_mask is not None: attention_probs attention_probs * head_mask context_layer torch.matmul(attention_probs, value_layer) context_layer context_layer.permute(0, 2, 1, 3).contiguous() new_context_layer_shape context_layer.size()[:-2] (self.all_head_size,) context_layer context_layer.view(*new_context_layer_shape) outputs (context_layer, attention_probs) if self.output_attentions else (context_layer,) return outputs