※ 引述《wu10200512 (廷廷)》之铭言：
: 老师要我用纯pytorch写transformer
: 我写不出来
: 有没有人会写可以救我
: 拜托 我求你了
当你想要使用PyTorch来实现Transformer模型时，下面是一个基本的范例，演示如何使用
PyTorch建构Transformer模型。
首先，你需要汇入所需的PyTorch模组：
import torch
import torch.nn as nn
import torch.nn.functional as F
接下来，定义一个Transformer模型的类别：
class Transformer(nn.Module):
def __init__(self, input_dim, hidden_dim, num_heads, num_layers):
super(Transformer, self).__init__()
self.embedding = nn.Embedding(input_dim, hidden_dim)
self.positional_encoding = PositionalEncoding(hidden_dim)
self.encoder = Encoder(hidden_dim, num_heads, num_layers)
def forward(self, x):
embedded = self.embedding(x)
encoded = self.positional_encoding(embedded)
output = self.encoder(encoded)
return output
在这个范例中，我们使用了三个子模组：Embedding、PositionalEncoding和Encoder。这
些子模组可以透过定义相应的类别来实现。
class PositionalEncoding(nn.Module):
def __init__(self, hidden_dim, max_len=1000):
super(PositionalEncoding, self).__init__()
self.hidden_dim = hidden_dim
position = torch.arange(0, max_len).unsqueeze(1)
div_term = torch.exp(torch.arange(0, hidden_dim, 2) *
-(math.log(10000.0) / hidden_dim))
pe = torch.zeros(max_len, hidden_dim)
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
pe = pe.unsqueeze(0).transpose(0, 1)
self.register_buffer('pe', pe)
def forward(self, x):
x = x * math.sqrt(self.hidden_dim)
x = x + self.pe[:x.size(0), :]
return x
class Encoder(nn.Module):
def __init__(self, hidden_dim, num_heads, num_layers):
super(Encoder, self).__init__()
self.layers = nn.ModuleList([EncoderLayer(hidden_dim, num_heads) for
_ in range(num_layers)])
self.norm = nn.LayerNorm(hidden_dim)
def forward(self, x):
for layer in self.layers:
x = layer(x)
x = self.norm(x)
return x
class EncoderLayer(nn.Module):
def __init__(self, hidden_dim, num_heads, dropout=0.1):
super(EncoderLayer, self).__init__()
self.self_attention = MultiHeadAttention(hidden_dim, num_heads)
self.feed_forward = FeedForward(hidden_dim)
self.norm1 = nn.LayerNorm(hidden_dim)
self.norm2 = nn.LayerNorm(hidden_dim)
self.dropout = nn.Dropout(dropout)
def forward(self, x):
residual = x
x = self.norm1(x + self.dropout(self.self_attention(x)))
x = self.norm2(x + self.dropout(self.feed_forward(x)))
return x
class MultiHeadAttention(nn.Module):
def __init__(self, hidden_dim, num_heads):
super(MultiHeadAttention, self).__init__()
self.hidden_dim = hidden_dim
self.num_heads = num_heads
self.head_dim = hidden_dim // num_heads
self.fc_query = nn.Linear(hidden_dim, hidden_dim)
self.fc_key = nn.Linear(hidden_dim, hidden_dim)
self.fc_value = nn.Linear(hidden_dim, hidden_dim)
self.fc_out = nn.Linear(hidden_dim, hidden_dim)
def forward(self, x):
batch_size = x.size(0)
query = self.fc_query(x)
key = self.fc_key(x)
value = self.fc_value(x)
query = query.view(batch_size, -1, self.num_heads,
self.head_dim).transpose(1, 2)
key = key.view(batch_size, -1, self.num_heads,
self.head_dim).transpose(1, 2)
value = value.view(batch_size, -1, self.num_heads,
self.head_dim).transpose(1, 2)
scores = torch.matmul(query, key.transpose(-2, -1)) /
math.sqrt(self.head_dim)
scores = F.softmax(scores, dim=-1)
attended = torch.matmul(scores, value)
attended = attended.transpose(1, 2).contiguous().view(batch_size, -1,
self.hidden_dim)
output = self.fc_out(attended)
return output
class FeedForward(nn.Module):
def __init__(self, hidden_dim, dropout=0.1):
super(FeedForward, self).__init__()
self.fc1 = nn.Linear(hidden_dim, hidden_dim * 4)
self.fc2 = nn.Linear(hidden_dim * 4, hidden_dim)
self.dropout = nn.Dropout(dropout)
def forward(self, x):
x = F.relu(self.fc1(x))
x = self.dropout(x)
x = self.fc2(x)
return x
这就是一个简单的Transformer模型的实现范例。你可以根据自己的需求进一步扩展和调
整模型的结构和参数。
请注意，以上范例仅实现了Transformer模型的一个简化版本，其中只包含了Encoder部分
。如果你需要实现完整的Transformer模型（包括Decoder部分），你需要进一步扩展这个
范例。
此外，还需要将你的输入数据传递给这个Transformer模型，以获得预测结果。这可以通
过创建一个Transformer对象并呼叫其forward方法来实现。例如：
model = Transformer(input_dim, hidden_dim, num_heads, num_layers)
input_data = torch.tensor([...]) # 根据你的输入数据进行调整
output = model(input_data)
这样你就可以使用PyTorch来建构和训练Transformer模型了。请注意，这个范例仅提供了
一个基本的实现，你可能需要根据自己的任务和需求进行进一步的修改和扩展。
恩嗯嗯