Hire Expert PyTorch
Developers from India

import torch import torch.nn as nn class TransformerBlock(nn.Module): def __init__(self, d_model=512, n_heads=8, dropout=0.1): super().__init__() self.attn = nn.MultiheadAttention(d_model, n_heads, batch_first=True) self.ff = nn.Sequential(nn.Linear(d_model, d_model * 4), nn.GELU(), nn.Linear(d_model * 4, d_model)) self.norm1, self.norm2 = nn.LayerNorm(d_model), nn.LayerNorm(d_model) self.drop = nn.Dropout(dropout) def forward(self, x, mask=None): attn_out, _ = self.attn(x, x, x, attn_mask=mask) x = self.norm1(x + self.drop(attn_out)) return self.norm2(x + self.drop(self.ff(x))) model = TransformerBlock().to('cuda') optimizer = torch.optim.AdamW(model.parameters(), lr=3e-4)