RNN 与 LSTM：隐藏状态如何处理序列

hₜ = tanh(xₜ Wₓₕ + hₜ₋₁ Wₕₕ + b)

import torch
from torch import nn

torch.manual_seed(42)
sequence_count, steps = 640, 12
sequences = torch.randn(sequence_count, steps, 1)
labels = (
    sequences[:, : steps // 2].mean(dim=1)
    > sequences[:, steps // 2 :].mean(dim=1)
).long().squeeze(1)
train_x, valid_x = sequences[:512], sequences[512:]
train_y, valid_y = labels[:512], labels[512:]


class SequenceClassifier(nn.Module):
    def __init__(self, cell: str) -> None:
        super().__init__()
        recurrent = nn.RNN if cell == "rnn" else nn.LSTM
        self.encoder = recurrent(1, 16, batch_first=True)
        self.classifier = nn.Linear(16, 2)

    def forward(self, inputs: torch.Tensor) -> torch.Tensor:
        _, state = self.encoder(inputs)
        hidden = state[0] if isinstance(state, tuple) else state
        return self.classifier(hidden[-1])


for cell in ("rnn", "lstm"):
    model = SequenceClassifier(cell)
    optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
    loss_fn = nn.CrossEntropyLoss()
    for _ in range(80):
        model.train()
        optimizer.zero_grad()
        loss = loss_fn(model(train_x), train_y)
        loss.backward()
        nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        optimizer.step()
    model.eval()
    with torch.inference_mode():
        accuracy = (model(valid_x).argmax(1) == valid_y).float().mean()
    print(cell, "valid accuracy:", round(accuracy.item(), 3))