分词与词表：文本怎样变成 Token ID

from dataclasses import dataclass


VOCAB = [
    "<pad>", "<unk>", "<cls>", "<sep>",
    "巡检", "模型", "发现", "异常", "节点", "已", "恢复",
]


@dataclass
class EncodedText:
    tokens: list[str]
    input_ids: list[int]
    attention_mask: list[int]


class GreedyTokenizer:
    def __init__(self, vocab: list[str]):
        self.token_to_id = {token: index for index, token in enumerate(vocab)}
        self.words = sorted(
            [token for token in vocab if not token.startswith("<")],
            key=len,
            reverse=True,
        )

    def tokenize(self, text: str) -> list[str]:
        tokens = []
        cursor = 0
        while cursor < len(text):
            if text[cursor].isspace():
                cursor += 1
                continue
            matched = next(
                (word for word in self.words if text.startswith(word, cursor)),
                None,
            )
            if matched:
                tokens.append(matched)
                cursor += len(matched)
            else:
                tokens.append(text[cursor])
                cursor += 1
        return tokens

    def encode(self, text: str, max_length: int = 10) -> EncodedText:
        content = self.tokenize(text)[: max_length - 2]
        tokens = ["<cls>", *content, "<sep>"]
        ids = [self.token_to_id.get(token, self.token_to_id["<unk>"]) for token in tokens]
        mask = [1] * len(ids)

        pad_count = max_length - len(ids)
        tokens.extend(["<pad>"] * pad_count)
        ids.extend([self.token_to_id["<pad>"]] * pad_count)
        mask.extend([0] * pad_count)
        return EncodedText(tokens, ids, mask)


tokenizer = GreedyTokenizer(VOCAB)
encoded = tokenizer.encode("巡检模型发现 NX-3 异常", max_length=10)
print("tokens:", encoded.tokens)
print("ids:   ", encoded.input_ids)
print("mask:  ", encoded.attention_mask)

tokens: ['<cls>', '巡检', '模型', '发现', 'N', 'X', '-', '3', '异常', '<sep>']
ids:    [2, 4, 5, 6, 1, 1, 1, 1, 7, 3]
mask:   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]