Embedding 与语义搜索：把文本映射到可比较的向量空间

import hashlib
import math
from functools import lru_cache


def tokens(text: str) -> list[str]:
    compact = "".join(text.lower().split())
    return [compact[i:i + 2] for i in range(max(1, len(compact) - 1))]


@lru_cache(maxsize=2048)
def embed(text: str, dimensions: int = 128) -> tuple[float, ...]:
    vector = [0.0] * dimensions
    for token in tokens(text):
        digest = hashlib.sha256(token.encode("utf-8")).digest()
        index = int.from_bytes(digest[:4], "big") % dimensions
        sign = 1.0 if digest[4] % 2 == 0 else -1.0
        vector[index] += sign
    norm = math.sqrt(sum(value * value for value in vector)) or 1.0
    return tuple(value / norm for value in vector)


def cosine(left: tuple[float, ...], right: tuple[float, ...]) -> float:
    if len(left) != len(right):
        raise ValueError("embedding_dimension_mismatch")
    return sum(a * b for a, b in zip(left, right))


documents = [
    "AX-3 终端失去网络连接",
    "AX-8 电池电量低于20%",
    "AX-5 已切换备用线路并恢复在线",
]
matrix = [embed(text) for text in documents]  # 批量接口应一次提交多条
query = embed("设备断网后怎样恢复")
ranking = sorted(
    [(cosine(query, vector), text) for text, vector in zip(documents, matrix)],
    reverse=True,
)
print(ranking)