文档切分：在长度、重叠和语义边界之间做可验证取舍

python -m pip install "langchain-text-splitters>=1,<2"

from langchain_core.documents import Document
from langchain_text_splitters import (
    MarkdownHeaderTextSplitter,
    RecursiveCharacterTextSplitter,
)


text = """# 值班规则
## 告警升级
一般告警由当班人员在15分钟内确认。关键告警需要立即通知负责人。

## 恢复检查
告警恢复后仍需观察10分钟，并记录设备编号、恢复时间和处理人。
"""

headers = [("#", "h1"), ("##", "h2")]
sections = MarkdownHeaderTextSplitter(headers_to_split_on=headers).split_text(text)
splitter = RecursiveCharacterTextSplitter(
    chunk_size=70,
    chunk_overlap=15,
    separators=["\n\n", "。", "；", "，", "\n", ""],
)

chunks = splitter.split_documents([
    Document(page_content=section.page_content, metadata={
        **section.metadata,
        "source": "duty-rules.md",
        "version": "v1",
    })
    for section in sections
])

for index, chunk in enumerate(chunks):
    chunk.metadata["chunk_id"] = f"duty-rules:v1:{index:03d}"
    print(chunk.metadata, repr(chunk.page_content))

checks = {
    "关键告警如何处理？": ["关键告警", "立即通知负责人"],
    "恢复后观察多久？": ["恢复后", "10分钟"],
}

for question, terms in checks.items():
    matched = [c for c in chunks if all(term in c.page_content for term in terms)]
    print(question, "PASS" if matched else "FAIL")