视觉语言模型：视觉编码器、投影层与语言模型

图像 → Processor → Vision Encoder → Projector → Visual Tokens
文本 → Tokenizer ────────────────────────────────┐
                                                 ▼
                                      Language Model → 回答

python -m pip install "transformers>=4.56,<6" "torch>=2.3,<3" pillow

from transformers import pipeline


model_id = "Qwen/Qwen2.5-VL-3B-Instruct"
pipe = pipeline(
    "image-text-to-text",
    model=model_id,
    device_map="auto",
    dtype="auto",
)
messages = [{
    "role": "user",
    "content": [
        {"type": "image", "path": "device-panel.png"},
        {"type": "text", "text": "只描述图片中能直接观察到的指示灯颜色；看不清就回答 unknown。"},
    ],
}]
result = pipe(text=messages, max_new_tokens=64, do_sample=False)
print(result[0]["generated_text"][-1]["content"])