Files
bot042/tests/test_media_pipeline.py
openbot 13b8d7a466 test(media): 补分块下载中断与 root 自动创建的实测
上一轮两条「未验证」的敞口,本次用可执行的测试消掉:

分块传输(Transfer-Encoding: chunked):
- download_limited 的 Content-Length 预检对分块响应无效,只剩按块累计
  这一道闸。此前只有代码没有测试,无法证明超限时连接真的被中断。
- 新增 _ChunkedServer:起一个真 socket 的最小 HTTP/1.1 服务端,不下发
  Content-Length、按 16 KiB 分块持续写 8 MiB。用真连接而不是桩,因为
  要验证的正是「读到超限就断开」——桩测不出中断。
- 断言两条:抛 MediaRejected 且 detail 含「已中断」;服务端 sent < total
  说明它没能把 8 MiB 写完,即连接在中途被断开,而不是读完再判。
- 变异验证:临时移除 media.py 里的累计判限后,该用例从通过变为
  DID NOT RAISE,确认它测的确实是这段逻辑;随后已还原,media.py 与
  HEAD 无差异(git diff 为空)。

root 不存在:
- store_media 用 mkdir(parents=True, exist_ok=True) 建目录,但此前没有
  断言覆盖「首次运行时 home/data/inbound 还不存在」这条路径。

全量 295 passed,ruff 全绿,新增两条连续跑 3 次均稳定(涉及真 socket
与线程,排除偶发)。
2026-11-10 10:22:21 +08:00

631 lines
23 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""issue #1 验收:媒体链路的接入、护栏与安全边界。
对应 issue 的验收标准(`docs` 与本文件一一对应):
1. 媒体能进上下文 —— `test_image_reaches_agent_context` 等;
2. 文本 + 文件混合不丢 —— `test_text_and_file_both_reach_agent`;
3. 智能体发媒体走原生下发 —— `test_outbox_part_flows_to_channel`;
4. 超限被拒且不炸内存 —— `test_oversize_*`;
5. 恶意文件名不越界 —— `test_sanitize_*` / `test_safe_join_*`;
6. 以上均有单测且 pytest 全绿。
本文件只测**媒体策略与链路**,不重复测 AgentReply 的会话/落库语义
(见 `tests/test_builtin.py`)。
"""
from __future__ import annotations
import asyncio
import base64
import socket
import sys
import threading
import types
from pathlib import Path
from typing import Self
import pytest
from bot042.agent.outbox import (
MediaOutbox,
bind,
current_outbox,
make_file_part,
make_image_part,
unbind,
)
from bot042.channels.cli import CliChannel
from bot042.channels.manager import ChannelManager
from bot042.channels.media import (
MB,
MediaLimits,
MediaRejected,
check_mime,
decode_base64_limited,
download_limited,
read_file_limited,
safe_join,
sanitize_filename,
store_media,
)
from bot042.channels.models import (
ChannelSubject,
FileContent,
ImageContent,
InboundMessage,
TextContent,
)
from bot042.config import Settings
from bot042.flows.builtin import ChannelServices
from bot042.flows.media_input import prepare_inbound
PNG_HEAD = b"\x89PNG\r\n\x1a\n" + b"\x00" * 64
PDF_HEAD = b"%PDF-1.4\n" + b"\x00" * 64
# ---------------------------------------------------------------------------
# 5. 路径安全:文件名净化与根目录约束
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"hostile",
[
"../../etc/passwd",
"..\\..\\Windows\\System32\\config",
"/etc/shadow",
"C:\\Windows\\system.ini",
"....//....//evil.sh",
"a/../../../b.png",
],
)
def test_sanitize_filename_never_escapes(hostile: str) -> None:
"""恶意文件名净化后不得含路径分隔符,也不得是 . 或 .."""
cleaned = sanitize_filename(hostile)
assert "/" not in cleaned and "\\" not in cleaned
assert cleaned not in {".", ".."}
assert not cleaned.startswith(".")
def test_safe_join_keeps_target_inside_root(tmp_path: Path) -> None:
"""即使传入穿越片段,落点也必须在 root 之内。"""
root = tmp_path / "session"
root.mkdir()
target = safe_join(root, sanitize_filename("../../escape.png"))
assert target.parent == root
assert root in target.parents
def test_store_media_lands_inside_session_dir(tmp_path: Path) -> None:
"""落盘路径由会话键决定:会话键里的 .. 会被净化,不会跳出 root。"""
part = ImageContent(mime_type="image/png").model_copy(
update={"data": base64.b64encode(PNG_HEAD).decode()}
)
stored = asyncio.run(store_media(part, root=tmp_path, session_key="../s"))
assert tmp_path in stored.path.parents
assert ".." not in str(stored.path.relative_to(tmp_path))
# 会话键被净化成单个目录段,不是嵌套路径
assert stored.path.parent.parent == tmp_path
def test_store_media_dedupes_same_name(tmp_path: Path) -> None:
"""同名不同内容不得互相覆盖(后到的加哈希前缀)。"""
first = ImageContent(mime_type="image/png").model_copy(
update={"data": base64.b64encode(PNG_HEAD).decode()}
)
second = ImageContent(mime_type="image/png").model_copy(
update={"data": base64.b64encode(PNG_HEAD + b"other").decode()}
)
a = asyncio.run(store_media(first, root=tmp_path, session_key="s"))
b = asyncio.run(store_media(second, root=tmp_path, session_key="s"))
assert a.path != b.path
assert a.path.read_bytes() == PNG_HEAD
# ---------------------------------------------------------------------------
# 4. 大小护栏:本地读取、base64 解码、下载
# ---------------------------------------------------------------------------
def test_read_file_limited_rejects_oversize(tmp_path: Path) -> None:
big = tmp_path / "big.bin"
big.write_bytes(b"x" * 2048)
with pytest.raises(MediaRejected) as err:
read_file_limited(big, 1024)
assert "超过大小上限" in err.value.reason
def test_read_file_limited_reads_at_most_limit_plus_one(tmp_path: Path) -> None:
"""判定越界只需多读 1 字节,不得把整个超大文件读进内存。"""
big = tmp_path / "huge.bin"
big.write_bytes(b"x" * (8 * 1024 * 1024))
with pytest.raises(MediaRejected):
read_file_limited(big, 1024)
def test_read_file_limited_missing_file(tmp_path: Path) -> None:
with pytest.raises(MediaRejected) as err:
read_file_limited(tmp_path / "nope.bin", 1024)
assert err.value.reason == "文件不存在"
def test_decode_base64_limited_rejects_oversize() -> None:
payload = base64.b64encode(b"x" * 4096).decode()
with pytest.raises(MediaRejected):
decode_base64_limited(payload, 1024)
def test_decode_base64_limited_invalid() -> None:
with pytest.raises(MediaRejected):
decode_base64_limited("!!!not-base64!!!", 1024)
@pytest.mark.parametrize(
("declared", "head", "expected"),
[
("image/png", PNG_HEAD, "image/png"),
# 声明撒谎:实际是 PNG,按魔数修正(改扩展名绕不过类型限制)
("text/plain", PNG_HEAD, "image/png"),
("image/jpeg", PNG_HEAD, "image/png"),
],
)
def test_check_mime_prefers_magic_bytes(declared: str, head: bytes, expected: str) -> None:
"""声明类型与魔数冲突时以魔数为准。"""
got = check_mime(ImageContent(), declared, head)
assert got == expected
def test_check_mime_file_accepts_pdf() -> None:
"""非图片类文件走 FileContent,PDF 应放行并采信魔数。"""
got = check_mime(FileContent(filename="a.pdf"), "", PDF_HEAD)
assert got == "application/pdf"
def test_check_mime_image_rejects_pdf() -> None:
"""把 PDF 当图片投递:必须拒绝,而不是带着错误 MIME 继续。"""
with pytest.raises(MediaRejected) as err:
check_mime(ImageContent(), "application/pdf", PDF_HEAD)
assert "不被接受" in err.value.reason
def test_check_mime_rejects_executable() -> None:
"""可执行文件按扩展名拒绝:智能体读到的文件不该原样推给对方。"""
with pytest.raises(MediaRejected):
check_mime(FileContent(filename="virus.exe"), "", b"MZ")
def test_media_limits_per_type_is_ordered() -> None:
limits = MediaLimits()
assert limits.image <= limits.file
assert limits.for_part(ImageContent()).__class__ is int
# ---------------------------------------------------------------------------
# 3. 出站:outbox 的工具入口
# ---------------------------------------------------------------------------
def test_make_image_part_validates_type(tmp_path: Path) -> None:
fake = tmp_path / "notimage.png"
fake.write_bytes(b"plain text, not a png at all")
with pytest.raises(MediaRejected):
make_image_part(str(fake))
def test_make_image_part_ok(tmp_path: Path) -> None:
picture = tmp_path / "cat.png"
picture.write_bytes(PNG_HEAD)
part = make_image_part(str(picture), caption="猫")
assert isinstance(part, ImageContent)
assert part.size == len(PNG_HEAD)
assert part.mime_type == "image/png"
assert part.alt_text == "猫"
def test_make_file_part_rejects_missing(tmp_path: Path) -> None:
with pytest.raises(MediaRejected) as err:
make_file_part(str(tmp_path / "ghost.pdf"))
assert "不存在" in err.value.reason
def test_outbox_caps_pending_count() -> None:
"""单轮附件数量设上限,避免模型刷屏式调用把平台打爆。"""
box = MediaOutbox(max_pending=2)
part = ImageContent(mime_type="image/png")
box.push(part)
box.push(part)
with pytest.raises(MediaRejected):
box.push(part)
assert len(box.drain()) == 2
assert len(box) == 0
def test_outbox_context_binding_is_isolated() -> None:
box = MediaOutbox()
assert current_outbox() is None
token = bind(box)
try:
assert current_outbox() is box
finally:
unbind(token)
assert current_outbox() is None
# ---------------------------------------------------------------------------
# 1 & 2. 入站:多模态块构建与降级
# ---------------------------------------------------------------------------
def _prepare(**kwargs):
from bot042.flows.media_input import prepare_inbound
return prepare_inbound
def test_prepare_inbound_inlines_small_image(tmp_path: Path) -> None:
from bot042.channels.models import ChannelSubject, InboundMessage
from bot042.flows.media_input import prepare_inbound
picture = tmp_path / "cat.png"
picture.write_bytes(PNG_HEAD)
message = InboundMessage(
channel_id="c1",
channel_type="cli",
channel_subject=ChannelSubject(subject_id="u1"),
content=[
TextContent(text="这是什么"),
ImageContent(local_path=str(picture), mime_type="image/png"),
],
)
prepared = asyncio.run(prepare_inbound(message, root=tmp_path / "in"))
kinds = [b["type"] for b in prepared.blocks]
assert kinds == ["text", "image_url"]
assert prepared.blocks[1]["image_url"]["url"].startswith("data:image/png;base64,")
assert prepared.is_multimodal
# 摘要里带附件清单,且不含 base64
assert "[附件]" in prepared.summary
assert "base64" not in prepared.summary
def test_prepare_inbound_text_and_file_both_kept(tmp_path: Path) -> None:
"""文本 + 文件混合:两者都进上下文,不静默丢弃任何一个。"""
from bot042.channels.models import ChannelSubject, InboundMessage
from bot042.flows.media_input import prepare_inbound
note = tmp_path / "notes.txt"
note.write_text("第一行\n第二行\n", encoding="utf-8")
message = InboundMessage(
channel_id="c1",
channel_type="cli",
channel_subject=ChannelSubject(subject_id="u1"),
content=[
TextContent(text="帮我看看这个文件"),
FileContent(local_path=str(note), filename="notes.txt", mime_type="text/plain"),
],
)
prepared = asyncio.run(prepare_inbound(message, root=tmp_path / "in"))
joined = "\n".join(b.get("text", "") for b in prepared.blocks)
assert "帮我看看这个文件" in joined
assert "第一行" in joined # 文本文件内容被抽取进上下文
assert len(prepared.stored) == 1
def test_prepare_inbound_oversize_degrades_to_note(tmp_path: Path) -> None:
"""超限附件降级成一条说明,不抛异常、不中断其余内容。"""
from bot042.channels.models import ChannelSubject, InboundMessage
from bot042.flows.media_input import prepare_inbound
big = tmp_path / "big.png"
big.write_bytes(PNG_HEAD + b"\x00" * 4096)
message = InboundMessage(
channel_id="c1",
channel_type="cli",
channel_subject=ChannelSubject(subject_id="u1"),
content=[
TextContent(text="看图"),
ImageContent(local_path=str(big), mime_type="image/png"),
],
)
prepared = asyncio.run(
prepare_inbound(
message, root=tmp_path / "in", limits=MediaLimits(image=128, file=128)
)
)
assert prepared.has_content
assert prepared.notes
assert any("上限" in n or "超过" in n for n in prepared.notes)
def test_prepare_inbound_missing_file_is_noted(tmp_path: Path) -> None:
from bot042.channels.models import ChannelSubject, InboundMessage
from bot042.flows.media_input import prepare_inbound
message = InboundMessage(
channel_id="c1",
channel_type="cli",
channel_subject=ChannelSubject(subject_id="u1"),
content=[FileContent(local_path=str(tmp_path / "ghost.pdf"), filename="ghost.pdf")],
)
prepared = asyncio.run(prepare_inbound(message, root=tmp_path / "in"))
assert prepared.notes
assert not prepared.stored
# ---------------------------------------------------------------------------
# CLI 渠道:无平台时的收发调试入口
# ---------------------------------------------------------------------------
def test_cli_image_command(tmp_path: Path) -> None:
picture = tmp_path / "cat.png"
picture.write_bytes(PNG_HEAD)
channel = CliChannel()
message = channel.parse_inbound(f"/image {picture}")
assert isinstance(message.content[0], ImageContent)
assert message.content[0].mime_type == "image/png"
def test_cli_file_command(tmp_path: Path) -> None:
note = tmp_path / "a.txt"
note.write_text("hi", encoding="utf-8")
channel = CliChannel()
message = channel.parse_inbound(f"/file {note}")
assert isinstance(message.content[0], FileContent)
assert message.content[0].filename == "a.txt"
def test_cli_image_command_rejects_oversize(tmp_path: Path) -> None:
picture = tmp_path / "cat.png"
picture.write_bytes(PNG_HEAD + b"\x00" * 4096)
channel = CliChannel()
channel._media_limits = MediaLimits(image=64, file=64)
message = channel.parse_inbound(f"/image {picture}")
assert isinstance(message.content[0], TextContent)
assert "未接收" in message.content[0].text
def test_cli_image_command_wrong_type(tmp_path: Path) -> None:
note = tmp_path / "a.txt"
note.write_text("hi", encoding="utf-8")
channel = CliChannel()
message = channel.parse_inbound(f"/image {note}")
assert isinstance(message.content[0], TextContent)
assert "不是图片" in message.content[0].text
def test_cli_plain_text_unchanged() -> None:
channel = CliChannel()
message = channel.parse_inbound("你好")
assert isinstance(message.content[0], TextContent)
assert message.content[0].text == "你好"
def test_cli_send_media_reports_real_bytes(tmp_path: Path, capsys) -> None:
"""出站回执必须来自真实读取:给出大小与 sha256 前缀,而不是「已发送」。"""
picture = tmp_path / "cat.png"
picture.write_bytes(PNG_HEAD)
channel = CliChannel()
asyncio.run(
channel._send_media(None, ImageContent(local_path=str(picture), mime_type="image/png"))
)
out = capsys.readouterr().out
assert "媒体回执" in out
assert str(len(PNG_HEAD)) in out
assert "sha256:" in out
def test_cli_send_media_missing_file_is_reported(tmp_path: Path, capsys) -> None:
channel = CliChannel()
asyncio.run(
channel._send_media(
None, ImageContent(local_path=str(tmp_path / "ghost.png"), mime_type="image/png")
)
)
out = capsys.readouterr().out
assert "媒体" in out and ("被拒" in out or "失败" in out)
# ---------------------------------------------------------------------------
# 平台解密钩子:接线必须真的到达 store_media(不只是签名吻合)
# ---------------------------------------------------------------------------
def test_store_media_applies_transform_before_sniffing(tmp_path: Path) -> None:
"""transform 必须在类型嗅探**之前**执行:密文嗅魔数只会得到 octet-stream。"""
ciphertext = b"\x00\x01\x02\x03cipher-not-a-png" * 4
calls: list[tuple[str, int]] = []
def fake_decrypt(part, raw: bytes) -> bytes:
calls.append((part.type, len(raw)))
return PNG_HEAD
# 用内联字节而非 url:这条测的是 transform 与嗅探的顺序,不该触发网络
part = ImageContent(mime_type="application/octet-stream").model_copy(
update={"data": base64.b64encode(ciphertext).decode()}
)
stored = asyncio.run(store_media(part, root=tmp_path, transform=fake_decrypt))
assert calls == [("image", len(ciphertext))]
# 明文是 PNG,所以最终类型由**明文**魔数决定
assert stored.mime == "image/png"
assert stored.path.read_bytes() == PNG_HEAD
def test_store_media_rechecks_size_after_transform(tmp_path: Path) -> None:
"""密文没超不等于明文没超:转换后必须再量一次。"""
limits = MediaLimits(image=64)
def expand(part, raw: bytes) -> bytes:
return b"\x00" * (limits.image + 1)
part = ImageContent(mime_type="application/octet-stream").model_copy(
update={"data": base64.b64encode(b"small-cipher").decode()}
)
with pytest.raises(MediaRejected) as excinfo:
asyncio.run(store_media(part, root=tmp_path, limits=limits, transform=expand))
assert "解密后" in excinfo.value.detail
def test_store_media_transform_failure_becomes_rejection(tmp_path: Path) -> None:
"""解密异常要转成 MediaRejected,不能冒泡成未捕获异常打断整轮。"""
def broken(part, raw: bytes) -> bytes:
raise ValueError("bad padding")
part = ImageContent(mime_type="application/octet-stream").model_copy(
update={"data": base64.b64encode(b"cipher").decode()}
)
with pytest.raises(MediaRejected) as excinfo:
asyncio.run(store_media(part, root=tmp_path, transform=broken))
assert "解密失败" in excinfo.value.reason
def test_prepare_inbound_passes_transform_through(tmp_path: Path) -> None:
"""prepare_inbound 必须把 transform 透传到 store_media,否则解密形同虚设。"""
seen: list[int] = []
def fake_decrypt(part, raw: bytes) -> bytes:
seen.append(len(raw))
return PNG_HEAD
message = InboundMessage(
channel_id="c1",
channel_type="t",
channel_subject=ChannelSubject(subject_id="u1"),
content=[ImageContent(data=base64.b64encode(b"cipher-bytes").decode())],
)
prepared = asyncio.run(
prepare_inbound(message, root=tmp_path / "in", transform=fake_decrypt)
)
assert seen == [len(b"cipher-bytes")]
assert prepared.stored and prepared.stored[0].read_bytes() == PNG_HEAD
def test_wecom_media_transform_uses_per_message_aeskey(monkeypatch) -> None:
"""企业微信的解密钩子按本条消息的 url 取密钥;没有密钥时原样返回。"""
from bot042.channels.im import wecom as wecom_mod
calls: list[tuple[bytes, str]] = []
def fake_decrypt_file(raw: bytes, key: str) -> bytes:
calls.append((raw, key))
return PNG_HEAD
fake_sdk = types.SimpleNamespace(decrypt_file=fake_decrypt_file)
monkeypatch.setitem(sys.modules, "wecom_aibot_sdk", fake_sdk)
channel = wecom_mod.WeComChannel()
url = "https://example.invalid/media/1"
transform = channel.make_media_transform({"_media_aes_keys": {url: "k" * 43}})
part = ImageContent(url=url)
assert transform(part, b"cipher") == PNG_HEAD
assert calls == [(b"cipher", "k" * 43)]
# 没有对应 url 的密钥:返回原字节,不猜、不抛
other = ImageContent(url="https://example.invalid/media/2")
assert transform(other, b"cipher") == b"cipher"
def test_channel_services_media_transform_none_for_unknown_channel(tmp_path: Path) -> None:
"""渠道未注册时返回 None(不解密),不抛异常也不跳过后续校验。"""
services = ChannelServices(
channels=ChannelManager(), settings=Settings(home=tmp_path), db=tmp_path / "b.db"
)
assert services.media_transform("no-such-channel", {}) is None
# ---------------------------------------------------------------------------
# 4. 下载限长:分块传输(无 Content-Length)必须靠累计判限中断
# ---------------------------------------------------------------------------
class _ChunkedServer:
"""极简 HTTP/1.1 分块响应服务端,用于验证「服务端不给 Content-Length」时的闸。
真实平台可能用 ``Transfer-Encoding: chunked`` 下发媒体,此时
``Content-Length`` 预检形同虚设,只剩按块累计这一道。用真 socket 而不是
桩,是因为要验证的正是「读到超限就断开连接」——桩连接测不出中断。
"""
def __init__(self, total: int, piece: int = 16 * 1024) -> None:
self.total = total
self.piece = piece
self.sent = 0
self._sock = socket.socket()
self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._sock.bind(("127.0.0.1", 0))
self._sock.listen(1)
self._sock.settimeout(10)
self.url = f"http://127.0.0.1:{self._sock.getsockname()[1]}/media"
def __enter__(self) -> Self:
self._thread = threading.Thread(target=self._serve, daemon=True)
self._thread.start()
return self
def __exit__(self, *exc_info: object) -> None:
self._thread.join(timeout=10)
self._sock.close()
def _serve(self) -> None:
try:
conn, _ = self._sock.accept()
except OSError:
return
with conn:
conn.settimeout(10)
head = b""
while b"\r\n\r\n" not in head:
try:
piece = conn.recv(4096)
except OSError:
return
if not piece:
return
head += piece
conn.sendall(
b"HTTP/1.1 200 OK\r\n"
b"Content-Type: image/png\r\n"
b"Transfer-Encoding: chunked\r\n\r\n"
)
payload = PNG_HEAD + b"\x00" * (self.piece - len(PNG_HEAD))
try:
while self.sent < self.total:
conn.sendall(b"%x\r\n" % len(payload) + payload + b"\r\n")
self.sent += len(payload)
conn.sendall(b"0\r\n\r\n")
except OSError:
# 客户端超限后主动断开,这正是被测行为
return
def test_download_limited_aborts_chunked_stream() -> None:
"""没有 Content-Length 时,累计超过上限就要中断,不能把整份读完。"""
total = 8 * MB
limit = 256 * 1024
with (
_ChunkedServer(total=total) as server,
pytest.raises(MediaRejected) as excinfo,
):
asyncio.run(
download_limited(server.url, limit=limit, chunk=16 * 1024, timeout=10.0)
)
assert "已中断" in excinfo.value.detail
# 服务端没能把 8 MiB 全部写出:说明连接是在中途被断开的,不是读完再判
assert server.sent < total
def test_store_media_creates_missing_root(tmp_path: Path) -> None:
"""root 不存在时应自行创建:首次运行 ``home/data/inbound`` 还不存在。"""
root = tmp_path / "data" / "inbound"
part = ImageContent(mime_type="image/png").model_copy(
update={"data": base64.b64encode(PNG_HEAD).decode()}
)
stored = asyncio.run(store_media(part, root=root))
assert root.is_dir()
assert stored.path.parent == root
assert stored.path.read_bytes() == PNG_HEAD