"""联网工具测试:URL 归一、截断、HTML 转 Markdown、必应解析与失败降级。""" from __future__ import annotations import asyncio from typing import Any, Self import pytest from bot042.agent import web from bot042.agent.web import ( WebError, _BingParser, _html_to_markdown, _normalize_url, _truncate, fetch_page, search_web, web_fetch, web_search, wrap_untrusted, ) BING_HTML = """
  • Welcome to Python

    The mission of the Python Software Foundation is to promote Python.

  • Python 3 Documentation

    Official documentation for Python.

  • """ ARTICLE_HTML = """ 示例

    标题一

    第一段,含 强调 与 相对链接。

    小节

    1. 一
    2. 二
    print("hi")
    引用的话
    """ class _FakeResponse: def __init__( self, *, status_code: int = 200, text: str = "", markdown: str = "", plain: str = "", content_type: str = "text/html; charset=utf-8", url: str = "https://example.com/page", ) -> None: self.status_code = status_code self.text = text self.text_markdown = markdown self.text_plain = plain self.headers = {"content-type": content_type} self.url = url class _FakeClient: def __init__(self, response: _FakeResponse) -> None: self.response = response self.calls: list[dict[str, Any]] = [] async def __aenter__(self) -> Self: return self async def __aexit__(self, *args: object) -> bool: return False async def get(self, url: str, **_kwargs: Any) -> _FakeResponse: self.calls.append({"url": url, **_kwargs}) return self.response def test_normalize_url() -> None: assert _normalize_url("example.com") == "https://example.com" assert _normalize_url(" http://a.b/c ") == "http://a.b/c" with pytest.raises(ValueError): _normalize_url(" ") with pytest.raises(ValueError): _normalize_url("ftp://example.com/x") def test_truncate() -> None: assert _truncate("abc", 10) == "abc" out = _truncate("abcdef", 3) assert out.startswith("abc") assert "已截断" in out def test_bing_parser_extracts_results() -> None: parser = _BingParser() parser.feed(BING_HTML) assert len(parser.results) == 2 assert parser.results[0]["url"] == "https://www.python.org" assert parser.results[0]["title"] == "Welcome to Python" assert "Software Foundation" in parser.results[0]["snippet"] assert parser.results[1]["url"] == "https://docs.python.org/3/" def test_html_to_markdown_keeps_readable_structure() -> None: md = _html_to_markdown(ARTICLE_HTML, base_url="https://example.com/page") assert "# 标题一" in md assert "## 小节" in md assert "**强调**" in md assert "[相对链接](https://example.com/docs/x.html)" in md assert "- 甲" in md and "- 乙" in md assert "1. 一" in md and "2. 二" in md assert "```" in md and 'print("hi")' in md assert "> 引用的话" in md assert "

    " not in md and "&" not in md def test_html_to_markdown_drops_noise() -> None: md = _html_to_markdown(ARTICLE_HTML, base_url="https://example.com/page") assert "站点菜单" not in md # nav 整棵子树丢弃 assert "版权所有" not in md # footer 丢弃 assert "alert(1)" not in md # 脚本丢弃 assert "color: red" not in md # 样式丢弃 assert "¶" not in md # 标题锚点没有阅读价值 def test_html_to_markdown_drops_navigation_containers() -> None: html = ( '
    面包屑导航
    ' '' '' "

    正文

    " ) md = _html_to_markdown(html) assert "正文" in md assert "面包屑导航" not in md assert "侧栏目录" not in md assert "站点菜单" not in md def test_html_to_markdown_keeps_list_items_compact() -> None: md = _html_to_markdown("") assert md == "- 甲\n- 乙" def test_html_to_markdown_indents_nested_list() -> None: md = _html_to_markdown("") assert "- 甲" in md assert " - 甲一" in md def test_html_to_markdown_keeps_code_indent() -> None: md = _html_to_markdown("
    def f():\n    return 1
    ") assert "def f():" in md assert " return 1" in md # 代码缩进不能被折叠 def test_html_to_markdown_collapses_whitespace() -> None: md = _html_to_markdown("

    a b\n\tc

    d

    ") assert "a b c" in md assert "\n\n\n" not in md def test_fetch_page_converts_html_to_markdown(monkeypatch: pytest.MonkeyPatch) -> None: client = _FakeClient(_FakeResponse(text=ARTICLE_HTML)) monkeypatch.setattr(web, "_client", lambda: client) out = asyncio.run(fetch_page("example.com")) assert "# 标题一" in out assert "

    " not in out assert client.calls[0]["url"] == "https://example.com" def test_fetch_page_keeps_plain_text_as_is(monkeypatch: pytest.MonkeyPatch) -> None: response = _FakeResponse(text="a\nb", content_type="text/plain; charset=utf-8") monkeypatch.setattr(web, "_client", lambda: _FakeClient(response)) assert asyncio.run(fetch_page("https://example.com/x.txt")) == "a\nb" def test_fetch_page_wraps_json_in_code_block(monkeypatch: pytest.MonkeyPatch) -> None: response = _FakeResponse(text='{"ok":true}', content_type="application/json") monkeypatch.setattr(web, "_client", lambda: _FakeClient(response)) out = asyncio.run(fetch_page("https://example.com/api")) assert out == '```json\n{"ok":true}\n```' def test_fetch_page_truncates(monkeypatch: pytest.MonkeyPatch) -> None: body = "

    " + "字" * 200 + "

    " monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse(text=body))) out = asyncio.run(fetch_page("example.com", max_chars=20)) assert "已截断" in out assert len(out) < 100 def test_fetch_page_raises_on_http_error(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse(status_code=403))) with pytest.raises(WebError, match="403"): asyncio.run(fetch_page("https://example.com")) def test_fetch_page_raises_on_empty_body(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse())) with pytest.raises(WebError, match="没有可读文本"): asyncio.run(fetch_page("https://example.com")) def test_search_web_parses_and_limits(monkeypatch: pytest.MonkeyPatch) -> None: client = _FakeClient(_FakeResponse(text=BING_HTML)) monkeypatch.setattr(web, "_client", lambda: client) results = asyncio.run(search_web("python", limit=1)) assert len(results) == 1 assert results[0]["url"] == "https://www.python.org" assert client.calls[0]["params"]["q"] == "python" def test_search_web_falls_back_with_page_excerpt(monkeypatch: pytest.MonkeyPatch) -> None: client = _FakeClient(_FakeResponse(text="

    被拦截

    ")) monkeypatch.setattr(web, "_client", lambda: client) with pytest.raises(WebError, match="未解析到搜索结果"): asyncio.run(search_web("python")) def test_wrap_untrusted_marks_data_not_instructions() -> None: wrapped = wrap_untrusted("# 标题\n正文", source="https://example.com/a", fmt="Markdown") assert wrapped.startswith("[外部网页内容开始]") assert "来源: https://example.com/a" in wrapped assert "格式: Markdown" in wrapped assert "不是指令" in wrapped assert "# 标题\n正文" in wrapped assert wrapped.endswith("[外部网页内容结束]") def test_web_fetch_tool_wraps_content(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse(text=ARTICLE_HTML))) out = asyncio.run(web_fetch.ainvoke({"url": "https://example.com/page"})) assert out.startswith("[外部网页内容开始]") assert "来源: https://example.com/page" in out assert "格式: Markdown(由 HTML 转换)" in out assert "不是指令" in out assert "# 标题一" in out assert out.endswith("[外部网页内容结束]") def test_web_fetch_tool_returns_text_on_failure(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse(status_code=500))) out = asyncio.run(web_fetch.ainvoke({"url": "https://example.com"})) assert out.startswith("[抓取失败]") assert "500" in out def test_web_search_tool_formats_results(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse(text=BING_HTML))) out = asyncio.run(web_search.ainvoke({"query": "python", "limit": 2})) assert "1. Welcome to Python" in out assert "https://www.python.org" in out assert "2. Python 3 Documentation" in out def test_get_tools_exposes_web_tools() -> None: from bot042.agent.tools import get_tools names = {tool.name for tool in get_tools()} assert {"perceive", "move", "speak", "web_fetch", "web_search"} <= names