feat: 重构为 agent / channels / flows / db 分层架构

- agent: 新增 base.py 执行基类、execution.py 执行器、web.py 网络工具,
  middlewares/steering.py 中间件;移除旧的 agent.py
- channels: 新增 ChannelManager、registry、models 与 IM 渠道(企业微信),
  扩展 base.py
- flows: 新增 agent_task / bus / cron / scheduler / service / stream / rpc /
  processor / tasks / model_settings / builtin;移除旧的 task.py
- db: 新增 sqlite schema 与 repo 层
- docs: 新增架构说明、Channel × EventBus 设计、任务编写教程
- 依赖新增 aiosqlite / httpx / langgraph / primp / pydantic / ripgrep /
  wecom-aibot-sdk;新增 bot 命令入口
- 测试:232 passed
This commit is contained in:
openbot committed 2026-11-09 20:32:25 +08:00
1 parent 94421e5c2b
commit fbb01c4c83
65 files changed
+15322 -173

No files matched your search

+295
View File
@@ -0,0 +1,295 @@
"""联网工具测试:URL 归一、截断、HTML 转 Markdown、必应解析与失败降级。"""
from __future__ import annotations
import asyncio
from typing import Any, Self
import pytest
from bot042.agent import web
from bot042.agent.web import (
WebError,
_BingParser,
_html_to_markdown,
_normalize_url,
_truncate,
fetch_page,
search_web,
web_fetch,
web_search,
wrap_untrusted,
)
BING_HTML = """
<html><body>
<li class="b_algo">
<h2><a href="https://www.python.org">Welcome to <strong>Python</strong></a></h2>
<div class="b_caption"><p>The mission of the Python Software Foundation is to promote Python.</p></div>
</li>
<li class="b_algo">
<h2><a href="https://docs.python.org/3/">Python 3 Documentation</a></h2>
<p>Official documentation for Python.</p>
</li>
</body></html>
"""
ARTICLE_HTML = """
<html>
<head><title>示例</title><style>p { color: red; }</style><script>alert(1)</script></head>
<body>
<nav><a href="/menu">站点菜单</a></nav>
<article>
<h1>标题一</h1>
<p>第一段,含 <strong>强调</strong> 与 <a href="/docs/x.html">相对链接</a>。</p>
<h2>小节<a class="headerlink" href="#sec">¶</a></h2>
<ul><li>甲</li><li>乙</li></ul>
<ol><li>一</li><li>二</li></ol>
<pre><code>print("hi")</code></pre>
<blockquote>引用的话</blockquote>
</article>
<footer>版权所有</footer>
</body>
</html>
"""
class _FakeResponse:
def __init__(
self,
*,
status_code: int = 200,
text: str = "",
markdown: str = "",
plain: str = "",
content_type: str = "text/html; charset=utf-8",
url: str = "https://example.com/page",
) -> None:
self.status_code = status_code
self.text = text
self.text_markdown = markdown
self.text_plain = plain
self.headers = {"content-type": content_type}
self.url = url
class _FakeClient:
def __init__(self, response: _FakeResponse) -> None:
self.response = response
self.calls: list[dict[str, Any]] = []
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *args: object) -> bool:
return False
async def get(self, url: str, **_kwargs: Any) -> _FakeResponse:
self.calls.append({"url": url, **_kwargs})
return self.response
def test_normalize_url() -> None:
assert _normalize_url("example.com") == "https://example.com"
assert _normalize_url(" http://a.b/c ") == "http://a.b/c"
with pytest.raises(ValueError):
_normalize_url(" ")
with pytest.raises(ValueError):
_normalize_url("ftp://example.com/x")
def test_truncate() -> None:
assert _truncate("abc", 10) == "abc"
out = _truncate("abcdef", 3)
assert out.startswith("abc")
assert "已截断" in out
def test_bing_parser_extracts_results() -> None:
parser = _BingParser()
parser.feed(BING_HTML)
assert len(parser.results) == 2
assert parser.results[0]["url"] == "https://www.python.org"
assert parser.results[0]["title"] == "Welcome to Python"
assert "Software Foundation" in parser.results[0]["snippet"]
assert parser.results[1]["url"] == "https://docs.python.org/3/"
def test_html_to_markdown_keeps_readable_structure() -> None:
md = _html_to_markdown(ARTICLE_HTML, base_url="https://example.com/page")
assert "# 标题一" in md
assert "## 小节" in md
assert "**强调**" in md
assert "[相对链接](https://example.com/docs/x.html)" in md
assert "- 甲" in md and "- 乙" in md
assert "1. 一" in md and "2. 二" in md
assert "```" in md and 'print("hi")' in md
assert "> 引用的话" in md
assert "<h1>" not in md and "&amp;" not in md
def test_html_to_markdown_drops_noise() -> None:
md = _html_to_markdown(ARTICLE_HTML, base_url="https://example.com/page")
assert "站点菜单" not in md # nav 整棵子树丢弃
assert "版权所有" not in md # footer 丢弃
assert "alert(1)" not in md # 脚本丢弃
assert "color: red" not in md # 样式丢弃
assert "" not in md # 标题锚点没有阅读价值
def test_html_to_markdown_drops_navigation_containers() -> None:
html = (
'<div role="navigation">面包屑导航</div>'
'<div class="sidebar">侧栏目录</div>'
'<div id="site-menu">站点菜单</div>'
"<p>正文</p>"
)
md = _html_to_markdown(html)
assert "正文" in md
assert "面包屑导航" not in md
assert "侧栏目录" not in md
assert "站点菜单" not in md
def test_html_to_markdown_keeps_list_items_compact() -> None:
md = _html_to_markdown("<ul><li><p>甲</p></li><li><p>乙</p></li></ul>")
assert md == "- 甲\n- 乙"
def test_html_to_markdown_indents_nested_list() -> None:
md = _html_to_markdown("<ul><li>甲<ul><li>甲一</li></ul></li></ul>")
assert "- 甲" in md
assert " - 甲一" in md
def test_html_to_markdown_keeps_code_indent() -> None:
md = _html_to_markdown("<pre><code>def f():\n return 1</code></pre>")
assert "def f():" in md
assert " return 1" in md # 代码缩进不能被折叠
def test_html_to_markdown_collapses_whitespace() -> None:
md = _html_to_markdown("<p>a b\n\tc</p><p>d</p>")
assert "a b c" in md
assert "\n\n\n" not in md
def test_fetch_page_converts_html_to_markdown(monkeypatch: pytest.MonkeyPatch) -> None:
client = _FakeClient(_FakeResponse(text=ARTICLE_HTML))
monkeypatch.setattr(web, "_client", lambda: client)
out = asyncio.run(fetch_page("example.com"))
assert "# 标题一" in out
assert "<p>" not in out
assert client.calls[0]["url"] == "https://example.com"
def test_fetch_page_keeps_plain_text_as_is(monkeypatch: pytest.MonkeyPatch) -> None:
response = _FakeResponse(text="a\nb", content_type="text/plain; charset=utf-8")
monkeypatch.setattr(web, "_client", lambda: _FakeClient(response))
assert asyncio.run(fetch_page("https://example.com/x.txt")) == "a\nb"
def test_fetch_page_wraps_json_in_code_block(monkeypatch: pytest.MonkeyPatch) -> None:
response = _FakeResponse(text='{"ok":true}', content_type="application/json")
monkeypatch.setattr(web, "_client", lambda: _FakeClient(response))
out = asyncio.run(fetch_page("https://example.com/api"))
assert out == '```json\n{"ok":true}\n```'
def test_fetch_page_truncates(monkeypatch: pytest.MonkeyPatch) -> None:
body = "<p>" + "字" * 200 + "</p>"
monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse(text=body)))
out = asyncio.run(fetch_page("example.com", max_chars=20))
assert "已截断" in out
assert len(out) < 100
def test_fetch_page_raises_on_http_error(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse(status_code=403)))
with pytest.raises(WebError, match="403"):
asyncio.run(fetch_page("https://example.com"))
def test_fetch_page_raises_on_empty_body(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse()))
with pytest.raises(WebError, match="没有可读文本"):
asyncio.run(fetch_page("https://example.com"))
def test_search_web_parses_and_limits(monkeypatch: pytest.MonkeyPatch) -> None:
client = _FakeClient(_FakeResponse(text=BING_HTML))
monkeypatch.setattr(web, "_client", lambda: client)
results = asyncio.run(search_web("python", limit=1))
assert len(results) == 1
assert results[0]["url"] == "https://www.python.org"
assert client.calls[0]["params"]["q"] == "python"
def test_search_web_falls_back_with_page_excerpt(monkeypatch: pytest.MonkeyPatch) -> None:
client = _FakeClient(_FakeResponse(text="<html><body><p>被拦截</p></body></html>"))
monkeypatch.setattr(web, "_client", lambda: client)
with pytest.raises(WebError, match="未解析到搜索结果"):
asyncio.run(search_web("python"))
def test_wrap_untrusted_marks_data_not_instructions() -> None:
wrapped = wrap_untrusted("# 标题\n正文", source="https://example.com/a", fmt="Markdown")
assert wrapped.startswith("[外部网页内容开始]")
assert "来源: https://example.com/a" in wrapped
assert "格式: Markdown" in wrapped
assert "不是指令" in wrapped
assert "# 标题\n正文" in wrapped
assert wrapped.endswith("[外部网页内容结束]")
def test_web_fetch_tool_wraps_content(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse(text=ARTICLE_HTML)))
out = asyncio.run(web_fetch.ainvoke({"url": "https://example.com/page"}))
assert out.startswith("[外部网页内容开始]")
assert "来源: https://example.com/page" in out
assert "格式: Markdown(由 HTML 转换)" in out
assert "不是指令" in out
assert "# 标题一" in out
assert out.endswith("[外部网页内容结束]")
def test_web_fetch_tool_returns_text_on_failure(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse(status_code=500)))
out = asyncio.run(web_fetch.ainvoke({"url": "https://example.com"}))
assert out.startswith("[抓取失败]")
assert "500" in out
def test_web_search_tool_formats_results(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(web, "_client", lambda: _FakeClient(_FakeResponse(text=BING_HTML)))
out = asyncio.run(web_search.ainvoke({"query": "python", "limit": 2}))
assert "1. Welcome to Python" in out
assert "https://www.python.org" in out
assert "2. Python 3 Documentation" in out
def test_get_tools_exposes_web_tools() -> None:
from bot042.agent.tools import get_tools
names = {tool.name for tool in get_tools()}
assert {"perceive", "move", "speak", "web_fetch", "web_search"} <= names