feat: 添加执行日志系统,跟踪各引擎/接口调用耗时和错误信息

新增功能:
- execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情
- 所有搜索引擎自动跟踪每次调用的性能指标
- CLI 搜索后自动显示执行摘要
- health 命令使用 ParallelSearch 检测所有引擎连通性
- 新增 log 命令查看历史执行日志
This commit is contained in:
openbot committed 2026-05-21 14:49:57 +08:00
1 parent f2d8919860
commit 9d97a3a7e8
18 files changed
+1671 -332

No files matched your search

+201
View File
@@ -0,0 +1,201 @@
"""News search engine — multi-source news aggregation."""
from __future__ import annotations
import time
import logging
from urllib.parse import quote_plus
from xml.etree import ElementTree as ET
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
logger = logging.getLogger(__name__)
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
class BingNewsEngine(BaseEngine):
"""Bing News scraping — good for both CN and EN news."""
name = "bing_news"
async def search(self, query: str, max_results: int = 10,
region: str = "cn", **kwargs) -> list[EngineResult]:
t0 = time.time()
base = "cn.bing.com" if region == "cn" else "www.bing.com"
url = f"https://{base}/news/search?q={quote_plus(query)}&setlang=zh-CN"
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=HEADERS)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse(resp.text, max_results)
logger.info(f"[bing_news] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[bing_news] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
for item in soup.select("a.title"):
title = item.get_text(strip=True)
href = item.get("href", "")
# Try to get snippet from parent container
parent = item.find_parent("div", class_=True)
snippet = ""
if parent:
snippet_el = parent.select_one("div.snippet")
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href and href.startswith("http"):
results.append(EngineResult(
title=title, url=href, snippet=snippet,
source="bing_news", category="news",
))
if len(results) >= max_results:
break
return results
class RSSNewsEngine(BaseEngine):
"""RSS-based news — multi-source aggregation."""
name = "rss_news"
# Free RSS feeds, no key needed
FEEDS = {
# Chinese tech/business
"36kr": "https://36kr.com/feed",
"少数派": "https://sspai.com/feed",
"IT之家": "https://www.ithome.com/rss/",
"钛媒体": "https://www.tmtpost.com/rss.xml",
"Solidot": "https://www.solidot.org/index.rss",
"InfoQ CN": "https://www.infoq.cn/feed",
"OSChina": "https://www.oschina.net/news/rss",
# English tech
"HackerNews": "https://hnrss.org/best?count=20",
"TechCrunch": "https://techcrunch.com/feed/",
"ArsTechnica": "https://feeds.arstechnica.com/arstechnica/index",
"TheVerge": "https://www.theverge.com/rss/index.xml",
"MIT Tech Review": "https://www.technologyreview.com/feed/",
"Wired": "https://www.wired.com/feed/rss",
"Lobste.rs": "https://lobste.rs/rss",
# English general
"ChinaDaily": "https://www.chinadaily.com.cn/rss/china_rss.xml",
}
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
t0 = time.time()
all_results = []
import asyncio
tasks = [self._fetch_feed(name, url, query)
for name, url in self.FEEDS.items()]
feeds = await asyncio.gather(*tasks, return_exceptions=True)
for feed_results in feeds:
if isinstance(feed_results, list):
all_results.extend(feed_results)
elapsed = time.time() - t0
logger.info(f"[rss_news] {len(all_results)} results in {elapsed:.2f}s")
return all_results[:max_results]
async def _fetch_feed(self, name: str, url: str, query: str) -> list[EngineResult]:
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=HEADERS)
resp.raise_for_status()
return self._parse_feed(name, resp.text, query)
except Exception as e:
logger.warning(f"[rss_news:{name}] failed: {e}")
return []
def _parse_feed(self, name: str, xml_text: str, query: str) -> list[EngineResult]:
try:
root = ET.fromstring(xml_text)
except ET.ParseError:
return []
results = []
# RSS 2.0: //channel/item, Atom: //entry
ns = {"atom": "http://www.w3.org/2005/Atom"}
items = root.findall(".//item") or root.findall(".//atom:entry", ns)
q_lower = query.lower()
q_words = [w for w in q_lower.split() if len(w) > 1]
for item in items[:50]:
title = (item.findtext("title") or
item.findtext("atom:title", namespaces=ns) or "")
link = (item.findtext("link") or "")
if not link:
link_el = item.find("atom:link", ns)
link = link_el.get("href", "") if link_el is not None else ""
desc = (item.findtext("description") or
item.findtext("atom:summary", namespaces=ns) or "")
# Clean HTML tags from description
import re
desc = re.sub(r'<[^>]+>', '', desc)
# Keyword scoring (not strict filter)
text_lower = f"{title} {desc}".lower()
if q_words:
match_count = sum(1 for w in q_words if w in text_lower)
if match_count == 0:
continue # Skip completely irrelevant items
results.append(EngineResult(
title=title.strip(),
url=link.strip(),
snippet=desc[:300].strip(),
source=f"rss_{name}",
category="news",
))
return results
class NewsSearch(BaseEngine):
"""Unified news search — Bing News + RSS in parallel."""
name = "news"
async def search(self, query: str, max_results: int = 10,
region: str = "cn", **kwargs) -> list[EngineResult]:
import asyncio
bing = BingNewsEngine(timeout=self.timeout)
rss = RSSNewsEngine(timeout=8.0) # RSS feeds may be slow
bing_results, rss_results = await asyncio.gather(
bing.search(query, max_results=max_results, region=region),
rss.search(query, max_results=max_results),
return_exceptions=True,
)
all_results = []
if isinstance(bing_results, list):
all_results.extend(bing_results)
if isinstance(rss_results, list):
all_results.extend(rss_results)
# Deduplicate by URL
seen = set()
deduped = []
for r in all_results:
if r.url not in seen:
seen.add(r.url)
deduped.append(r)
return deduped[:max_results]