feat: 添加执行日志系统,跟踪各引擎/接口调用耗时和错误信息
新增功能: - execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情 - 所有搜索引擎自动跟踪每次调用的性能指标 - CLI 搜索后自动显示执行摘要 - health 命令使用 ParallelSearch 检测所有引擎连通性 - 新增 log 命令查看历史执行日志
This commit is contained in:
1 parent
f2d8919860
commit
9d97a3a7e8
18 files changed
+1671
-332
No files matched your search
@@ -0,0 +1,69 @@
|
||||
"""Baidu HTML scraping engine — free, fastest in China, best Chinese results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import logging
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
|
||||
|
||||
class BaiduScraper(BaseEngine):
|
||||
"""Baidu web scraping — fastest in China, best Chinese content."""
|
||||
|
||||
name = "baidu"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
t0 = time.time()
|
||||
url = f"https://www.baidu.com/s?wd={quote_plus(query)}&rn={max_results}"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
logger.info(f"[baidu] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[baidu] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
for item in soup.select("div.result, div.c-container"):
|
||||
title_el = item.select_one("h3 a")
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
href = title_el.get("href", "")
|
||||
snippet_el = item.select_one(
|
||||
"span.content-right_8Zs40, div.c-abstract, "
|
||||
"span[class*='content'], div[class*='abstract']"
|
||||
)
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href:
|
||||
results.append(EngineResult(
|
||||
title=title, url=href, snippet=snippet, source="baidu",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
break
|
||||
|
||||
return results
|
||||
Reference in new issue
Block a user