Files
openbot 9d97a3a7e8 feat: 添加执行日志系统,跟踪各引擎/接口调用耗时和错误信息
新增功能:
- execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情
- 所有搜索引擎自动跟踪每次调用的性能指标
- CLI 搜索后自动显示执行摘要
- health 命令使用 ParallelSearch 检测所有引擎连通性
- 新增 log 命令查看历史执行日志
2026-05-21 14:49:57 +08:00

200 lines
7.0 KiB
Python

"""Parallel search orchestrator — runs multiple engines concurrently."""
from __future__ import annotations
import asyncio
import time
import logging
from dataclasses import dataclass, field
from typing import Any
from scripts.engines.base import BaseEngine, EngineResult
from scripts.engines.bing_scraper import BingScraper
from scripts.engines.sogou_scraper import SogouScraper
from scripts.engines.baidu_scraper import BaiduScraper
from scripts.engines.search360_scraper import Search360Scraper
from scripts.engines.news_engine import NewsSearch
from scripts.engines.academic_engine import AcademicSearch
from scripts.engines.github_engine import GitHubEngine
from scripts.core.execution_log import execution_log
logger = logging.getLogger(__name__)
@dataclass
class SearchStats:
"""Search statistics."""
total_engines: int = 0
successful_engines: int = 0
failed_engines: list[str] = field(default_factory=list)
total_results: int = 0
deduplicated: int = 0
duration_ms: int = 0
engine_timings: dict[str, float] = field(default_factory=dict)
class ParallelSearch:
"""Run multiple engines in parallel and merge results."""
# Default engine sets
GENERAL_ENGINES = ["bing", "sogou", "baidu", "360"]
VERTICAL_ENGINES = ["news", "academic", "github"]
def __init__(self, timeout: float = 10.0):
self.timeout = timeout
self._engines: dict[str, BaseEngine] = {
"bing": BingScraper(timeout=timeout),
"sogou": SogouScraper(timeout=timeout),
"baidu": BaiduScraper(timeout=timeout),
"360": Search360Scraper(timeout=timeout),
"news": NewsSearch(timeout=timeout),
"academic": AcademicSearch(timeout=timeout),
"github": GitHubEngine(timeout=timeout),
}
async def search(
self,
query: str,
engines: list[str] | None = None,
max_results: int = 10,
region: str = "cn",
**kwargs,
) -> tuple[list[EngineResult], SearchStats]:
"""
Search across specified engines in parallel.
Args:
query: Search query
engines: List of engine names to use (default: all general)
max_results: Max results per engine
region: 'cn' or 'global'
**kwargs: Extra params passed to engines
Returns:
(results, stats)
"""
if engines is None:
engines = self.GENERAL_ENGINES.copy()
t0 = time.time()
stats = SearchStats(total_engines=len(engines))
# Build tasks
tasks = {}
for name in engines:
engine = self._engines.get(name)
if engine is None:
logger.warning(f"Unknown engine: {name}")
continue
kwargs_extra = dict(kwargs)
if name in ("bing", "bing_news"):
kwargs_extra["region"] = region
tasks[name] = self._run_engine(engine, query, max_results, kwargs_extra)
# Run all in parallel
results_map = await asyncio.gather(
*tasks.values(), return_exceptions=True
)
all_results = []
for name, result in zip(tasks.keys(), results_map):
if isinstance(result, Exception):
stats.failed_engines.append(name)
logger.warning(f"[{name}] exception: {result}")
# 记录异常到 execution_log
execution_log.record(execution_log._logs[-1].__class__(
engine=name,
operation="search",
success=False,
error=str(result),
error_type=type(result).__name__,
duration_ms=int((time.time() - t0) * 1000),
))
elif isinstance(result, tuple):
engine_results, elapsed = result
stats.engine_timings[name] = elapsed
if engine_results:
stats.successful_engines += 1
all_results.extend(engine_results)
else:
stats.failed_engines.append(name)
else:
stats.failed_engines.append(name)
# Deduplicate by URL
seen_urls = set()
deduped = []
for r in all_results:
key = r.url or r.title # fallback to title if no URL
if key not in seen_urls:
seen_urls.add(key)
deduped.append(r)
stats.total_results = len(all_results)
stats.deduplicated = len(deduped)
stats.duration_ms = int((time.time() - t0) * 1000)
return deduped, stats
async def _run_engine(
self, engine: BaseEngine, query: str, max_results: int, kwargs: dict
) -> tuple[list[EngineResult], float]:
"""Run a single engine with timing."""
with execution_log.track(engine.name, "search", query=query) as ctx:
t0 = time.time()
try:
results = await engine.search(
query, max_results=max_results, **kwargs
)
elapsed = time.time() - t0
ctx.result_count = len(results)
return results, elapsed
except Exception as e:
elapsed = time.time() - t0
ctx.success = False
ctx.error = str(e)
ctx.error_type = type(e).__name__
raise
async def search_vertical(
self,
query: str,
vertical: str,
max_results: int = 10,
**kwargs,
) -> tuple[list[EngineResult], SearchStats]:
"""Search a specific vertical (news/academic/github)."""
if vertical == "github":
# GitHub supports sub-types
search_type = kwargs.pop("search_type", "repositories")
engine = self._engines["github"]
with execution_log.track("github", "search",
search_type=search_type) as ctx:
t0 = time.time()
results = await engine.search(
query, max_results=max_results,
search_type=search_type, **kwargs
)
ctx.result_count = len(results)
stats = SearchStats(
total_engines=1,
successful_engines=1 if results else 0,
total_results=len(results),
duration_ms=int((time.time() - t0) * 1000),
)
return results, stats
else:
return await self.search(
query, engines=[vertical], max_results=max_results, **kwargs
)
async def health(self) -> dict[str, dict]:
"""Check health of all engines."""
results = {}
for name, engine in self._engines.items():
try:
results[name] = await engine.health()
except Exception as e:
results[name] = {"name": name, "status": "error", "error": str(e)}
return results