feat: 添加执行日志系统,跟踪各引擎/接口调用耗时和错误信息
新增功能: - execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情 - 所有搜索引擎自动跟踪每次调用的性能指标 - CLI 搜索后自动显示执行摘要 - health 命令使用 ParallelSearch 检测所有引擎连通性 - 新增 log 命令查看历史执行日志
This commit is contained in:
1 parent
f2d8919860
commit
9d97a3a7e8
18 files changed
+1671
-332
No files matched your search
@@ -0,0 +1,199 @@
|
||||
"""Parallel search orchestrator — runs multiple engines concurrently."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.engines.bing_scraper import BingScraper
|
||||
from scripts.engines.sogou_scraper import SogouScraper
|
||||
from scripts.engines.baidu_scraper import BaiduScraper
|
||||
from scripts.engines.search360_scraper import Search360Scraper
|
||||
from scripts.engines.news_engine import NewsSearch
|
||||
from scripts.engines.academic_engine import AcademicSearch
|
||||
from scripts.engines.github_engine import GitHubEngine
|
||||
from scripts.core.execution_log import execution_log
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchStats:
|
||||
"""Search statistics."""
|
||||
total_engines: int = 0
|
||||
successful_engines: int = 0
|
||||
failed_engines: list[str] = field(default_factory=list)
|
||||
total_results: int = 0
|
||||
deduplicated: int = 0
|
||||
duration_ms: int = 0
|
||||
engine_timings: dict[str, float] = field(default_factory=dict)
|
||||
|
||||
|
||||
class ParallelSearch:
|
||||
"""Run multiple engines in parallel and merge results."""
|
||||
|
||||
# Default engine sets
|
||||
GENERAL_ENGINES = ["bing", "sogou", "baidu", "360"]
|
||||
VERTICAL_ENGINES = ["news", "academic", "github"]
|
||||
|
||||
def __init__(self, timeout: float = 10.0):
|
||||
self.timeout = timeout
|
||||
self._engines: dict[str, BaseEngine] = {
|
||||
"bing": BingScraper(timeout=timeout),
|
||||
"sogou": SogouScraper(timeout=timeout),
|
||||
"baidu": BaiduScraper(timeout=timeout),
|
||||
"360": Search360Scraper(timeout=timeout),
|
||||
"news": NewsSearch(timeout=timeout),
|
||||
"academic": AcademicSearch(timeout=timeout),
|
||||
"github": GitHubEngine(timeout=timeout),
|
||||
}
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
engines: list[str] | None = None,
|
||||
max_results: int = 10,
|
||||
region: str = "cn",
|
||||
**kwargs,
|
||||
) -> tuple[list[EngineResult], SearchStats]:
|
||||
"""
|
||||
Search across specified engines in parallel.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
engines: List of engine names to use (default: all general)
|
||||
max_results: Max results per engine
|
||||
region: 'cn' or 'global'
|
||||
**kwargs: Extra params passed to engines
|
||||
|
||||
Returns:
|
||||
(results, stats)
|
||||
"""
|
||||
if engines is None:
|
||||
engines = self.GENERAL_ENGINES.copy()
|
||||
|
||||
t0 = time.time()
|
||||
stats = SearchStats(total_engines=len(engines))
|
||||
|
||||
# Build tasks
|
||||
tasks = {}
|
||||
for name in engines:
|
||||
engine = self._engines.get(name)
|
||||
if engine is None:
|
||||
logger.warning(f"Unknown engine: {name}")
|
||||
continue
|
||||
kwargs_extra = dict(kwargs)
|
||||
if name in ("bing", "bing_news"):
|
||||
kwargs_extra["region"] = region
|
||||
tasks[name] = self._run_engine(engine, query, max_results, kwargs_extra)
|
||||
|
||||
# Run all in parallel
|
||||
results_map = await asyncio.gather(
|
||||
*tasks.values(), return_exceptions=True
|
||||
)
|
||||
|
||||
all_results = []
|
||||
for name, result in zip(tasks.keys(), results_map):
|
||||
if isinstance(result, Exception):
|
||||
stats.failed_engines.append(name)
|
||||
logger.warning(f"[{name}] exception: {result}")
|
||||
# 记录异常到 execution_log
|
||||
execution_log.record(execution_log._logs[-1].__class__(
|
||||
engine=name,
|
||||
operation="search",
|
||||
success=False,
|
||||
error=str(result),
|
||||
error_type=type(result).__name__,
|
||||
duration_ms=int((time.time() - t0) * 1000),
|
||||
))
|
||||
elif isinstance(result, tuple):
|
||||
engine_results, elapsed = result
|
||||
stats.engine_timings[name] = elapsed
|
||||
if engine_results:
|
||||
stats.successful_engines += 1
|
||||
all_results.extend(engine_results)
|
||||
else:
|
||||
stats.failed_engines.append(name)
|
||||
else:
|
||||
stats.failed_engines.append(name)
|
||||
|
||||
# Deduplicate by URL
|
||||
seen_urls = set()
|
||||
deduped = []
|
||||
for r in all_results:
|
||||
key = r.url or r.title # fallback to title if no URL
|
||||
if key not in seen_urls:
|
||||
seen_urls.add(key)
|
||||
deduped.append(r)
|
||||
|
||||
stats.total_results = len(all_results)
|
||||
stats.deduplicated = len(deduped)
|
||||
stats.duration_ms = int((time.time() - t0) * 1000)
|
||||
|
||||
return deduped, stats
|
||||
|
||||
async def _run_engine(
|
||||
self, engine: BaseEngine, query: str, max_results: int, kwargs: dict
|
||||
) -> tuple[list[EngineResult], float]:
|
||||
"""Run a single engine with timing."""
|
||||
with execution_log.track(engine.name, "search", query=query) as ctx:
|
||||
t0 = time.time()
|
||||
try:
|
||||
results = await engine.search(
|
||||
query, max_results=max_results, **kwargs
|
||||
)
|
||||
elapsed = time.time() - t0
|
||||
ctx.result_count = len(results)
|
||||
return results, elapsed
|
||||
except Exception as e:
|
||||
elapsed = time.time() - t0
|
||||
ctx.success = False
|
||||
ctx.error = str(e)
|
||||
ctx.error_type = type(e).__name__
|
||||
raise
|
||||
|
||||
async def search_vertical(
|
||||
self,
|
||||
query: str,
|
||||
vertical: str,
|
||||
max_results: int = 10,
|
||||
**kwargs,
|
||||
) -> tuple[list[EngineResult], SearchStats]:
|
||||
"""Search a specific vertical (news/academic/github)."""
|
||||
if vertical == "github":
|
||||
# GitHub supports sub-types
|
||||
search_type = kwargs.pop("search_type", "repositories")
|
||||
engine = self._engines["github"]
|
||||
with execution_log.track("github", "search",
|
||||
search_type=search_type) as ctx:
|
||||
t0 = time.time()
|
||||
results = await engine.search(
|
||||
query, max_results=max_results,
|
||||
search_type=search_type, **kwargs
|
||||
)
|
||||
ctx.result_count = len(results)
|
||||
stats = SearchStats(
|
||||
total_engines=1,
|
||||
successful_engines=1 if results else 0,
|
||||
total_results=len(results),
|
||||
duration_ms=int((time.time() - t0) * 1000),
|
||||
)
|
||||
return results, stats
|
||||
else:
|
||||
return await self.search(
|
||||
query, engines=[vertical], max_results=max_results, **kwargs
|
||||
)
|
||||
|
||||
async def health(self) -> dict[str, dict]:
|
||||
"""Check health of all engines."""
|
||||
results = {}
|
||||
for name, engine in self._engines.items():
|
||||
try:
|
||||
results[name] = await engine.health()
|
||||
except Exception as e:
|
||||
results[name] = {"name": name, "status": "error", "error": str(e)}
|
||||
return results
|
||||
Reference in new issue
Block a user