新增功能: - execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情 - 所有搜索引擎自动跟踪每次调用的性能指标 - CLI 搜索后自动显示执行摘要 - health 命令使用 ParallelSearch 检测所有引擎连通性 - 新增 log 命令查看历史执行日志
200 lines
7.0 KiB
Python
200 lines
7.0 KiB
Python
"""Parallel search orchestrator — runs multiple engines concurrently."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from scripts.engines.base import BaseEngine, EngineResult
|
|
from scripts.engines.bing_scraper import BingScraper
|
|
from scripts.engines.sogou_scraper import SogouScraper
|
|
from scripts.engines.baidu_scraper import BaiduScraper
|
|
from scripts.engines.search360_scraper import Search360Scraper
|
|
from scripts.engines.news_engine import NewsSearch
|
|
from scripts.engines.academic_engine import AcademicSearch
|
|
from scripts.engines.github_engine import GitHubEngine
|
|
from scripts.core.execution_log import execution_log
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class SearchStats:
|
|
"""Search statistics."""
|
|
total_engines: int = 0
|
|
successful_engines: int = 0
|
|
failed_engines: list[str] = field(default_factory=list)
|
|
total_results: int = 0
|
|
deduplicated: int = 0
|
|
duration_ms: int = 0
|
|
engine_timings: dict[str, float] = field(default_factory=dict)
|
|
|
|
|
|
class ParallelSearch:
|
|
"""Run multiple engines in parallel and merge results."""
|
|
|
|
# Default engine sets
|
|
GENERAL_ENGINES = ["bing", "sogou", "baidu", "360"]
|
|
VERTICAL_ENGINES = ["news", "academic", "github"]
|
|
|
|
def __init__(self, timeout: float = 10.0):
|
|
self.timeout = timeout
|
|
self._engines: dict[str, BaseEngine] = {
|
|
"bing": BingScraper(timeout=timeout),
|
|
"sogou": SogouScraper(timeout=timeout),
|
|
"baidu": BaiduScraper(timeout=timeout),
|
|
"360": Search360Scraper(timeout=timeout),
|
|
"news": NewsSearch(timeout=timeout),
|
|
"academic": AcademicSearch(timeout=timeout),
|
|
"github": GitHubEngine(timeout=timeout),
|
|
}
|
|
|
|
async def search(
|
|
self,
|
|
query: str,
|
|
engines: list[str] | None = None,
|
|
max_results: int = 10,
|
|
region: str = "cn",
|
|
**kwargs,
|
|
) -> tuple[list[EngineResult], SearchStats]:
|
|
"""
|
|
Search across specified engines in parallel.
|
|
|
|
Args:
|
|
query: Search query
|
|
engines: List of engine names to use (default: all general)
|
|
max_results: Max results per engine
|
|
region: 'cn' or 'global'
|
|
**kwargs: Extra params passed to engines
|
|
|
|
Returns:
|
|
(results, stats)
|
|
"""
|
|
if engines is None:
|
|
engines = self.GENERAL_ENGINES.copy()
|
|
|
|
t0 = time.time()
|
|
stats = SearchStats(total_engines=len(engines))
|
|
|
|
# Build tasks
|
|
tasks = {}
|
|
for name in engines:
|
|
engine = self._engines.get(name)
|
|
if engine is None:
|
|
logger.warning(f"Unknown engine: {name}")
|
|
continue
|
|
kwargs_extra = dict(kwargs)
|
|
if name in ("bing", "bing_news"):
|
|
kwargs_extra["region"] = region
|
|
tasks[name] = self._run_engine(engine, query, max_results, kwargs_extra)
|
|
|
|
# Run all in parallel
|
|
results_map = await asyncio.gather(
|
|
*tasks.values(), return_exceptions=True
|
|
)
|
|
|
|
all_results = []
|
|
for name, result in zip(tasks.keys(), results_map):
|
|
if isinstance(result, Exception):
|
|
stats.failed_engines.append(name)
|
|
logger.warning(f"[{name}] exception: {result}")
|
|
# 记录异常到 execution_log
|
|
execution_log.record(execution_log._logs[-1].__class__(
|
|
engine=name,
|
|
operation="search",
|
|
success=False,
|
|
error=str(result),
|
|
error_type=type(result).__name__,
|
|
duration_ms=int((time.time() - t0) * 1000),
|
|
))
|
|
elif isinstance(result, tuple):
|
|
engine_results, elapsed = result
|
|
stats.engine_timings[name] = elapsed
|
|
if engine_results:
|
|
stats.successful_engines += 1
|
|
all_results.extend(engine_results)
|
|
else:
|
|
stats.failed_engines.append(name)
|
|
else:
|
|
stats.failed_engines.append(name)
|
|
|
|
# Deduplicate by URL
|
|
seen_urls = set()
|
|
deduped = []
|
|
for r in all_results:
|
|
key = r.url or r.title # fallback to title if no URL
|
|
if key not in seen_urls:
|
|
seen_urls.add(key)
|
|
deduped.append(r)
|
|
|
|
stats.total_results = len(all_results)
|
|
stats.deduplicated = len(deduped)
|
|
stats.duration_ms = int((time.time() - t0) * 1000)
|
|
|
|
return deduped, stats
|
|
|
|
async def _run_engine(
|
|
self, engine: BaseEngine, query: str, max_results: int, kwargs: dict
|
|
) -> tuple[list[EngineResult], float]:
|
|
"""Run a single engine with timing."""
|
|
with execution_log.track(engine.name, "search", query=query) as ctx:
|
|
t0 = time.time()
|
|
try:
|
|
results = await engine.search(
|
|
query, max_results=max_results, **kwargs
|
|
)
|
|
elapsed = time.time() - t0
|
|
ctx.result_count = len(results)
|
|
return results, elapsed
|
|
except Exception as e:
|
|
elapsed = time.time() - t0
|
|
ctx.success = False
|
|
ctx.error = str(e)
|
|
ctx.error_type = type(e).__name__
|
|
raise
|
|
|
|
async def search_vertical(
|
|
self,
|
|
query: str,
|
|
vertical: str,
|
|
max_results: int = 10,
|
|
**kwargs,
|
|
) -> tuple[list[EngineResult], SearchStats]:
|
|
"""Search a specific vertical (news/academic/github)."""
|
|
if vertical == "github":
|
|
# GitHub supports sub-types
|
|
search_type = kwargs.pop("search_type", "repositories")
|
|
engine = self._engines["github"]
|
|
with execution_log.track("github", "search",
|
|
search_type=search_type) as ctx:
|
|
t0 = time.time()
|
|
results = await engine.search(
|
|
query, max_results=max_results,
|
|
search_type=search_type, **kwargs
|
|
)
|
|
ctx.result_count = len(results)
|
|
stats = SearchStats(
|
|
total_engines=1,
|
|
successful_engines=1 if results else 0,
|
|
total_results=len(results),
|
|
duration_ms=int((time.time() - t0) * 1000),
|
|
)
|
|
return results, stats
|
|
else:
|
|
return await self.search(
|
|
query, engines=[vertical], max_results=max_results, **kwargs
|
|
)
|
|
|
|
async def health(self) -> dict[str, dict]:
|
|
"""Check health of all engines."""
|
|
results = {}
|
|
for name, engine in self._engines.items():
|
|
try:
|
|
results[name] = await engine.health()
|
|
except Exception as e:
|
|
results[name] = {"name": name, "status": "error", "error": str(e)}
|
|
return results
|