"""Parallel search orchestrator — runs multiple engines concurrently.""" from __future__ import annotations import asyncio import time import logging from dataclasses import dataclass, field from typing import Any from scripts.core.base import BaseEngine, SearchResult from scripts.engines.bing_scraper import BingScraper from scripts.engines.sogou_scraper import SogouScraper from scripts.engines.baidu_scraper import BaiduScraper from scripts.engines.search360_scraper import Search360Scraper from scripts.engines.news_engine import NewsSearch from scripts.engines.academic_engine import AcademicSearch from scripts.engines.github_engine import GitHubEngine from scripts.core.execution_log import execution_log logger = logging.getLogger(__name__) @dataclass class SearchStats: """Search statistics.""" total_engines: int = 0 successful_engines: int = 0 failed_engines: list[str] = field(default_factory=list) total_results: int = 0 deduplicated: int = 0 duration_ms: int = 0 engine_timings: dict[str, float] = field(default_factory=dict) class ParallelSearch: """Run multiple engines in parallel and merge results.""" # Default engine sets GENERAL_ENGINES = ["bing", "sogou", "baidu", "360"] VERTICAL_ENGINES = ["news", "academic", "github"] def __init__(self, timeout: float = 10.0): self.timeout = timeout self._engines: dict[str, BaseEngine] = { "bing": BingScraper(timeout=timeout), "sogou": SogouScraper(timeout=timeout), "baidu": BaiduScraper(timeout=timeout), "360": Search360Scraper(timeout=timeout), "news": NewsSearch(timeout=timeout), "academic": AcademicSearch(timeout=timeout), "github": GitHubEngine(timeout=timeout), } async def search( self, query: str, engines: list[str] | None = None, max_results: int = 10, region: str = "cn", **kwargs, ) -> tuple[list[SearchResult], SearchStats]: """ Search across specified engines in parallel. Args: query: Search query engines: List of engine names to use (default: all general) max_results: Max results per engine region: 'cn' or 'global' **kwargs: Extra params passed to engines Returns: (results, stats) """ if engines is None: engines = self.GENERAL_ENGINES.copy() t0 = time.time() stats = SearchStats(total_engines=len(engines)) # Build tasks tasks = {} for name in engines: engine = self._engines.get(name) if engine is None: logger.warning(f"Unknown engine: {name}") continue kwargs_extra = dict(kwargs) if name in ("bing", "bing_news"): kwargs_extra["region"] = region tasks[name] = self._run_engine(engine, query, max_results, kwargs_extra) # Run all in parallel results_map = await asyncio.gather( *tasks.values(), return_exceptions=True ) all_results = [] for name, result in zip(tasks.keys(), results_map): if isinstance(result, Exception): stats.failed_engines.append(name) logger.warning(f"[{name}] exception: {result}") # 记录异常到 execution_log execution_log.record(execution_log._logs[-1].__class__( engine=name, operation="search", success=False, error=str(result), error_type=type(result).__name__, duration_ms=int((time.time() - t0) * 1000), )) elif isinstance(result, tuple): engine_results, elapsed = result stats.engine_timings[name] = elapsed if engine_results: stats.successful_engines += 1 all_results.extend(engine_results) else: stats.failed_engines.append(name) else: stats.failed_engines.append(name) # Deduplicate by URL seen_urls = set() deduped = [] for r in all_results: key = r.url or r.title # fallback to title if no URL if key not in seen_urls: seen_urls.add(key) deduped.append(r) stats.total_results = len(all_results) stats.deduplicated = len(deduped) stats.duration_ms = int((time.time() - t0) * 1000) return deduped, stats async def _run_engine( self, engine: BaseEngine, query: str, max_results: int, kwargs: dict ) -> tuple[list[SearchResult], float]: """Run a single engine with timing.""" with execution_log.track(engine.name, "search", query=query) as ctx: t0 = time.time() try: results = await engine.search( query, max_results=max_results, **kwargs ) elapsed = time.time() - t0 ctx.result_count = len(results) return results, elapsed except Exception as e: elapsed = time.time() - t0 ctx.success = False ctx.error = str(e) ctx.error_type = type(e).__name__ raise async def search_vertical( self, query: str, vertical: str, max_results: int = 10, **kwargs, ) -> tuple[list[SearchResult], SearchStats]: """Search a specific vertical (news/academic/github).""" if vertical == "github": # GitHub supports sub-types search_type = kwargs.pop("search_type", "repositories") engine = self._engines["github"] with execution_log.track("github", "search", search_type=search_type) as ctx: t0 = time.time() results = await engine.search( query, max_results=max_results, search_type=search_type, **kwargs ) ctx.result_count = len(results) stats = SearchStats( total_engines=1, successful_engines=1 if results else 0, total_results=len(results), duration_ms=int((time.time() - t0) * 1000), ) return results, stats else: return await self.search( query, engines=[vertical], max_results=max_results, **kwargs ) async def health(self) -> dict[str, dict]: """Check health of all engines.""" results = {} for name, engine in self._engines.items(): try: results[name] = await engine.health() except Exception as e: results[name] = {"name": name, "status": "error", "error": str(e)} return results