""" Unified multi-source search — Python API. Provides a clean async interface for programmatic use. All engines run in parallel with configurable concurrency. Usage: from scripts.unified_search import UnifiedSearch us = UnifiedSearch() resp = await us.search("quantum computing", sources=["bing", "arxiv"], max_results=5) for r in resp.results: print(f"[{r.rank}] {r.title}") await us.close() """ from __future__ import annotations import asyncio import time from typing import Any, Optional from scripts.core.base import BaseEngine, SearchResult from scripts.engines.bing_scraper import BingScraper from scripts.engines.sogou_scraper import SogouScraper from scripts.engines.baidu_scraper import BaiduScraper from scripts.engines.search360_scraper import Search360Scraper from scripts.engines.parser_engines import DuckDuckGoParser, BraveParser from scripts.engines.news_engine import NewsSearch from scripts.engines.academic_engine import AcademicSearch from scripts.engines.github_engine import GitHubEngine from scripts.core.aggregator import ResultAggregator from scripts.models import SearchResponse, SourceRef, SearchMetadata, EngineTiming # Default engine registry _ALL_ENGINES: dict[str, BaseEngine] = { "bing": BingScraper(), "sogou": SogouScraper(), "baidu": BaiduScraper(), "360": Search360Scraper(), "duckduckgo": DuckDuckGoParser(), "brave": BraveParser(), "news": NewsSearch(), "academic": AcademicSearch(), "github": GitHubEngine(), } class UnifiedSearch: """Unified multi-source search — Python API entry point.""" def __init__(self, timeout: float = 15.0, max_concurrent: int = 5): self.timeout = timeout self.max_concurrent = max_concurrent self._engines: dict[str, BaseEngine] = { name: eng.__class__(timeout=timeout) for name, eng in _ALL_ENGINES.items() } async def search( self, query: str, sources: Optional[list[str]] = None, max_results: int = 10, region: str = "cn", **kwargs, ) -> SearchResponse: """ Search across multiple sources in parallel. Args: query: Search query string sources: List of engine names to use (default: all) max_results: Max results to return (deduped) region: "cn" or "global" **kwargs: Extra params passed to engines Returns: SearchResponse with results, sources, and metadata """ t0 = time.time() # Select engines if sources: selected = {k: v for k, v in self._engines.items() if k in sources} else: selected = dict(self._engines) # Build region kwarg eng_kwargs = {"region": region} if region else {} # Run all engines concurrently with semaphore semaphore = asyncio.Semaphore(self.max_concurrent) async def run_engine(name: str, eng: BaseEngine) -> tuple[str, list[SearchResult], float]: async with semaphore: start = time.time() try: results = await asyncio.wait_for( eng.search(query, max_results=max_results, **eng_kwargs), timeout=eng.timeout + 2.0, ) elapsed = time.time() - start return name, list(results), elapsed except asyncio.TimeoutError: return name, [], time.time() - start except Exception: return name, [], time.time() - start tasks = [run_engine(n, e) for n, e in selected.items()] done = await asyncio.gather(*tasks, return_exceptions=True) # Collect results and timings all_results: list[SearchResult] = [] engine_timings: list[EngineTiming] = [] engines_used: set[str] = set() for item in done: if isinstance(item, tuple): name, results, elapsed = item all_results.extend(results) if results: engines_used.add(name) engine_timings.append(EngineTiming( engine_name=name, success=bool(results), duration_ms=int(elapsed * 1000), result_count=len(results), )) elif isinstance(item, Exception): # Exception in gather — log but don't crash pass # Deduplicate and rank aggregator = ResultAggregator() merged = aggregator.merge_and_rank(all_results, max_results=max_results) elapsed_ms = int((time.time() - t0) * 1000) # Build SourceRef list sources_list = [ SourceRef( index=r.rank, title=r.title, url=r.url, source=r.source, ) for r in merged ] metadata = SearchMetadata( total_engines=len(selected), total_raw_results=len(all_results), engines_used=list(engines_used), duration_ms=elapsed_ms, engine_timings=engine_timings, ) return SearchResponse( query=query, results=merged, sources=sources_list, metadata=metadata, ) async def health(self) -> dict[str, Any]: """Check connectivity of all engines.""" results = {} for name, eng in self._engines.items(): try: health = await asyncio.wait_for(eng.health(), timeout=5.0) results[name] = health except Exception as e: results[name] = {"name": name, "status": "fail", "error": str(e)} return results async def close(self): """Cleanup (no-op for current implementation).""" pass