Files
web-search-skills/scripts/engines/parallel_search.py
T
openbot-coder c077a7ae40 refactor: cleanup dead code, fix model confusion, streamline CLI
- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/)
- Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py)
- scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py)
- scripts/engines/base.py: re-exports from core.base for compatibility
- scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py)
- scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py)
- scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources)
- scripts/unified_search.py: clean async Python API (was a stub)
- ws.py: lightweight launcher
- README.md + SKILL.md: updated to reflect actual 15+ engines
- All EngineResult → SearchResult
- All engines return list[SearchResult]
2026-05-30 16:25:41 +08:00

200 lines
7.0 KiB
Python

"""Parallel search orchestrator — runs multiple engines concurrently."""
from __future__ import annotations
import asyncio
import time
import logging
from dataclasses import dataclass, field
from typing import Any
from scripts.core.base import BaseEngine, SearchResult
from scripts.engines.bing_scraper import BingScraper
from scripts.engines.sogou_scraper import SogouScraper
from scripts.engines.baidu_scraper import BaiduScraper
from scripts.engines.search360_scraper import Search360Scraper
from scripts.engines.news_engine import NewsSearch
from scripts.engines.academic_engine import AcademicSearch
from scripts.engines.github_engine import GitHubEngine
from scripts.core.execution_log import execution_log
logger = logging.getLogger(__name__)
@dataclass
class SearchStats:
"""Search statistics."""
total_engines: int = 0
successful_engines: int = 0
failed_engines: list[str] = field(default_factory=list)
total_results: int = 0
deduplicated: int = 0
duration_ms: int = 0
engine_timings: dict[str, float] = field(default_factory=dict)
class ParallelSearch:
"""Run multiple engines in parallel and merge results."""
# Default engine sets
GENERAL_ENGINES = ["bing", "sogou", "baidu", "360"]
VERTICAL_ENGINES = ["news", "academic", "github"]
def __init__(self, timeout: float = 10.0):
self.timeout = timeout
self._engines: dict[str, BaseEngine] = {
"bing": BingScraper(timeout=timeout),
"sogou": SogouScraper(timeout=timeout),
"baidu": BaiduScraper(timeout=timeout),
"360": Search360Scraper(timeout=timeout),
"news": NewsSearch(timeout=timeout),
"academic": AcademicSearch(timeout=timeout),
"github": GitHubEngine(timeout=timeout),
}
async def search(
self,
query: str,
engines: list[str] | None = None,
max_results: int = 10,
region: str = "cn",
**kwargs,
) -> tuple[list[SearchResult], SearchStats]:
"""
Search across specified engines in parallel.
Args:
query: Search query
engines: List of engine names to use (default: all general)
max_results: Max results per engine
region: 'cn' or 'global'
**kwargs: Extra params passed to engines
Returns:
(results, stats)
"""
if engines is None:
engines = self.GENERAL_ENGINES.copy()
t0 = time.time()
stats = SearchStats(total_engines=len(engines))
# Build tasks
tasks = {}
for name in engines:
engine = self._engines.get(name)
if engine is None:
logger.warning(f"Unknown engine: {name}")
continue
kwargs_extra = dict(kwargs)
if name in ("bing", "bing_news"):
kwargs_extra["region"] = region
tasks[name] = self._run_engine(engine, query, max_results, kwargs_extra)
# Run all in parallel
results_map = await asyncio.gather(
*tasks.values(), return_exceptions=True
)
all_results = []
for name, result in zip(tasks.keys(), results_map):
if isinstance(result, Exception):
stats.failed_engines.append(name)
logger.warning(f"[{name}] exception: {result}")
# 记录异常到 execution_log
execution_log.record(execution_log._logs[-1].__class__(
engine=name,
operation="search",
success=False,
error=str(result),
error_type=type(result).__name__,
duration_ms=int((time.time() - t0) * 1000),
))
elif isinstance(result, tuple):
engine_results, elapsed = result
stats.engine_timings[name] = elapsed
if engine_results:
stats.successful_engines += 1
all_results.extend(engine_results)
else:
stats.failed_engines.append(name)
else:
stats.failed_engines.append(name)
# Deduplicate by URL
seen_urls = set()
deduped = []
for r in all_results:
key = r.url or r.title # fallback to title if no URL
if key not in seen_urls:
seen_urls.add(key)
deduped.append(r)
stats.total_results = len(all_results)
stats.deduplicated = len(deduped)
stats.duration_ms = int((time.time() - t0) * 1000)
return deduped, stats
async def _run_engine(
self, engine: BaseEngine, query: str, max_results: int, kwargs: dict
) -> tuple[list[SearchResult], float]:
"""Run a single engine with timing."""
with execution_log.track(engine.name, "search", query=query) as ctx:
t0 = time.time()
try:
results = await engine.search(
query, max_results=max_results, **kwargs
)
elapsed = time.time() - t0
ctx.result_count = len(results)
return results, elapsed
except Exception as e:
elapsed = time.time() - t0
ctx.success = False
ctx.error = str(e)
ctx.error_type = type(e).__name__
raise
async def search_vertical(
self,
query: str,
vertical: str,
max_results: int = 10,
**kwargs,
) -> tuple[list[SearchResult], SearchStats]:
"""Search a specific vertical (news/academic/github)."""
if vertical == "github":
# GitHub supports sub-types
search_type = kwargs.pop("search_type", "repositories")
engine = self._engines["github"]
with execution_log.track("github", "search",
search_type=search_type) as ctx:
t0 = time.time()
results = await engine.search(
query, max_results=max_results,
search_type=search_type, **kwargs
)
ctx.result_count = len(results)
stats = SearchStats(
total_engines=1,
successful_engines=1 if results else 0,
total_results=len(results),
duration_ms=int((time.time() - t0) * 1000),
)
return results, stats
else:
return await self.search(
query, engines=[vertical], max_results=max_results, **kwargs
)
async def health(self) -> dict[str, dict]:
"""Check health of all engines."""
results = {}
for name, engine in self._engines.items():
try:
results[name] = await engine.health()
except Exception as e:
results[name] = {"name": name, "status": "error", "error": str(e)}
return results