Files
web-search-skills/scripts/unified_search.py
openbot-coder c077a7ae40 refactor: cleanup dead code, fix model confusion, streamline CLI
- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/)
- Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py)
- scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py)
- scripts/engines/base.py: re-exports from core.base for compatibility
- scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py)
- scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py)
- scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources)
- scripts/unified_search.py: clean async Python API (was a stub)
- ws.py: lightweight launcher
- README.md + SKILL.md: updated to reflect actual 15+ engines
- All EngineResult → SearchResult
- All engines return list[SearchResult]
2026-05-30 16:25:41 +08:00

181 lines
5.8 KiB
Python

"""
Unified multi-source search — Python API.
Provides a clean async interface for programmatic use.
All engines run in parallel with configurable concurrency.
Usage:
from scripts.unified_search import UnifiedSearch
us = UnifiedSearch()
resp = await us.search("quantum computing", sources=["bing", "arxiv"], max_results=5)
for r in resp.results:
print(f"[{r.rank}] {r.title}")
await us.close()
"""
from __future__ import annotations
import asyncio
import time
from typing import Any, Optional
from scripts.core.base import BaseEngine, SearchResult
from scripts.engines.bing_scraper import BingScraper
from scripts.engines.sogou_scraper import SogouScraper
from scripts.engines.baidu_scraper import BaiduScraper
from scripts.engines.search360_scraper import Search360Scraper
from scripts.engines.parser_engines import DuckDuckGoParser, BraveParser
from scripts.engines.news_engine import NewsSearch
from scripts.engines.academic_engine import AcademicSearch
from scripts.engines.github_engine import GitHubEngine
from scripts.core.aggregator import ResultAggregator
from scripts.models import SearchResponse, SourceRef, SearchMetadata, EngineTiming
# Default engine registry
_ALL_ENGINES: dict[str, BaseEngine] = {
"bing": BingScraper(),
"sogou": SogouScraper(),
"baidu": BaiduScraper(),
"360": Search360Scraper(),
"duckduckgo": DuckDuckGoParser(),
"brave": BraveParser(),
"news": NewsSearch(),
"academic": AcademicSearch(),
"github": GitHubEngine(),
}
class UnifiedSearch:
"""Unified multi-source search — Python API entry point."""
def __init__(self, timeout: float = 15.0, max_concurrent: int = 5):
self.timeout = timeout
self.max_concurrent = max_concurrent
self._engines: dict[str, BaseEngine] = {
name: eng.__class__(timeout=timeout)
for name, eng in _ALL_ENGINES.items()
}
async def search(
self,
query: str,
sources: Optional[list[str]] = None,
max_results: int = 10,
region: str = "cn",
**kwargs,
) -> SearchResponse:
"""
Search across multiple sources in parallel.
Args:
query: Search query string
sources: List of engine names to use (default: all)
max_results: Max results to return (deduped)
region: "cn" or "global"
**kwargs: Extra params passed to engines
Returns:
SearchResponse with results, sources, and metadata
"""
t0 = time.time()
# Select engines
if sources:
selected = {k: v for k, v in self._engines.items() if k in sources}
else:
selected = dict(self._engines)
# Build region kwarg
eng_kwargs = {"region": region} if region else {}
# Run all engines concurrently with semaphore
semaphore = asyncio.Semaphore(self.max_concurrent)
async def run_engine(name: str, eng: BaseEngine) -> tuple[str, list[SearchResult], float]:
async with semaphore:
start = time.time()
try:
results = await asyncio.wait_for(
eng.search(query, max_results=max_results, **eng_kwargs),
timeout=eng.timeout + 2.0,
)
elapsed = time.time() - start
return name, list(results), elapsed
except asyncio.TimeoutError:
return name, [], time.time() - start
except Exception:
return name, [], time.time() - start
tasks = [run_engine(n, e) for n, e in selected.items()]
done = await asyncio.gather(*tasks, return_exceptions=True)
# Collect results and timings
all_results: list[SearchResult] = []
engine_timings: list[EngineTiming] = []
engines_used: set[str] = set()
for item in done:
if isinstance(item, tuple):
name, results, elapsed = item
all_results.extend(results)
if results:
engines_used.add(name)
engine_timings.append(EngineTiming(
engine_name=name,
success=bool(results),
duration_ms=int(elapsed * 1000),
result_count=len(results),
))
elif isinstance(item, Exception):
# Exception in gather — log but don't crash
pass
# Deduplicate and rank
aggregator = ResultAggregator()
merged = aggregator.merge_and_rank(all_results, max_results=max_results)
elapsed_ms = int((time.time() - t0) * 1000)
# Build SourceRef list
sources_list = [
SourceRef(
index=r.rank,
title=r.title,
url=r.url,
source=r.source,
)
for r in merged
]
metadata = SearchMetadata(
total_engines=len(selected),
total_raw_results=len(all_results),
engines_used=list(engines_used),
duration_ms=elapsed_ms,
engine_timings=engine_timings,
)
return SearchResponse(
query=query,
results=merged,
sources=sources_list,
metadata=metadata,
)
async def health(self) -> dict[str, Any]:
"""Check connectivity of all engines."""
results = {}
for name, eng in self._engines.items():
try:
health = await asyncio.wait_for(eng.health(), timeout=5.0)
results[name] = health
except Exception as e:
results[name] = {"name": name, "status": "fail", "error": str(e)}
return results
async def close(self):
"""Cleanup (no-op for current implementation)."""
pass