- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/) - Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py) - scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py) - scripts/engines/base.py: re-exports from core.base for compatibility - scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py) - scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py) - scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources) - scripts/unified_search.py: clean async Python API (was a stub) - ws.py: lightweight launcher - README.md + SKILL.md: updated to reflect actual 15+ engines - All EngineResult → SearchResult - All engines return list[SearchResult]
181 lines
5.8 KiB
Python
181 lines
5.8 KiB
Python
"""
|
|
Unified multi-source search — Python API.
|
|
|
|
Provides a clean async interface for programmatic use.
|
|
All engines run in parallel with configurable concurrency.
|
|
|
|
Usage:
|
|
from scripts.unified_search import UnifiedSearch
|
|
|
|
us = UnifiedSearch()
|
|
resp = await us.search("quantum computing", sources=["bing", "arxiv"], max_results=5)
|
|
for r in resp.results:
|
|
print(f"[{r.rank}] {r.title}")
|
|
await us.close()
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
from typing import Any, Optional
|
|
|
|
from scripts.core.base import BaseEngine, SearchResult
|
|
from scripts.engines.bing_scraper import BingScraper
|
|
from scripts.engines.sogou_scraper import SogouScraper
|
|
from scripts.engines.baidu_scraper import BaiduScraper
|
|
from scripts.engines.search360_scraper import Search360Scraper
|
|
from scripts.engines.parser_engines import DuckDuckGoParser, BraveParser
|
|
from scripts.engines.news_engine import NewsSearch
|
|
from scripts.engines.academic_engine import AcademicSearch
|
|
from scripts.engines.github_engine import GitHubEngine
|
|
from scripts.core.aggregator import ResultAggregator
|
|
from scripts.models import SearchResponse, SourceRef, SearchMetadata, EngineTiming
|
|
|
|
|
|
# Default engine registry
|
|
_ALL_ENGINES: dict[str, BaseEngine] = {
|
|
"bing": BingScraper(),
|
|
"sogou": SogouScraper(),
|
|
"baidu": BaiduScraper(),
|
|
"360": Search360Scraper(),
|
|
"duckduckgo": DuckDuckGoParser(),
|
|
"brave": BraveParser(),
|
|
"news": NewsSearch(),
|
|
"academic": AcademicSearch(),
|
|
"github": GitHubEngine(),
|
|
}
|
|
|
|
|
|
class UnifiedSearch:
|
|
"""Unified multi-source search — Python API entry point."""
|
|
|
|
def __init__(self, timeout: float = 15.0, max_concurrent: int = 5):
|
|
self.timeout = timeout
|
|
self.max_concurrent = max_concurrent
|
|
self._engines: dict[str, BaseEngine] = {
|
|
name: eng.__class__(timeout=timeout)
|
|
for name, eng in _ALL_ENGINES.items()
|
|
}
|
|
|
|
async def search(
|
|
self,
|
|
query: str,
|
|
sources: Optional[list[str]] = None,
|
|
max_results: int = 10,
|
|
region: str = "cn",
|
|
**kwargs,
|
|
) -> SearchResponse:
|
|
"""
|
|
Search across multiple sources in parallel.
|
|
|
|
Args:
|
|
query: Search query string
|
|
sources: List of engine names to use (default: all)
|
|
max_results: Max results to return (deduped)
|
|
region: "cn" or "global"
|
|
**kwargs: Extra params passed to engines
|
|
|
|
Returns:
|
|
SearchResponse with results, sources, and metadata
|
|
"""
|
|
t0 = time.time()
|
|
|
|
# Select engines
|
|
if sources:
|
|
selected = {k: v for k, v in self._engines.items() if k in sources}
|
|
else:
|
|
selected = dict(self._engines)
|
|
|
|
# Build region kwarg
|
|
eng_kwargs = {"region": region} if region else {}
|
|
|
|
# Run all engines concurrently with semaphore
|
|
semaphore = asyncio.Semaphore(self.max_concurrent)
|
|
|
|
async def run_engine(name: str, eng: BaseEngine) -> tuple[str, list[SearchResult], float]:
|
|
async with semaphore:
|
|
start = time.time()
|
|
try:
|
|
results = await asyncio.wait_for(
|
|
eng.search(query, max_results=max_results, **eng_kwargs),
|
|
timeout=eng.timeout + 2.0,
|
|
)
|
|
elapsed = time.time() - start
|
|
return name, list(results), elapsed
|
|
except asyncio.TimeoutError:
|
|
return name, [], time.time() - start
|
|
except Exception:
|
|
return name, [], time.time() - start
|
|
|
|
tasks = [run_engine(n, e) for n, e in selected.items()]
|
|
done = await asyncio.gather(*tasks, return_exceptions=True)
|
|
|
|
# Collect results and timings
|
|
all_results: list[SearchResult] = []
|
|
engine_timings: list[EngineTiming] = []
|
|
engines_used: set[str] = set()
|
|
|
|
for item in done:
|
|
if isinstance(item, tuple):
|
|
name, results, elapsed = item
|
|
all_results.extend(results)
|
|
if results:
|
|
engines_used.add(name)
|
|
engine_timings.append(EngineTiming(
|
|
engine_name=name,
|
|
success=bool(results),
|
|
duration_ms=int(elapsed * 1000),
|
|
result_count=len(results),
|
|
))
|
|
elif isinstance(item, Exception):
|
|
# Exception in gather — log but don't crash
|
|
pass
|
|
|
|
# Deduplicate and rank
|
|
aggregator = ResultAggregator()
|
|
merged = aggregator.merge_and_rank(all_results, max_results=max_results)
|
|
|
|
elapsed_ms = int((time.time() - t0) * 1000)
|
|
|
|
# Build SourceRef list
|
|
sources_list = [
|
|
SourceRef(
|
|
index=r.rank,
|
|
title=r.title,
|
|
url=r.url,
|
|
source=r.source,
|
|
)
|
|
for r in merged
|
|
]
|
|
|
|
metadata = SearchMetadata(
|
|
total_engines=len(selected),
|
|
total_raw_results=len(all_results),
|
|
engines_used=list(engines_used),
|
|
duration_ms=elapsed_ms,
|
|
engine_timings=engine_timings,
|
|
)
|
|
|
|
return SearchResponse(
|
|
query=query,
|
|
results=merged,
|
|
sources=sources_list,
|
|
metadata=metadata,
|
|
)
|
|
|
|
async def health(self) -> dict[str, Any]:
|
|
"""Check connectivity of all engines."""
|
|
results = {}
|
|
for name, eng in self._engines.items():
|
|
try:
|
|
health = await asyncio.wait_for(eng.health(), timeout=5.0)
|
|
results[name] = health
|
|
except Exception as e:
|
|
results[name] = {"name": name, "status": "fail", "error": str(e)}
|
|
return results
|
|
|
|
async def close(self):
|
|
"""Cleanup (no-op for current implementation)."""
|
|
pass
|