refactor: cleanup dead code, fix model confusion, streamline CLI
- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/) - Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py) - scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py) - scripts/engines/base.py: re-exports from core.base for compatibility - scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py) - scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py) - scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources) - scripts/unified_search.py: clean async Python API (was a stub) - ws.py: lightweight launcher - README.md + SKILL.md: updated to reflect actual 15+ engines - All EngineResult → SearchResult - All engines return list[SearchResult]
This commit is contained in:
1 parent
e20478fcbd
commit
c077a7ae40
31 files changed
+1188
-3148
No files matched your search
+149
-78
@@ -1,9 +1,17 @@
|
||||
"""
|
||||
Unified multi-source search — URL-based engines.
|
||||
Unified multi-source search — Python API.
|
||||
|
||||
Architecture:
|
||||
Primary: URL engines (Baidu, Bing, Google, DuckDuckGo, Sogou)
|
||||
No external service dependency.
|
||||
Provides a clean async interface for programmatic use.
|
||||
All engines run in parallel with configurable concurrency.
|
||||
|
||||
Usage:
|
||||
from scripts.unified_search import UnifiedSearch
|
||||
|
||||
us = UnifiedSearch()
|
||||
resp = await us.search("quantum computing", sources=["bing", "arxiv"], max_results=5)
|
||||
for r in resp.results:
|
||||
print(f"[{r.rank}] {r.title}")
|
||||
await us.close()
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -12,98 +20,161 @@ import asyncio
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from scripts.models import SearchQuery, SearchResponse, SourceRef, SearchMetadata, EngineTiming
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
from scripts.engines.bing_scraper import BingScraper
|
||||
from scripts.engines.sogou_scraper import SogouScraper
|
||||
from scripts.engines.baidu_scraper import BaiduScraper
|
||||
from scripts.engines.search360_scraper import Search360Scraper
|
||||
from scripts.engines.parser_engines import DuckDuckGoParser, BraveParser
|
||||
from scripts.engines.news_engine import NewsSearch
|
||||
from scripts.engines.academic_engine import AcademicSearch
|
||||
from scripts.engines.github_engine import GitHubEngine
|
||||
from scripts.core.aggregator import ResultAggregator
|
||||
from scripts.models import SearchResponse, SourceRef, SearchMetadata, EngineTiming
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
title: str
|
||||
url: str
|
||||
content: str
|
||||
source: str = ""
|
||||
engine: str = ""
|
||||
score: float = 0.0
|
||||
category: str = ""
|
||||
published: str = ""
|
||||
# Default engine registry
|
||||
_ALL_ENGINES: dict[str, BaseEngine] = {
|
||||
"bing": BingScraper(),
|
||||
"sogou": SogouScraper(),
|
||||
"baidu": BaiduScraper(),
|
||||
"360": Search360Scraper(),
|
||||
"duckduckgo": DuckDuckGoParser(),
|
||||
"brave": BraveParser(),
|
||||
"news": NewsSearch(),
|
||||
"academic": AcademicSearch(),
|
||||
"github": GitHubEngine(),
|
||||
}
|
||||
|
||||
|
||||
class UnifiedSearch:
|
||||
"""Unified search across multiple open-source engines."""
|
||||
"""Unified multi-source search — Python API entry point."""
|
||||
|
||||
def __init__(self):
|
||||
self._start_time = 0.0
|
||||
def __init__(self, timeout: float = 15.0, max_concurrent: int = 5):
|
||||
self.timeout = timeout
|
||||
self.max_concurrent = max_concurrent
|
||||
self._engines: dict[str, BaseEngine] = {
|
||||
name: eng.__class__(timeout=timeout)
|
||||
for name, eng in _ALL_ENGINES.items()
|
||||
}
|
||||
|
||||
async def search(self, query: str, max_results: int = 5,
|
||||
language: str = "zh-CN") -> SearchResponse:
|
||||
"""Search using URL-based engines."""
|
||||
self._start_time = time.time()
|
||||
metadata = SearchMetadata()
|
||||
metadata.total_engines = 1
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
sources: Optional[list[str]] = None,
|
||||
max_results: int = 10,
|
||||
region: str = "cn",
|
||||
**kwargs,
|
||||
) -> SearchResponse:
|
||||
"""
|
||||
Search across multiple sources in parallel.
|
||||
|
||||
all_results, timings = await self._search_urls(query, max_results)
|
||||
Args:
|
||||
query: Search query string
|
||||
sources: List of engine names to use (default: all)
|
||||
max_results: Max results to return (deduped)
|
||||
region: "cn" or "global"
|
||||
**kwargs: Extra params passed to engines
|
||||
|
||||
elapsed_ms = int((time.time() - self._start_time) * 1000)
|
||||
metadata.duration_ms = elapsed_ms
|
||||
metadata.engine_timings = timings
|
||||
metadata.total_raw_results = len(all_results)
|
||||
Returns:
|
||||
SearchResponse with results, sources, and metadata
|
||||
"""
|
||||
t0 = time.time()
|
||||
|
||||
sources = []
|
||||
seen_urls = set()
|
||||
for i, r in enumerate(all_results[:max_results]):
|
||||
if r.url and r.url not in seen_urls:
|
||||
seen_urls.add(r.url)
|
||||
sources.append(SourceRef(
|
||||
index=i + 1,
|
||||
title=r.title or r.content[:50],
|
||||
url=r.url or "",
|
||||
source=r.source,
|
||||
# Select engines
|
||||
if sources:
|
||||
selected = {k: v for k, v in self._engines.items() if k in sources}
|
||||
else:
|
||||
selected = dict(self._engines)
|
||||
|
||||
# Build region kwarg
|
||||
eng_kwargs = {"region": region} if region else {}
|
||||
|
||||
# Run all engines concurrently with semaphore
|
||||
semaphore = asyncio.Semaphore(self.max_concurrent)
|
||||
|
||||
async def run_engine(name: str, eng: BaseEngine) -> tuple[str, list[SearchResult], float]:
|
||||
async with semaphore:
|
||||
start = time.time()
|
||||
try:
|
||||
results = await asyncio.wait_for(
|
||||
eng.search(query, max_results=max_results, **eng_kwargs),
|
||||
timeout=eng.timeout + 2.0,
|
||||
)
|
||||
elapsed = time.time() - start
|
||||
return name, list(results), elapsed
|
||||
except asyncio.TimeoutError:
|
||||
return name, [], time.time() - start
|
||||
except Exception:
|
||||
return name, [], time.time() - start
|
||||
|
||||
tasks = [run_engine(n, e) for n, e in selected.items()]
|
||||
done = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# Collect results and timings
|
||||
all_results: list[SearchResult] = []
|
||||
engine_timings: list[EngineTiming] = []
|
||||
engines_used: set[str] = set()
|
||||
|
||||
for item in done:
|
||||
if isinstance(item, tuple):
|
||||
name, results, elapsed = item
|
||||
all_results.extend(results)
|
||||
if results:
|
||||
engines_used.add(name)
|
||||
engine_timings.append(EngineTiming(
|
||||
engine_name=name,
|
||||
success=bool(results),
|
||||
duration_ms=int(elapsed * 1000),
|
||||
result_count=len(results),
|
||||
))
|
||||
elif isinstance(item, Exception):
|
||||
# Exception in gather — log but don't crash
|
||||
pass
|
||||
|
||||
metadata.engines_used = list(set(s.source for s in sources))
|
||||
# Deduplicate and rank
|
||||
aggregator = ResultAggregator()
|
||||
merged = aggregator.merge_and_rank(all_results, max_results=max_results)
|
||||
|
||||
elapsed_ms = int((time.time() - t0) * 1000)
|
||||
|
||||
# Build SourceRef list
|
||||
sources_list = [
|
||||
SourceRef(
|
||||
index=r.rank,
|
||||
title=r.title,
|
||||
url=r.url,
|
||||
source=r.source,
|
||||
)
|
||||
for r in merged
|
||||
]
|
||||
|
||||
metadata = SearchMetadata(
|
||||
total_engines=len(selected),
|
||||
total_raw_results=len(all_results),
|
||||
engines_used=list(engines_used),
|
||||
duration_ms=elapsed_ms,
|
||||
engine_timings=engine_timings,
|
||||
)
|
||||
|
||||
return SearchResponse(
|
||||
query=query,
|
||||
results=list(all_results[:max_results]),
|
||||
sources=sources,
|
||||
results=merged,
|
||||
sources=sources_list,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
async def _search_urls(self, query: str, max_results: int
|
||||
) -> tuple[list[SearchResult], list[EngineTiming]]:
|
||||
"""Build search URLs for major engines."""
|
||||
timings = []
|
||||
engines = {
|
||||
"baidu": f"https://www.baidu.com/s?wd={query}",
|
||||
"bing": f"https://cn.bing.com/search?q={query}",
|
||||
"google": f"https://www.google.com/search?q={query}",
|
||||
"duckduckgo": f"https://html.duckduckgo.com/html/?q={query}",
|
||||
"sogou": f"https://www.sogou.com/web?query={query}",
|
||||
}
|
||||
results = []
|
||||
for name, url in engines.items():
|
||||
results.append(SearchResult(
|
||||
title=f"{name}: {query}",
|
||||
url=url,
|
||||
content=f"Search {query} on {name}",
|
||||
source=name,
|
||||
))
|
||||
|
||||
timings.append(EngineTiming(
|
||||
engine_name="url_fallback",
|
||||
success=True,
|
||||
duration_ms=1,
|
||||
result_count=len(results),
|
||||
))
|
||||
return results, timings
|
||||
|
||||
async def health(self) -> dict:
|
||||
"""Check status."""
|
||||
return {
|
||||
"status": "ok",
|
||||
"engines": ["baidu", "bing", "google", "duckduckgo", "sogou"],
|
||||
}
|
||||
async def health(self) -> dict[str, Any]:
|
||||
"""Check connectivity of all engines."""
|
||||
results = {}
|
||||
for name, eng in self._engines.items():
|
||||
try:
|
||||
health = await asyncio.wait_for(eng.health(), timeout=5.0)
|
||||
results[name] = health
|
||||
except Exception as e:
|
||||
results[name] = {"name": name, "status": "fail", "error": str(e)}
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
"""Cleanup (no-op for current implementation)."""
|
||||
pass
|
||||
Reference in new issue
Block a user