refactor: cleanup dead code, fix model confusion, streamline CLI

- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/)
- Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py)
- scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py)
- scripts/engines/base.py: re-exports from core.base for compatibility
- scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py)
- scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py)
- scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources)
- scripts/unified_search.py: clean async Python API (was a stub)
- ws.py: lightweight launcher
- README.md + SKILL.md: updated to reflect actual 15+ engines
- All EngineResult → SearchResult
- All engines return list[SearchResult]
This commit is contained in:
openbot-coder committed 2026-05-30 16:25:41 +08:00
1 parent e20478fcbd
commit c077a7ae40
31 files changed
+1188 -3148

No files matched your search

+149 -78
View File
@@ -1,9 +1,17 @@
"""
Unified multi-source search — URL-based engines.
Unified multi-source search — Python API.
Architecture:
Primary: URL engines (Baidu, Bing, Google, DuckDuckGo, Sogou)
No external service dependency.
Provides a clean async interface for programmatic use.
All engines run in parallel with configurable concurrency.
Usage:
from scripts.unified_search import UnifiedSearch
us = UnifiedSearch()
resp = await us.search("quantum computing", sources=["bing", "arxiv"], max_results=5)
for r in resp.results:
print(f"[{r.rank}] {r.title}")
await us.close()
"""
from __future__ import annotations
@@ -12,98 +20,161 @@ import asyncio
import time
from typing import Any, Optional
from dataclasses import dataclass
from scripts.models import SearchQuery, SearchResponse, SourceRef, SearchMetadata, EngineTiming
from scripts.core.base import BaseEngine, SearchResult
from scripts.engines.bing_scraper import BingScraper
from scripts.engines.sogou_scraper import SogouScraper
from scripts.engines.baidu_scraper import BaiduScraper
from scripts.engines.search360_scraper import Search360Scraper
from scripts.engines.parser_engines import DuckDuckGoParser, BraveParser
from scripts.engines.news_engine import NewsSearch
from scripts.engines.academic_engine import AcademicSearch
from scripts.engines.github_engine import GitHubEngine
from scripts.core.aggregator import ResultAggregator
from scripts.models import SearchResponse, SourceRef, SearchMetadata, EngineTiming
@dataclass
class SearchResult:
title: str
url: str
content: str
source: str = ""
engine: str = ""
score: float = 0.0
category: str = ""
published: str = ""
# Default engine registry
_ALL_ENGINES: dict[str, BaseEngine] = {
"bing": BingScraper(),
"sogou": SogouScraper(),
"baidu": BaiduScraper(),
"360": Search360Scraper(),
"duckduckgo": DuckDuckGoParser(),
"brave": BraveParser(),
"news": NewsSearch(),
"academic": AcademicSearch(),
"github": GitHubEngine(),
}
class UnifiedSearch:
"""Unified search across multiple open-source engines."""
"""Unified multi-source search — Python API entry point."""
def __init__(self):
self._start_time = 0.0
def __init__(self, timeout: float = 15.0, max_concurrent: int = 5):
self.timeout = timeout
self.max_concurrent = max_concurrent
self._engines: dict[str, BaseEngine] = {
name: eng.__class__(timeout=timeout)
for name, eng in _ALL_ENGINES.items()
}
async def search(self, query: str, max_results: int = 5,
language: str = "zh-CN") -> SearchResponse:
"""Search using URL-based engines."""
self._start_time = time.time()
metadata = SearchMetadata()
metadata.total_engines = 1
async def search(
self,
query: str,
sources: Optional[list[str]] = None,
max_results: int = 10,
region: str = "cn",
**kwargs,
) -> SearchResponse:
"""
Search across multiple sources in parallel.
all_results, timings = await self._search_urls(query, max_results)
Args:
query: Search query string
sources: List of engine names to use (default: all)
max_results: Max results to return (deduped)
region: "cn" or "global"
**kwargs: Extra params passed to engines
elapsed_ms = int((time.time() - self._start_time) * 1000)
metadata.duration_ms = elapsed_ms
metadata.engine_timings = timings
metadata.total_raw_results = len(all_results)
Returns:
SearchResponse with results, sources, and metadata
"""
t0 = time.time()
sources = []
seen_urls = set()
for i, r in enumerate(all_results[:max_results]):
if r.url and r.url not in seen_urls:
seen_urls.add(r.url)
sources.append(SourceRef(
index=i + 1,
title=r.title or r.content[:50],
url=r.url or "",
source=r.source,
# Select engines
if sources:
selected = {k: v for k, v in self._engines.items() if k in sources}
else:
selected = dict(self._engines)
# Build region kwarg
eng_kwargs = {"region": region} if region else {}
# Run all engines concurrently with semaphore
semaphore = asyncio.Semaphore(self.max_concurrent)
async def run_engine(name: str, eng: BaseEngine) -> tuple[str, list[SearchResult], float]:
async with semaphore:
start = time.time()
try:
results = await asyncio.wait_for(
eng.search(query, max_results=max_results, **eng_kwargs),
timeout=eng.timeout + 2.0,
)
elapsed = time.time() - start
return name, list(results), elapsed
except asyncio.TimeoutError:
return name, [], time.time() - start
except Exception:
return name, [], time.time() - start
tasks = [run_engine(n, e) for n, e in selected.items()]
done = await asyncio.gather(*tasks, return_exceptions=True)
# Collect results and timings
all_results: list[SearchResult] = []
engine_timings: list[EngineTiming] = []
engines_used: set[str] = set()
for item in done:
if isinstance(item, tuple):
name, results, elapsed = item
all_results.extend(results)
if results:
engines_used.add(name)
engine_timings.append(EngineTiming(
engine_name=name,
success=bool(results),
duration_ms=int(elapsed * 1000),
result_count=len(results),
))
elif isinstance(item, Exception):
# Exception in gather — log but don't crash
pass
metadata.engines_used = list(set(s.source for s in sources))
# Deduplicate and rank
aggregator = ResultAggregator()
merged = aggregator.merge_and_rank(all_results, max_results=max_results)
elapsed_ms = int((time.time() - t0) * 1000)
# Build SourceRef list
sources_list = [
SourceRef(
index=r.rank,
title=r.title,
url=r.url,
source=r.source,
)
for r in merged
]
metadata = SearchMetadata(
total_engines=len(selected),
total_raw_results=len(all_results),
engines_used=list(engines_used),
duration_ms=elapsed_ms,
engine_timings=engine_timings,
)
return SearchResponse(
query=query,
results=list(all_results[:max_results]),
sources=sources,
results=merged,
sources=sources_list,
metadata=metadata,
)
async def _search_urls(self, query: str, max_results: int
) -> tuple[list[SearchResult], list[EngineTiming]]:
"""Build search URLs for major engines."""
timings = []
engines = {
"baidu": f"https://www.baidu.com/s?wd={query}",
"bing": f"https://cn.bing.com/search?q={query}",
"google": f"https://www.google.com/search?q={query}",
"duckduckgo": f"https://html.duckduckgo.com/html/?q={query}",
"sogou": f"https://www.sogou.com/web?query={query}",
}
results = []
for name, url in engines.items():
results.append(SearchResult(
title=f"{name}: {query}",
url=url,
content=f"Search {query} on {name}",
source=name,
))
timings.append(EngineTiming(
engine_name="url_fallback",
success=True,
duration_ms=1,
result_count=len(results),
))
return results, timings
async def health(self) -> dict:
"""Check status."""
return {
"status": "ok",
"engines": ["baidu", "bing", "google", "duckduckgo", "sogou"],
}
async def health(self) -> dict[str, Any]:
"""Check connectivity of all engines."""
results = {}
for name, eng in self._engines.items():
try:
health = await asyncio.wait_for(eng.health(), timeout=5.0)
results[name] = health
except Exception as e:
results[name] = {"name": name, "status": "fail", "error": str(e)}
return results
async def close(self):
"""Cleanup (no-op for current implementation)."""
pass