refactor: cleanup dead code, fix model confusion, streamline CLI

- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/)
- Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py)
- scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py)
- scripts/engines/base.py: re-exports from core.base for compatibility
- scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py)
- scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py)
- scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources)
- scripts/unified_search.py: clean async Python API (was a stub)
- ws.py: lightweight launcher
- README.md + SKILL.md: updated to reflect actual 15+ engines
- All EngineResult → SearchResult
- All engines return list[SearchResult]
This commit is contained in:
openbot-coder committed 2026-05-30 16:25:41 +08:00
1 parent e20478fcbd
commit c077a7ae40
31 files changed
+1188 -3148

No files matched your search

+19 -23
View File
@@ -1,4 +1,4 @@
"""Academic search engine — ArXiv + CrossRef APIs (free, no key)."""
"""Academic search — ArXiv + CrossRef APIs (free, no API key)."""
from __future__ import annotations
@@ -9,18 +9,18 @@ from xml.etree import ElementTree as ET
import httpx
from scripts.engines.base import BaseEngine, EngineResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
class ArxivEngine(BaseEngine):
"""ArXiv paper search — free, no key needed."""
"""ArXiv paper search via public API."""
name = "arxiv"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
**kwargs) -> list[SearchResult]:
t0 = time.time()
url = (
f"http://export.arxiv.org/api/query?"
@@ -40,8 +40,12 @@ class ArxivEngine(BaseEngine):
logger.warning(f"[arxiv] failed: {e}")
return []
def _parse_xml(self, xml_text: str, max_results: int) -> list[EngineResult]:
root = ET.fromstring(xml_text)
def _parse_xml(self, xml_text: str, max_results: int) -> list[SearchResult]:
try:
root = ET.fromstring(xml_text)
except ET.ParseError:
return []
ns = {"atom": "http://www.w3.org/2005/Atom"}
results = []
@@ -53,14 +57,11 @@ class ArxivEngine(BaseEngine):
link_el = entry.find("atom:id", ns)
url = link_el.get("href", "") if link_el is not None else ""
published = entry.findtext("atom:published", "", ns)
authors = [a.findtext("atom:name", "", ns)
for a in entry.findall("atom:author", ns)]
categories = [c.get("term", "")
for c in entry.findall("atom:category", ns)]
authors = [a.findtext("atom:name", "", ns) for a in entry.findall("atom:author", ns)]
categories = [c.get("term", "") for c in entry.findall("atom:category", ns)]
if title:
results.append(EngineResult(
results.append(SearchResult(
title=title, url=url, snippet=summary[:300],
source="arxiv", category="academic",
published=published,
@@ -71,20 +72,16 @@ class ArxivEngine(BaseEngine):
class CrossRefEngine(BaseEngine):
"""CrossRef academic search — free, no key needed."""
"""CrossRef academic search via public API."""
name = "crossref"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
**kwargs) -> list[SearchResult]:
t0 = time.time()
url = "https://api.crossref.org/works"
params = {
"query": query,
"rows": min(max_results, 20),
"sort": "relevance",
"order": "desc",
}
params = {"query": query, "rows": min(max_results, 20),
"sort": "relevance", "order": "desc"}
headers = {"User-Agent": "web-search-skills/1.0 (mailto:search@example.com)"}
try:
@@ -109,7 +106,7 @@ class CrossRefEngine(BaseEngine):
for a in item.get("author", [])]
if title:
results.append(EngineResult(
results.append(SearchResult(
title=title, url=url_link, snippet=abstract,
source="crossref", category="academic",
published=pub_date,
@@ -134,7 +131,7 @@ class AcademicSearch(BaseEngine):
name = "academic"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
**kwargs) -> list[SearchResult]:
import asyncio
arxiv = ArxivEngine(timeout=self.timeout)
crossref = CrossRefEngine(timeout=self.timeout)
@@ -151,7 +148,6 @@ class AcademicSearch(BaseEngine):
if isinstance(crossref_results, list):
all_results.extend(crossref_results)
# Deduplicate by URL
seen = set()
deduped = []
for r in all_results: