refactor: cleanup dead code, fix model confusion, streamline CLI
- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/) - Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py) - scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py) - scripts/engines/base.py: re-exports from core.base for compatibility - scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py) - scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py) - scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources) - scripts/unified_search.py: clean async Python API (was a stub) - ws.py: lightweight launcher - README.md + SKILL.md: updated to reflect actual 15+ engines - All EngineResult → SearchResult - All engines return list[SearchResult]
This commit is contained in:
1 parent
e20478fcbd
commit
c077a7ae40
31 files changed
+1188
-3148
No files matched your search
@@ -1,4 +1,4 @@
|
||||
"""Academic search engine — ArXiv + CrossRef APIs (free, no key)."""
|
||||
"""Academic search — ArXiv + CrossRef APIs (free, no API key)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -9,18 +9,18 @@ from xml.etree import ElementTree as ET
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ArxivEngine(BaseEngine):
|
||||
"""ArXiv paper search — free, no key needed."""
|
||||
"""ArXiv paper search via public API."""
|
||||
|
||||
name = "arxiv"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
url = (
|
||||
f"http://export.arxiv.org/api/query?"
|
||||
@@ -40,8 +40,12 @@ class ArxivEngine(BaseEngine):
|
||||
logger.warning(f"[arxiv] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_xml(self, xml_text: str, max_results: int) -> list[EngineResult]:
|
||||
root = ET.fromstring(xml_text)
|
||||
def _parse_xml(self, xml_text: str, max_results: int) -> list[SearchResult]:
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError:
|
||||
return []
|
||||
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||
results = []
|
||||
|
||||
@@ -53,14 +57,11 @@ class ArxivEngine(BaseEngine):
|
||||
link_el = entry.find("atom:id", ns)
|
||||
url = link_el.get("href", "") if link_el is not None else ""
|
||||
published = entry.findtext("atom:published", "", ns)
|
||||
|
||||
authors = [a.findtext("atom:name", "", ns)
|
||||
for a in entry.findall("atom:author", ns)]
|
||||
categories = [c.get("term", "")
|
||||
for c in entry.findall("atom:category", ns)]
|
||||
authors = [a.findtext("atom:name", "", ns) for a in entry.findall("atom:author", ns)]
|
||||
categories = [c.get("term", "") for c in entry.findall("atom:category", ns)]
|
||||
|
||||
if title:
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title, url=url, snippet=summary[:300],
|
||||
source="arxiv", category="academic",
|
||||
published=published,
|
||||
@@ -71,20 +72,16 @@ class ArxivEngine(BaseEngine):
|
||||
|
||||
|
||||
class CrossRefEngine(BaseEngine):
|
||||
"""CrossRef academic search — free, no key needed."""
|
||||
"""CrossRef academic search via public API."""
|
||||
|
||||
name = "crossref"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
url = "https://api.crossref.org/works"
|
||||
params = {
|
||||
"query": query,
|
||||
"rows": min(max_results, 20),
|
||||
"sort": "relevance",
|
||||
"order": "desc",
|
||||
}
|
||||
params = {"query": query, "rows": min(max_results, 20),
|
||||
"sort": "relevance", "order": "desc"}
|
||||
headers = {"User-Agent": "web-search-skills/1.0 (mailto:search@example.com)"}
|
||||
|
||||
try:
|
||||
@@ -109,7 +106,7 @@ class CrossRefEngine(BaseEngine):
|
||||
for a in item.get("author", [])]
|
||||
|
||||
if title:
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title, url=url_link, snippet=abstract,
|
||||
source="crossref", category="academic",
|
||||
published=pub_date,
|
||||
@@ -134,7 +131,7 @@ class AcademicSearch(BaseEngine):
|
||||
name = "academic"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
import asyncio
|
||||
arxiv = ArxivEngine(timeout=self.timeout)
|
||||
crossref = CrossRefEngine(timeout=self.timeout)
|
||||
@@ -151,7 +148,6 @@ class AcademicSearch(BaseEngine):
|
||||
if isinstance(crossref_results, list):
|
||||
all_results.extend(crossref_results)
|
||||
|
||||
# Deduplicate by URL
|
||||
seen = set()
|
||||
deduped = []
|
||||
for r in all_results:
|
||||
|
||||
Reference in new issue
Block a user