- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/) - Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py) - scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py) - scripts/engines/base.py: re-exports from core.base for compatibility - scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py) - scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py) - scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources) - scripts/unified_search.py: clean async Python API (was a stub) - ws.py: lightweight launcher - README.md + SKILL.md: updated to reflect actual 15+ engines - All EngineResult → SearchResult - All engines return list[SearchResult]
159 lines
5.9 KiB
Python
159 lines
5.9 KiB
Python
"""Academic search — ArXiv + CrossRef APIs (free, no API key)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
import logging
|
|
from urllib.parse import quote_plus
|
|
from xml.etree import ElementTree as ET
|
|
|
|
import httpx
|
|
|
|
from scripts.core.base import BaseEngine, SearchResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ArxivEngine(BaseEngine):
|
|
"""ArXiv paper search via public API."""
|
|
|
|
name = "arxiv"
|
|
|
|
async def search(self, query: str, max_results: int = 10,
|
|
**kwargs) -> list[SearchResult]:
|
|
t0 = time.time()
|
|
url = (
|
|
f"http://export.arxiv.org/api/query?"
|
|
f"search_query=all:{quote_plus(query)}&start=0&max_results={max_results}"
|
|
f"&sortBy=relevance&sortOrder=descending"
|
|
)
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
|
resp = await client.get(url)
|
|
resp.raise_for_status()
|
|
elapsed = time.time() - t0
|
|
results = self._parse_xml(resp.text, max_results)
|
|
logger.info(f"[arxiv] {len(results)} results in {elapsed:.2f}s")
|
|
return results
|
|
except Exception as e:
|
|
logger.warning(f"[arxiv] failed: {e}")
|
|
return []
|
|
|
|
def _parse_xml(self, xml_text: str, max_results: int) -> list[SearchResult]:
|
|
try:
|
|
root = ET.fromstring(xml_text)
|
|
except ET.ParseError:
|
|
return []
|
|
|
|
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
|
results = []
|
|
|
|
for entry in root.findall("atom:entry", ns)[:max_results]:
|
|
title = entry.findtext("atom:title", "", ns).strip().replace("\n", " ")
|
|
summary = entry.findtext("atom:summary", "", ns).strip().replace("\n", " ")
|
|
link_el = entry.find("atom:link[@type='text/html']", ns)
|
|
if link_el is None:
|
|
link_el = entry.find("atom:id", ns)
|
|
url = link_el.get("href", "") if link_el is not None else ""
|
|
published = entry.findtext("atom:published", "", ns)
|
|
authors = [a.findtext("atom:name", "", ns) for a in entry.findall("atom:author", ns)]
|
|
categories = [c.get("term", "") for c in entry.findall("atom:category", ns)]
|
|
|
|
if title:
|
|
results.append(SearchResult(
|
|
title=title, url=url, snippet=summary[:300],
|
|
source="arxiv", category="academic",
|
|
published=published,
|
|
extra={"authors": authors[:5], "categories": categories},
|
|
))
|
|
|
|
return results
|
|
|
|
|
|
class CrossRefEngine(BaseEngine):
|
|
"""CrossRef academic search via public API."""
|
|
|
|
name = "crossref"
|
|
|
|
async def search(self, query: str, max_results: int = 10,
|
|
**kwargs) -> list[SearchResult]:
|
|
t0 = time.time()
|
|
url = "https://api.crossref.org/works"
|
|
params = {"query": query, "rows": min(max_results, 20),
|
|
"sort": "relevance", "order": "desc"}
|
|
headers = {"User-Agent": "web-search-skills/1.0 (mailto:search@example.com)"}
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
|
resp = await client.get(url, params=params, headers=headers)
|
|
resp.raise_for_status()
|
|
elapsed = time.time() - t0
|
|
data = resp.json()
|
|
results = []
|
|
for item in data.get("message", {}).get("items", [])[:max_results]:
|
|
title_list = item.get("title", [])
|
|
title = title_list[0] if title_list else ""
|
|
doi = item.get("DOI", "")
|
|
url_link = item.get("URL", f"https://doi.org/{doi}")
|
|
abstract = item.get("abstract", "")[:300]
|
|
pub_date = ""
|
|
if item.get("published-print"):
|
|
dp = item["published-print"].get("date-parts", [[]])[0]
|
|
pub_date = "-".join(str(p) for p in dp if p) if dp else ""
|
|
|
|
authors = [f"{a.get('given', '')} {a.get('family', '')}".strip()
|
|
for a in item.get("author", [])]
|
|
|
|
if title:
|
|
results.append(SearchResult(
|
|
title=title, url=url_link, snippet=abstract,
|
|
source="crossref", category="academic",
|
|
published=pub_date,
|
|
extra={
|
|
"doi": doi,
|
|
"authors": authors[:5],
|
|
"journal": (item.get("container-title", [""])[0]
|
|
if item.get("container-title") else ""),
|
|
},
|
|
))
|
|
|
|
logger.info(f"[crossref] {len(results)} results in {elapsed:.2f}s")
|
|
return results
|
|
except Exception as e:
|
|
logger.warning(f"[crossref] failed: {e}")
|
|
return []
|
|
|
|
|
|
class AcademicSearch(BaseEngine):
|
|
"""Unified academic search — ArXiv + CrossRef in parallel."""
|
|
|
|
name = "academic"
|
|
|
|
async def search(self, query: str, max_results: int = 10,
|
|
**kwargs) -> list[SearchResult]:
|
|
import asyncio
|
|
arxiv = ArxivEngine(timeout=self.timeout)
|
|
crossref = CrossRefEngine(timeout=self.timeout)
|
|
|
|
arxiv_results, crossref_results = await asyncio.gather(
|
|
arxiv.search(query, max_results=max_results),
|
|
crossref.search(query, max_results=max_results),
|
|
return_exceptions=True,
|
|
)
|
|
|
|
all_results = []
|
|
if isinstance(arxiv_results, list):
|
|
all_results.extend(arxiv_results)
|
|
if isinstance(crossref_results, list):
|
|
all_results.extend(crossref_results)
|
|
|
|
seen = set()
|
|
deduped = []
|
|
for r in all_results:
|
|
if r.url not in seen:
|
|
seen.add(r.url)
|
|
deduped.append(r)
|
|
|
|
return deduped[:max_results]
|