Files
web-search-skills/scripts/engines/academic_engine.py
T
openbot 9d97a3a7e8 feat: 添加执行日志系统,跟踪各引擎/接口调用耗时和错误信息
新增功能:
- execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情
- 所有搜索引擎自动跟踪每次调用的性能指标
- CLI 搜索后自动显示执行摘要
- health 命令使用 ParallelSearch 检测所有引擎连通性
- 新增 log 命令查看历史执行日志
2026-05-21 14:49:57 +08:00

163 lines
5.9 KiB
Python

"""Academic search engine — ArXiv + CrossRef APIs (free, no key)."""
from __future__ import annotations
import time
import logging
from urllib.parse import quote_plus
from xml.etree import ElementTree as ET
import httpx
from scripts.engines.base import BaseEngine, EngineResult
logger = logging.getLogger(__name__)
class ArxivEngine(BaseEngine):
"""ArXiv paper search — free, no key needed."""
name = "arxiv"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
t0 = time.time()
url = (
f"http://export.arxiv.org/api/query?"
f"search_query=all:{quote_plus(query)}&start=0&max_results={max_results}"
f"&sortBy=relevance&sortOrder=descending"
)
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse_xml(resp.text, max_results)
logger.info(f"[arxiv] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[arxiv] failed: {e}")
return []
def _parse_xml(self, xml_text: str, max_results: int) -> list[EngineResult]:
root = ET.fromstring(xml_text)
ns = {"atom": "http://www.w3.org/2005/Atom"}
results = []
for entry in root.findall("atom:entry", ns)[:max_results]:
title = entry.findtext("atom:title", "", ns).strip().replace("\n", " ")
summary = entry.findtext("atom:summary", "", ns).strip().replace("\n", " ")
link_el = entry.find("atom:link[@type='text/html']", ns)
if link_el is None:
link_el = entry.find("atom:id", ns)
url = link_el.get("href", "") if link_el is not None else ""
published = entry.findtext("atom:published", "", ns)
authors = [a.findtext("atom:name", "", ns)
for a in entry.findall("atom:author", ns)]
categories = [c.get("term", "")
for c in entry.findall("atom:category", ns)]
if title:
results.append(EngineResult(
title=title, url=url, snippet=summary[:300],
source="arxiv", category="academic",
published=published,
extra={"authors": authors[:5], "categories": categories},
))
return results
class CrossRefEngine(BaseEngine):
"""CrossRef academic search — free, no key needed."""
name = "crossref"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
t0 = time.time()
url = "https://api.crossref.org/works"
params = {
"query": query,
"rows": min(max_results, 20),
"sort": "relevance",
"order": "desc",
}
headers = {"User-Agent": "web-search-skills/1.0 (mailto:search@example.com)"}
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, params=params, headers=headers)
resp.raise_for_status()
elapsed = time.time() - t0
data = resp.json()
results = []
for item in data.get("message", {}).get("items", [])[:max_results]:
title_list = item.get("title", [])
title = title_list[0] if title_list else ""
doi = item.get("DOI", "")
url_link = item.get("URL", f"https://doi.org/{doi}")
abstract = item.get("abstract", "")[:300]
pub_date = ""
if item.get("published-print"):
dp = item["published-print"].get("date-parts", [[]])[0]
pub_date = "-".join(str(p) for p in dp if p) if dp else ""
authors = [f"{a.get('given', '')} {a.get('family', '')}".strip()
for a in item.get("author", [])]
if title:
results.append(EngineResult(
title=title, url=url_link, snippet=abstract,
source="crossref", category="academic",
published=pub_date,
extra={
"doi": doi,
"authors": authors[:5],
"journal": (item.get("container-title", [""])[0]
if item.get("container-title") else ""),
},
))
logger.info(f"[crossref] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[crossref] failed: {e}")
return []
class AcademicSearch(BaseEngine):
"""Unified academic search — ArXiv + CrossRef in parallel."""
name = "academic"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
import asyncio
arxiv = ArxivEngine(timeout=self.timeout)
crossref = CrossRefEngine(timeout=self.timeout)
arxiv_results, crossref_results = await asyncio.gather(
arxiv.search(query, max_results=max_results),
crossref.search(query, max_results=max_results),
return_exceptions=True,
)
all_results = []
if isinstance(arxiv_results, list):
all_results.extend(arxiv_results)
if isinstance(crossref_results, list):
all_results.extend(crossref_results)
# Deduplicate by URL
seen = set()
deduped = []
for r in all_results:
if r.url not in seen:
seen.add(r.url)
deduped.append(r)
return deduped[:max_results]