refactor: restructure project with CLI tool, multi-engine support, and skill-builder compliant SKILL.md
- Replace setup.py with pyproject.toml (uv-compatible) - Add CLI entry point (web-search) with 9 subcommands - Add multi-engine architecture (14 web + 7 special sources) - Add WeChat/CLS/WallStreetCN/ArXiv/Twitter search engines - Remove old src/ and examples/ directories - Remove redundant requirements.txt and setup.py - Update README.md and SKILL.md with full documentation - SKILL.md now compliant with skill-builder standards (frontmatter, triggers, structure)
This commit is contained in:
1 parent
80ac14491e
commit
87935b59ce
29 files changed
+2364
-354
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Academic search engines."""
|
||||
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
ArXiv academic paper search engine.
|
||||
|
||||
Uses the ArXiv API (export.arxiv.org/api/query) to search for papers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ArXivEngine(SearchEngine):
|
||||
"""ArXiv academic paper search engine via official API."""
|
||||
|
||||
API_URL = "https://export.arxiv.org/api/query"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("ArXiv", config)
|
||||
timeout = (config or {}).get("timeout", 30)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"ArXivSearchSkill/1.0",
|
||||
),
|
||||
"Accept": "application/atom+xml",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, max_results: int = 10, **kwargs) -> str:
|
||||
search_field = kwargs.get("search_field", "all")
|
||||
return (
|
||||
f"{self.API_URL}?search_query={search_field}:{quote_plus(query)}"
|
||||
f"&max_results={max_results}&sortBy=submittedDate&sortOrder=descending"
|
||||
)
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search ArXiv papers."""
|
||||
url = self.build_url(query, max_results=max_results, **kwargs)
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return self._parse_response(response.text, max_results)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"ArXiv search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_response(self, xml_text: str, max_results: int) -> list[SearchResult]:
|
||||
"""Parse ArXiv Atom XML response."""
|
||||
ns = {
|
||||
"atom": "http://www.w3.org/2005/Atom",
|
||||
"arxiv": "http://arxiv.org/schemas/atom",
|
||||
}
|
||||
results = []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError as e:
|
||||
logger.error(f"ArXiv XML parse error: {e}")
|
||||
return []
|
||||
|
||||
for i, entry in enumerate(root.findall("atom:entry", ns)[:max_results]):
|
||||
title = entry.findtext("atom:title", "", ns)
|
||||
summary = entry.findtext("atom:summary", "", ns)
|
||||
published = entry.findtext("atom:published", "", ns)
|
||||
updated = entry.findtext("atom:updated", "", ns)
|
||||
|
||||
link_el = entry.find("atom:link", ns)
|
||||
link = link_el.get("href", "") if link_el is not None else ""
|
||||
|
||||
authors = []
|
||||
for author in entry.findall("atom:author", ns):
|
||||
name = author.findtext("atom:name", "", ns)
|
||||
if name:
|
||||
authors.append(name)
|
||||
|
||||
# Clean title and summary
|
||||
title = title.replace("\n", " ").strip()
|
||||
snippet = summary.replace("\n", " ").strip()[:300]
|
||||
|
||||
parsed_date = self._parse_date(published) if published else (
|
||||
self._parse_date(updated) if updated else
|
||||
datetime.min.replace(tzinfo=timezone.utc)
|
||||
)
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title or f"ArXiv: {summary[:60]}",
|
||||
url=link,
|
||||
snippet=snippet or "No abstract available",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="academic",
|
||||
extra={
|
||||
"published": published or "",
|
||||
"updated": updated or "",
|
||||
"authors": ", ".join(authors),
|
||||
"parsed_date": parsed_date,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _parse_date(self, date_str: str) -> datetime:
|
||||
"""Parse ArXiv date format."""
|
||||
try:
|
||||
# ArXiv format: 2024-01-01T00:00:00Z
|
||||
return datetime.strptime(
|
||||
date_str.strip(), "%Y-%m-%dT%H:%M:%SZ"
|
||||
).replace(tzinfo=timezone.utc)
|
||||
except (ValueError, TypeError):
|
||||
return datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
Reference in new issue
Block a user