refactor: restructure project with CLI tool, multi-engine support, and skill-builder compliant SKILL.md

- Replace setup.py with pyproject.toml (uv-compatible)
- Add CLI entry point (web-search) with 9 subcommands
- Add multi-engine architecture (14 web + 7 special sources)
- Add WeChat/CLS/WallStreetCN/ArXiv/Twitter search engines
- Remove old src/ and examples/ directories
- Remove redundant requirements.txt and setup.py
- Update README.md and SKILL.md with full documentation
- SKILL.md now compliant with skill-builder standards (frontmatter, triggers, structure)
This commit is contained in:
vex1023 committed 2026-05-14 16:31:54 +08:00
1 parent 80ac14491e
commit 87935b59ce
29 files changed
+2364 -354

No files matched your search

+130
View File
@@ -0,0 +1,130 @@
"""
ArXiv academic paper search engine.
Uses the ArXiv API (export.arxiv.org/api/query) to search for papers.
"""
from __future__ import annotations
import logging
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from typing import Optional
from urllib.parse import quote_plus
import httpx
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
class ArXivEngine(SearchEngine):
"""ArXiv academic paper search engine via official API."""
API_URL = "https://export.arxiv.org/api/query"
def __init__(self, config: Optional[dict] = None):
super().__init__("ArXiv", config)
timeout = (config or {}).get("timeout", 30)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"User-Agent": (config or {}).get(
"user_agent",
"ArXivSearchSkill/1.0",
),
"Accept": "application/atom+xml",
},
follow_redirects=True,
)
def build_url(self, query: str, max_results: int = 10, **kwargs) -> str:
search_field = kwargs.get("search_field", "all")
return (
f"{self.API_URL}?search_query={search_field}:{quote_plus(query)}"
f"&max_results={max_results}&sortBy=submittedDate&sortOrder=descending"
)
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search ArXiv papers."""
url = self.build_url(query, max_results=max_results, **kwargs)
try:
response = await self._client.get(url)
response.raise_for_status()
return self._parse_response(response.text, max_results)
except httpx.HTTPError as e:
logger.error(f"ArXiv search failed: {e}")
return []
def _parse_response(self, xml_text: str, max_results: int) -> list[SearchResult]:
"""Parse ArXiv Atom XML response."""
ns = {
"atom": "http://www.w3.org/2005/Atom",
"arxiv": "http://arxiv.org/schemas/atom",
}
results = []
try:
root = ET.fromstring(xml_text)
except ET.ParseError as e:
logger.error(f"ArXiv XML parse error: {e}")
return []
for i, entry in enumerate(root.findall("atom:entry", ns)[:max_results]):
title = entry.findtext("atom:title", "", ns)
summary = entry.findtext("atom:summary", "", ns)
published = entry.findtext("atom:published", "", ns)
updated = entry.findtext("atom:updated", "", ns)
link_el = entry.find("atom:link", ns)
link = link_el.get("href", "") if link_el is not None else ""
authors = []
for author in entry.findall("atom:author", ns):
name = author.findtext("atom:name", "", ns)
if name:
authors.append(name)
# Clean title and summary
title = title.replace("\n", " ").strip()
snippet = summary.replace("\n", " ").strip()[:300]
parsed_date = self._parse_date(published) if published else (
self._parse_date(updated) if updated else
datetime.min.replace(tzinfo=timezone.utc)
)
results.append(
SearchResult(
title=title or f"ArXiv: {summary[:60]}",
url=link,
snippet=snippet or "No abstract available",
source=self.name,
rank=i + 1,
category="academic",
extra={
"published": published or "",
"updated": updated or "",
"authors": ", ".join(authors),
"parsed_date": parsed_date,
},
)
)
return results
def _parse_date(self, date_str: str) -> datetime:
"""Parse ArXiv date format."""
try:
# ArXiv format: 2024-01-01T00:00:00Z
return datetime.strptime(
date_str.strip(), "%Y-%m-%dT%H:%M:%SZ"
).replace(tzinfo=timezone.utc)
except (ValueError, TypeError):
return datetime.min.replace(tzinfo=timezone.utc)
async def close(self):
await self._client.aclose()