refactor: cleanup dead code, fix model confusion, streamline CLI
- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/) - Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py) - scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py) - scripts/engines/base.py: re-exports from core.base for compatibility - scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py) - scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py) - scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources) - scripts/unified_search.py: clean async Python API (was a stub) - ws.py: lightweight launcher - README.md + SKILL.md: updated to reflect actual 15+ engines - All EngineResult → SearchResult - All engines return list[SearchResult]
This commit is contained in:
1 parent
e20478fcbd
commit
c077a7ae40
31 files changed
+1188
-3148
No files matched your search
@@ -1 +0,0 @@
|
||||
"""Academic search engines."""
|
||||
@@ -1,130 +0,0 @@
|
||||
"""
|
||||
ArXiv academic paper search engine.
|
||||
|
||||
Uses the ArXiv API (export.arxiv.org/api/query) to search for papers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ArXivEngine(SearchEngine):
|
||||
"""ArXiv academic paper search engine via official API."""
|
||||
|
||||
API_URL = "https://export.arxiv.org/api/query"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("ArXiv", config)
|
||||
timeout = (config or {}).get("timeout", 30)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"ArXivSearchSkill/1.0",
|
||||
),
|
||||
"Accept": "application/atom+xml",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, max_results: int = 10, **kwargs) -> str:
|
||||
search_field = kwargs.get("search_field", "all")
|
||||
return (
|
||||
f"{self.API_URL}?search_query={search_field}:{quote_plus(query)}"
|
||||
f"&max_results={max_results}&sortBy=submittedDate&sortOrder=descending"
|
||||
)
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search ArXiv papers."""
|
||||
url = self.build_url(query, max_results=max_results, **kwargs)
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return self._parse_response(response.text, max_results)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"ArXiv search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_response(self, xml_text: str, max_results: int) -> list[SearchResult]:
|
||||
"""Parse ArXiv Atom XML response."""
|
||||
ns = {
|
||||
"atom": "http://www.w3.org/2005/Atom",
|
||||
"arxiv": "http://arxiv.org/schemas/atom",
|
||||
}
|
||||
results = []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError as e:
|
||||
logger.error(f"ArXiv XML parse error: {e}")
|
||||
return []
|
||||
|
||||
for i, entry in enumerate(root.findall("atom:entry", ns)[:max_results]):
|
||||
title = entry.findtext("atom:title", "", ns)
|
||||
summary = entry.findtext("atom:summary", "", ns)
|
||||
published = entry.findtext("atom:published", "", ns)
|
||||
updated = entry.findtext("atom:updated", "", ns)
|
||||
|
||||
link_el = entry.find("atom:link", ns)
|
||||
link = link_el.get("href", "") if link_el is not None else ""
|
||||
|
||||
authors = []
|
||||
for author in entry.findall("atom:author", ns):
|
||||
name = author.findtext("atom:name", "", ns)
|
||||
if name:
|
||||
authors.append(name)
|
||||
|
||||
# Clean title and summary
|
||||
title = title.replace("\n", " ").strip()
|
||||
snippet = summary.replace("\n", " ").strip()[:300]
|
||||
|
||||
parsed_date = self._parse_date(published) if published else (
|
||||
self._parse_date(updated) if updated else
|
||||
datetime.min.replace(tzinfo=timezone.utc)
|
||||
)
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title or f"ArXiv: {summary[:60]}",
|
||||
url=link,
|
||||
snippet=snippet or "No abstract available",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="academic",
|
||||
extra={
|
||||
"published": published or "",
|
||||
"updated": updated or "",
|
||||
"authors": ", ".join(authors),
|
||||
"parsed_date": parsed_date,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _parse_date(self, date_str: str) -> datetime:
|
||||
"""Parse ArXiv date format."""
|
||||
try:
|
||||
# ArXiv format: 2024-01-01T00:00:00Z
|
||||
return datetime.strptime(
|
||||
date_str.strip(), "%Y-%m-%dT%H:%M:%SZ"
|
||||
).replace(tzinfo=timezone.utc)
|
||||
except (ValueError, TypeError):
|
||||
return datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
+537
-643
File diff suppressed because it is too large.
Load diff
+48
-27
@@ -1,48 +1,69 @@
|
||||
"""
|
||||
Core data types and base classes for search engines.
|
||||
Unified data models — single source of truth for all search results.
|
||||
|
||||
Provides:
|
||||
- SearchResult: Standard result dataclass used across all engines
|
||||
- SearchEngine: Abstract base class for all search engine implementations
|
||||
Defines:
|
||||
- SearchResult: unified result from any engine (immutable dataclass)
|
||||
- BaseEngine: abstract base for all engines
|
||||
- rank field: assigned by aggregator, not engines
|
||||
|
||||
All engine-specific models must convert to SearchResult before returning.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""Represents a single search result."""
|
||||
title: str = ""
|
||||
url: str = ""
|
||||
"""Unified search result — returned by all engines."""
|
||||
title: str
|
||||
url: str
|
||||
snippet: str = ""
|
||||
source: str = ""
|
||||
rank: int = 0
|
||||
category: str = ""
|
||||
source: str = "" # engine name (e.g. "bing", "arxiv")
|
||||
category: str = "" # "web" | "news" | "academic" | "github" | "social"
|
||||
rank: int = 0 # assigned by aggregator, not engine
|
||||
published: str = "" # ISO date string (optional)
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"title": self.title,
|
||||
"url": self.url,
|
||||
"snippet": self.snippet,
|
||||
"source": self.source,
|
||||
"category": self.category,
|
||||
"rank": self.rank,
|
||||
"published": self.published,
|
||||
"extra": self.extra,
|
||||
}
|
||||
|
||||
class SearchEngine(ABC):
|
||||
"""Abstract base class for search engines."""
|
||||
|
||||
def __init__(self, name: str, config: Optional[dict[str, Any]] = None):
|
||||
self.name = name
|
||||
self.config = config or {}
|
||||
class BaseEngine(ABC):
|
||||
"""Abstract base for all search engines.
|
||||
|
||||
Subclasses override `search()`, which returns list[SearchResult].
|
||||
`health()` provides a quick connectivity check.
|
||||
"""
|
||||
|
||||
name: str = "unknown"
|
||||
timeout: float = 15.0
|
||||
|
||||
def __init__(self, timeout: float = 15.0):
|
||||
self.timeout = timeout
|
||||
|
||||
@abstractmethod
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Execute a search and return results."""
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[SearchResult]:
|
||||
"""Search and return unified results."""
|
||||
...
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
"""Build a search URL for this engine (optional)."""
|
||||
return ""
|
||||
|
||||
async def close(self):
|
||||
"""Clean up resources."""
|
||||
pass
|
||||
async def health(self) -> dict[str, Any]:
|
||||
"""Quick connectivity check. Returns {"status": "ok"} or {"status": "fail", "error": "..."}."""
|
||||
try:
|
||||
results = await self.search("test", max_results=1)
|
||||
return {"name": self.name, "status": "ok", "count": len(results)}
|
||||
except Exception as e:
|
||||
return {"name": self.name, "status": "fail", "error": str(e)}
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Academic search engine — ArXiv + CrossRef APIs (free, no key)."""
|
||||
"""Academic search — ArXiv + CrossRef APIs (free, no API key)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -9,18 +9,18 @@ from xml.etree import ElementTree as ET
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ArxivEngine(BaseEngine):
|
||||
"""ArXiv paper search — free, no key needed."""
|
||||
"""ArXiv paper search via public API."""
|
||||
|
||||
name = "arxiv"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
url = (
|
||||
f"http://export.arxiv.org/api/query?"
|
||||
@@ -40,8 +40,12 @@ class ArxivEngine(BaseEngine):
|
||||
logger.warning(f"[arxiv] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_xml(self, xml_text: str, max_results: int) -> list[EngineResult]:
|
||||
root = ET.fromstring(xml_text)
|
||||
def _parse_xml(self, xml_text: str, max_results: int) -> list[SearchResult]:
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError:
|
||||
return []
|
||||
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||
results = []
|
||||
|
||||
@@ -53,14 +57,11 @@ class ArxivEngine(BaseEngine):
|
||||
link_el = entry.find("atom:id", ns)
|
||||
url = link_el.get("href", "") if link_el is not None else ""
|
||||
published = entry.findtext("atom:published", "", ns)
|
||||
|
||||
authors = [a.findtext("atom:name", "", ns)
|
||||
for a in entry.findall("atom:author", ns)]
|
||||
categories = [c.get("term", "")
|
||||
for c in entry.findall("atom:category", ns)]
|
||||
authors = [a.findtext("atom:name", "", ns) for a in entry.findall("atom:author", ns)]
|
||||
categories = [c.get("term", "") for c in entry.findall("atom:category", ns)]
|
||||
|
||||
if title:
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title, url=url, snippet=summary[:300],
|
||||
source="arxiv", category="academic",
|
||||
published=published,
|
||||
@@ -71,20 +72,16 @@ class ArxivEngine(BaseEngine):
|
||||
|
||||
|
||||
class CrossRefEngine(BaseEngine):
|
||||
"""CrossRef academic search — free, no key needed."""
|
||||
"""CrossRef academic search via public API."""
|
||||
|
||||
name = "crossref"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
url = "https://api.crossref.org/works"
|
||||
params = {
|
||||
"query": query,
|
||||
"rows": min(max_results, 20),
|
||||
"sort": "relevance",
|
||||
"order": "desc",
|
||||
}
|
||||
params = {"query": query, "rows": min(max_results, 20),
|
||||
"sort": "relevance", "order": "desc"}
|
||||
headers = {"User-Agent": "web-search-skills/1.0 (mailto:search@example.com)"}
|
||||
|
||||
try:
|
||||
@@ -109,7 +106,7 @@ class CrossRefEngine(BaseEngine):
|
||||
for a in item.get("author", [])]
|
||||
|
||||
if title:
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title, url=url_link, snippet=abstract,
|
||||
source="crossref", category="academic",
|
||||
published=pub_date,
|
||||
@@ -134,7 +131,7 @@ class AcademicSearch(BaseEngine):
|
||||
name = "academic"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
import asyncio
|
||||
arxiv = ArxivEngine(timeout=self.timeout)
|
||||
crossref = CrossRefEngine(timeout=self.timeout)
|
||||
@@ -151,7 +148,6 @@ class AcademicSearch(BaseEngine):
|
||||
if isinstance(crossref_results, list):
|
||||
all_results.extend(crossref_results)
|
||||
|
||||
# Deduplicate by URL
|
||||
seen = set()
|
||||
deduped = []
|
||||
for r in all_results:
|
||||
|
||||
@@ -18,7 +18,7 @@ from typing import Any, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -51,18 +51,18 @@ def _get_api_key() -> str:
|
||||
return env_from_file.get(ENV_KEY, "")
|
||||
|
||||
|
||||
class BaiduQianFanEngine(SearchEngine):
|
||||
class BaiduQianFanEngine(BaseEngine):
|
||||
"""百度千帆 AI 搜索 API 引擎."""
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("Baidu Qianfan", config)
|
||||
name = "baidu_qianfan"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None, timeout: float = 30.0):
|
||||
super().__init__(timeout=timeout)
|
||||
self.api_key = (config or {}).get("api_key", "") or _get_api_key()
|
||||
if not self.api_key:
|
||||
logger.warning(
|
||||
f"BAIDU_API_KEY not set! Set it in .env file or as environment variable."
|
||||
"BAIDU_API_KEY not set. Set it in .env or as environment variable."
|
||||
)
|
||||
|
||||
timeout = (config or {}).get("timeout", 30)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
@@ -76,17 +76,7 @@ class BaiduQianFanEngine(SearchEngine):
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search via Baidu Qianfan AI Search API.
|
||||
|
||||
Args:
|
||||
query: Search keywords.
|
||||
max_results: Max results to return.
|
||||
**kwargs:
|
||||
site: Restrict search to a specific site (e.g. "mp.weixin.qq.com").
|
||||
recency: Time filter: "week", "month", "semiyear", "year" (default: "year").
|
||||
edition: "standard" (default) or "lite".
|
||||
safe_search: bool, default False.
|
||||
"""
|
||||
"""Search via Baidu Qianfan AI Search API."""
|
||||
if not self.api_key:
|
||||
return [
|
||||
SearchResult(
|
||||
@@ -100,19 +90,14 @@ class BaiduQianFanEngine(SearchEngine):
|
||||
)
|
||||
]
|
||||
|
||||
# Build resource_type_filter based on max_results
|
||||
top_k = min(max_results, 50)
|
||||
resource_type_filter = [
|
||||
{"type": "web", "top_k": top_k},
|
||||
]
|
||||
resource_type_filter = [{"type": "web", "top_k": top_k}]
|
||||
|
||||
# Build search_filter if site is specified
|
||||
search_filter = {}
|
||||
site = kwargs.get("site")
|
||||
if site:
|
||||
search_filter["match"] = {"site": [site]}
|
||||
search_filter: dict[str, Any] = {}
|
||||
if kwargs.get("site"):
|
||||
search_filter["match"] = {"site": [kwargs["site"]]}
|
||||
|
||||
request_body = {
|
||||
request_body: dict[str, Any] = {
|
||||
"messages": [{"content": query, "role": "user"}],
|
||||
"edition": kwargs.get("edition", "standard"),
|
||||
"search_source": "baidu_search_v2",
|
||||
@@ -134,38 +119,24 @@ class BaiduQianFanEngine(SearchEngine):
|
||||
logger.error(f"Baidu Qianfan API error: {data.get('message', '')}")
|
||||
return []
|
||||
|
||||
references = data.get("references", [])
|
||||
results = []
|
||||
for i, ref in enumerate(references[:max_results]):
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=ref.get("title", ""),
|
||||
url=ref.get("url", ""),
|
||||
snippet=ref.get("snippet", ""),
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="web",
|
||||
extra={
|
||||
k: ref[k]
|
||||
for k in ("site", "date", "type", "source")
|
||||
if k in ref
|
||||
},
|
||||
)
|
||||
)
|
||||
for i, ref in enumerate(data.get("references", [])[:max_results]):
|
||||
results.append(SearchResult(
|
||||
title=ref.get("title", ""),
|
||||
url=ref.get("url", ""),
|
||||
snippet=ref.get("snippet", ""),
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="web",
|
||||
extra={k: ref[k] for k in ("site", "date", "type", "source") if k in ref},
|
||||
))
|
||||
return results
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.error(f"Baidu Qianfan API HTTP error: {e.response.status_code}")
|
||||
if e.response.status_code == 401:
|
||||
logger.error("API Key 无效或未授权,请检查 BAIDU_API_KEY")
|
||||
elif e.response.status_code == 429:
|
||||
logger.error("API 配额不足")
|
||||
logger.error(f"Baidu Qianfan HTTP {e.response.status_code}: {e}")
|
||||
return []
|
||||
except httpx.RequestError as e:
|
||||
logger.error(f"Baidu Qianfan API request failed: {e}")
|
||||
return []
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Baidu Qianfan API response parse failed: {e}")
|
||||
logger.error(f"Baidu Qianfan request failed: {e}")
|
||||
return []
|
||||
|
||||
async def close(self):
|
||||
|
||||
@@ -9,7 +9,7 @@ from urllib.parse import quote_plus
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -27,7 +27,7 @@ class BaiduScraper(BaseEngine):
|
||||
name = "baidu"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
url = f"https://www.baidu.com/s?wd={quote_plus(query)}&rn={max_results}"
|
||||
|
||||
@@ -43,7 +43,7 @@ class BaiduScraper(BaseEngine):
|
||||
logger.warning(f"[baidu] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
@@ -60,7 +60,7 @@ class BaiduScraper(BaseEngine):
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href:
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title, url=href, snippet=snippet, source="baidu",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
|
||||
+3
-55
@@ -6,59 +6,7 @@ from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from scripts.core.execution_log import execution_log
|
||||
# Re-export SearchResult/BaseEngine from scripts.core.base for engine code compatibility
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineResult:
|
||||
"""Unified result from any engine."""
|
||||
title: str
|
||||
url: str
|
||||
snippet: str = ""
|
||||
source: str = ""
|
||||
category: str = "" # "general" | "news" | "academic" | "github"
|
||||
published: str = ""
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class BaseEngine(ABC):
|
||||
"""Base class for all search engines.
|
||||
|
||||
Automatically tracks execution timing and errors via execution_log.
|
||||
"""
|
||||
|
||||
name: str = "unknown"
|
||||
timeout: float = 10.0
|
||||
|
||||
def __init__(self, timeout: float = 10.0):
|
||||
self.timeout = timeout
|
||||
|
||||
async def search_with_log(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[EngineResult]:
|
||||
"""Search with automatic execution logging.
|
||||
|
||||
Subclasses override `search()`, not this method.
|
||||
"""
|
||||
with execution_log.track(self.name, "search", query=query) as ctx:
|
||||
results = await self.search(query, max_results=max_results, **kwargs)
|
||||
ctx.result_count = len(results)
|
||||
return results
|
||||
|
||||
@abstractmethod
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
...
|
||||
|
||||
async def health(self) -> dict:
|
||||
"""Quick connectivity check."""
|
||||
with execution_log.track(self.name, "health") as ctx:
|
||||
try:
|
||||
results = await self.search("test", max_results=1)
|
||||
ctx.result_count = len(results)
|
||||
return {"name": self.name, "status": "ok", "count": len(results)}
|
||||
except Exception as e:
|
||||
ctx.success = False
|
||||
ctx.error = str(e)
|
||||
ctx.error_type = type(e).__name__
|
||||
return {"name": self.name, "status": "fail", "error": str(e)}
|
||||
__all__ = ["BaseEngine", "SearchResult"]
|
||||
@@ -10,7 +10,7 @@ from urllib.parse import quote_plus
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,7 +28,7 @@ class BingScraper(BaseEngine):
|
||||
name = "bing"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
region: str = "global", **kwargs) -> list[EngineResult]:
|
||||
region: str = "global", **kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
url = f"https://cn.bing.com/search?q={quote_plus(query)}&setlang=zh-CN"
|
||||
if region == "global":
|
||||
@@ -46,7 +46,7 @@ class BingScraper(BaseEngine):
|
||||
logger.warning(f"[bing] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
@@ -60,7 +60,7 @@ class BingScraper(BaseEngine):
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href:
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title, url=href, snippet=snippet, source="bing",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
|
||||
@@ -9,7 +9,7 @@ from urllib.parse import quote_plus
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -27,7 +27,7 @@ class BraveScraper(BaseEngine):
|
||||
name = "brave"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
url = f"https://search.brave.com/search?q={quote_plus(query)}&source=web"
|
||||
|
||||
@@ -43,7 +43,7 @@ class BraveScraper(BaseEngine):
|
||||
logger.warning(f"[brave] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
@@ -60,7 +60,7 @@ class BraveScraper(BaseEngine):
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href and href.startswith("http"):
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title, url=href, snippet=snippet, source="brave",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
|
||||
@@ -8,7 +8,7 @@ from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -24,7 +24,7 @@ class GitHubEngine(BaseEngine):
|
||||
name = "github"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
search_type: str = "repositories", **kwargs) -> list[EngineResult]:
|
||||
search_type: str = "repositories", **kwargs) -> list[SearchResult]:
|
||||
"""search_type: 'repositories' | 'code' | 'issues'"""
|
||||
t0 = time.time()
|
||||
endpoint = f"https://api.github.com/search/{search_type}"
|
||||
@@ -40,7 +40,7 @@ class GitHubEngine(BaseEngine):
|
||||
results = []
|
||||
for item in data.get("items", [])[:max_results]:
|
||||
if search_type == "repositories":
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=item.get("full_name", ""),
|
||||
url=item.get("html_url", ""),
|
||||
snippet=item.get("description", "") or "",
|
||||
@@ -53,7 +53,7 @@ class GitHubEngine(BaseEngine):
|
||||
},
|
||||
))
|
||||
elif search_type == "code":
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=item.get("name", ""),
|
||||
url=item.get("html_url", ""),
|
||||
snippet=item.get("text_matches", [{}])[0].get("fragment", "")
|
||||
@@ -66,7 +66,7 @@ class GitHubEngine(BaseEngine):
|
||||
},
|
||||
))
|
||||
elif search_type == "issues":
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=item.get("title", ""),
|
||||
url=item.get("html_url", ""),
|
||||
snippet=item.get("body", "")[:300] if item.get("body") else "",
|
||||
|
||||
@@ -1,40 +1,62 @@
|
||||
"""News search engine — multi-source news aggregation."""
|
||||
"""News search — multi-source aggregation (Bing News + RSS)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
from urllib.parse import quote_plus
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADERS = {
|
||||
_HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
# Free RSS feeds — no API key needed
|
||||
_RSS_FEEDS = {
|
||||
# Chinese tech/business
|
||||
"36kr": "https://36kr.com/feed",
|
||||
"少数派": "https://sspai.com/feed",
|
||||
"IT之家": "https://www.ithome.com/rss/",
|
||||
"钛媒体": "https://www.tmtpost.com/rss.xml",
|
||||
"Solidot": "https://www.solidot.org/index.rss",
|
||||
"InfoQ CN": "https://www.infoq.cn/feed",
|
||||
"OSChina": "https://www.oschina.net/news/rss",
|
||||
# English tech
|
||||
"HackerNews": "https://hnrss.org/best?count=20",
|
||||
"TechCrunch": "https://techcrunch.com/feed/",
|
||||
"ArsTechnica": "https://feeds.arstechnica.com/arstechnica/index",
|
||||
"TheVerge": "https://www.theverge.com/rss/index.xml",
|
||||
"MIT Tech Review": "https://www.technologyreview.com/feed/",
|
||||
"Wired": "https://www.wired.com/feed/rss",
|
||||
"Lobste.rs": "https://lobste.rs/rss",
|
||||
# English general
|
||||
"ChinaDaily": "https://www.chinadaily.com.cn/rss/china_rss.xml",
|
||||
}
|
||||
|
||||
|
||||
class BingNewsEngine(BaseEngine):
|
||||
"""Bing News scraping — good for both CN and EN news."""
|
||||
"""Bing News scraping — works for both CN and global news."""
|
||||
|
||||
name = "bing_news"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
region: str = "cn", **kwargs) -> list[EngineResult]:
|
||||
region: str = "cn", **kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
base = "cn.bing.com" if region == "cn" else "www.bing.com"
|
||||
url = f"https://{base}/news/search?q={quote_plus(query)}&setlang=zh-CN"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp = await client.get(url, headers=_HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
@@ -44,20 +66,20 @@ class BingNewsEngine(BaseEngine):
|
||||
logger.warning(f"[bing_news] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
for item in soup.select("a.title"):
|
||||
title = item.get_text(strip=True)
|
||||
href = item.get("href", "")
|
||||
# Try to get snippet from parent container
|
||||
parent = item.find_parent("div", class_=True)
|
||||
snippet = ""
|
||||
if parent:
|
||||
snippet_el = parent.select_one("div.snippet")
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
if title and href and href.startswith("http"):
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title, url=href, snippet=snippet,
|
||||
source="bing_news", category="news",
|
||||
))
|
||||
@@ -67,40 +89,18 @@ class BingNewsEngine(BaseEngine):
|
||||
|
||||
|
||||
class RSSNewsEngine(BaseEngine):
|
||||
"""RSS-based news — multi-source aggregation."""
|
||||
"""RSS feed aggregation — multi-source news from curated feeds."""
|
||||
|
||||
name = "rss_news"
|
||||
|
||||
# Free RSS feeds, no key needed
|
||||
FEEDS = {
|
||||
# Chinese tech/business
|
||||
"36kr": "https://36kr.com/feed",
|
||||
"少数派": "https://sspai.com/feed",
|
||||
"IT之家": "https://www.ithome.com/rss/",
|
||||
"钛媒体": "https://www.tmtpost.com/rss.xml",
|
||||
"Solidot": "https://www.solidot.org/index.rss",
|
||||
"InfoQ CN": "https://www.infoq.cn/feed",
|
||||
"OSChina": "https://www.oschina.net/news/rss",
|
||||
# English tech
|
||||
"HackerNews": "https://hnrss.org/best?count=20",
|
||||
"TechCrunch": "https://techcrunch.com/feed/",
|
||||
"ArsTechnica": "https://feeds.arstechnica.com/arstechnica/index",
|
||||
"TheVerge": "https://www.theverge.com/rss/index.xml",
|
||||
"MIT Tech Review": "https://www.technologyreview.com/feed/",
|
||||
"Wired": "https://www.wired.com/feed/rss",
|
||||
"Lobste.rs": "https://lobste.rs/rss",
|
||||
# English general
|
||||
"ChinaDaily": "https://www.chinadaily.com.cn/rss/china_rss.xml",
|
||||
}
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
all_results = []
|
||||
|
||||
import asyncio
|
||||
tasks = [self._fetch_feed(name, url, query)
|
||||
for name, url in self.FEEDS.items()]
|
||||
for name, url in _RSS_FEEDS.items()]
|
||||
feeds = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
for feed_results in feeds:
|
||||
@@ -111,52 +111,45 @@ class RSSNewsEngine(BaseEngine):
|
||||
logger.info(f"[rss_news] {len(all_results)} results in {elapsed:.2f}s")
|
||||
return all_results[:max_results]
|
||||
|
||||
async def _fetch_feed(self, name: str, url: str, query: str) -> list[EngineResult]:
|
||||
async def _fetch_feed(self, name: str, url: str, query: str) -> list[SearchResult]:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp = await client.get(url, headers=_HEADERS)
|
||||
resp.raise_for_status()
|
||||
return self._parse_feed(name, resp.text, query)
|
||||
except Exception as e:
|
||||
logger.warning(f"[rss_news:{name}] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_feed(self, name: str, xml_text: str, query: str) -> list[EngineResult]:
|
||||
def _parse_feed(self, name: str, xml_text: str, query: str) -> list[SearchResult]:
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError:
|
||||
return []
|
||||
|
||||
results = []
|
||||
# RSS 2.0: //channel/item, Atom: //entry
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||
|
||||
items = root.findall(".//item") or root.findall(".//atom:entry", ns)
|
||||
|
||||
q_lower = query.lower()
|
||||
q_words = [w for w in q_lower.split() if len(w) > 1]
|
||||
|
||||
results = []
|
||||
for item in items[:50]:
|
||||
title = (item.findtext("title") or
|
||||
item.findtext("atom:title", namespaces=ns) or "")
|
||||
link = (item.findtext("link") or "")
|
||||
link = item.findtext("link") or ""
|
||||
if not link:
|
||||
link_el = item.find("atom:link", ns)
|
||||
link = link_el.get("href", "") if link_el is not None else ""
|
||||
desc = (item.findtext("description") or
|
||||
item.findtext("atom:summary", namespaces=ns) or "")
|
||||
# Clean HTML tags from description
|
||||
import re
|
||||
desc = re.sub(r'<[^>]+>', '', desc)
|
||||
|
||||
# Keyword scoring (not strict filter)
|
||||
text_lower = f"{title} {desc}".lower()
|
||||
if q_words:
|
||||
match_count = sum(1 for w in q_words if w in text_lower)
|
||||
if match_count == 0:
|
||||
continue # Skip completely irrelevant items
|
||||
if q_words and not any(w in text_lower for w in q_words):
|
||||
continue # Skip items with no query keyword match
|
||||
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title.strip(),
|
||||
url=link.strip(),
|
||||
snippet=desc[:300].strip(),
|
||||
@@ -168,15 +161,15 @@ class RSSNewsEngine(BaseEngine):
|
||||
|
||||
|
||||
class NewsSearch(BaseEngine):
|
||||
"""Unified news search — Bing News + RSS in parallel."""
|
||||
"""Unified news search — Bing News + RSS feeds in parallel."""
|
||||
|
||||
name = "news"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
region: str = "cn", **kwargs) -> list[EngineResult]:
|
||||
region: str = "cn", **kwargs) -> list[SearchResult]:
|
||||
import asyncio
|
||||
bing = BingNewsEngine(timeout=self.timeout)
|
||||
rss = RSSNewsEngine(timeout=8.0) # RSS feeds may be slow
|
||||
rss = RSSNewsEngine(timeout=8.0)
|
||||
|
||||
bing_results, rss_results = await asyncio.gather(
|
||||
bing.search(query, max_results=max_results, region=region),
|
||||
|
||||
@@ -8,7 +8,7 @@ import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
from scripts.engines.bing_scraper import BingScraper
|
||||
from scripts.engines.sogou_scraper import SogouScraper
|
||||
from scripts.engines.baidu_scraper import BaiduScraper
|
||||
@@ -59,7 +59,7 @@ class ParallelSearch:
|
||||
max_results: int = 10,
|
||||
region: str = "cn",
|
||||
**kwargs,
|
||||
) -> tuple[list[EngineResult], SearchStats]:
|
||||
) -> tuple[list[SearchResult], SearchStats]:
|
||||
"""
|
||||
Search across specified engines in parallel.
|
||||
|
||||
@@ -138,7 +138,7 @@ class ParallelSearch:
|
||||
|
||||
async def _run_engine(
|
||||
self, engine: BaseEngine, query: str, max_results: int, kwargs: dict
|
||||
) -> tuple[list[EngineResult], float]:
|
||||
) -> tuple[list[SearchResult], float]:
|
||||
"""Run a single engine with timing."""
|
||||
with execution_log.track(engine.name, "search", query=query) as ctx:
|
||||
t0 = time.time()
|
||||
@@ -162,7 +162,7 @@ class ParallelSearch:
|
||||
vertical: str,
|
||||
max_results: int = 10,
|
||||
**kwargs,
|
||||
) -> tuple[list[EngineResult], SearchStats]:
|
||||
) -> tuple[list[SearchResult], SearchStats]:
|
||||
"""Search a specific vertical (news/academic/github)."""
|
||||
if vertical == "github":
|
||||
# GitHub supports sub-types
|
||||
|
||||
@@ -1,80 +1,114 @@
|
||||
"""
|
||||
Parsing-based search engines (e.g. DuckDuckGo HTML scraper).
|
||||
"""
|
||||
"""Parsing-based search engines — DuckDuckGo, Brave (fallback HTML scraper)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_HEADERS = {
|
||||
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/125.0.0.0 Safari/537.36"),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
class DuckDuckGoEngine(SearchEngine):
|
||||
"""DuckDuckGo HTML search engine (no API key required)."""
|
||||
|
||||
SEARCH_URL = "https://html.duckduckgo.com/html/"
|
||||
class DuckDuckGoParser(BaseEngine):
|
||||
"""DuckDuckGo HTML scraper (no API key needed)."""
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("DuckDuckGo", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
name = "duckduckgo"
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
"""Build DuckDuckGo search URL."""
|
||||
from urllib.parse import quote_plus
|
||||
return f"https://duckduckgo.com/?q={quote_plus(query)}"
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[SearchResult]:
|
||||
import time
|
||||
t0 = time.time()
|
||||
url = f"https://html.duckduckgo.com/html/?q={quote_plus(query)}"
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search DuckDuckGo and parse HTML results."""
|
||||
params = {"q": query}
|
||||
try:
|
||||
response = await self._client.post(self.SEARCH_URL, data=params)
|
||||
response.raise_for_status()
|
||||
return self._parse_results(response.text, max_results)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"DuckDuckGo search failed: {e}")
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=_HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
logger.info(f"[duckduckgo] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[duckduckgo] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_results(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
"""Parse DuckDuckGo HTML search results."""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
for i, result in enumerate(soup.select(".result")[:max_results]):
|
||||
for result in soup.select(".result")[:max_results]:
|
||||
title_el = result.select_one(".result__title a")
|
||||
snippet_el = result.select_one(".result__snippet")
|
||||
if title_el:
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title_el.get_text(strip=True),
|
||||
url=title_el.get("href", ""),
|
||||
snippet=snippet_el.get_text(strip=True)
|
||||
if snippet_el else "",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="web",
|
||||
)
|
||||
)
|
||||
if not title_el:
|
||||
continue
|
||||
results.append(SearchResult(
|
||||
title=title_el.get_text(strip=True),
|
||||
url=title_el.get("href", ""),
|
||||
snippet=snippet_el.get_text(strip=True) if snippet_el else "",
|
||||
source="duckduckgo",
|
||||
category="web",
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
class BraveParser(BaseEngine):
|
||||
"""Brave Search fallback HTML scraper (no API key needed)."""
|
||||
|
||||
name = "brave"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[SearchResult]:
|
||||
import time
|
||||
t0 = time.time()
|
||||
url = f"https://search.brave.com/search?q={quote_plus(query)}&source=web"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=_HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
logger.info(f"[brave] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[brave] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
for item in soup.select("div.snippet, div.result")[:max_results]:
|
||||
title_el = item.select_one("a.result-header, a.snippet-title, a")
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
href = title_el.get("href", "")
|
||||
snippet_el = item.select_one("p.snippet-description, div.snippet-description")
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href and href.startswith("http"):
|
||||
results.append(SearchResult(
|
||||
title=title, url=href, snippet=snippet,
|
||||
source="brave", category="web",
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# Backwards-compatibility aliases
|
||||
DuckDuckGoEngine = DuckDuckGoParser
|
||||
BraveScraper = BraveParser
|
||||
@@ -9,7 +9,7 @@ from urllib.parse import quote_plus
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -27,7 +27,7 @@ class Search360Scraper(BaseEngine):
|
||||
name = "360"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
url = f"https://www.so.com/s?q={quote_plus(query)}"
|
||||
|
||||
@@ -43,7 +43,7 @@ class Search360Scraper(BaseEngine):
|
||||
logger.warning(f"[360] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
@@ -57,7 +57,7 @@ class Search360Scraper(BaseEngine):
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href:
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title, url=href, snippet=snippet, source="360",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
|
||||
@@ -9,7 +9,7 @@ from urllib.parse import quote_plus
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -27,7 +27,7 @@ class SogouScraper(BaseEngine):
|
||||
name = "sogou"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
url = f"https://www.sogou.com/web?query={quote_plus(query)}"
|
||||
|
||||
@@ -43,7 +43,7 @@ class SogouScraper(BaseEngine):
|
||||
logger.warning(f"[sogou] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
@@ -57,7 +57,7 @@ class SogouScraper(BaseEngine):
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href and not href.startswith("/"):
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title, url=href, snippet=snippet, source="sogou",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
|
||||
@@ -1,31 +1,31 @@
|
||||
"""
|
||||
Generic URL-based search engine.
|
||||
|
||||
Builds search URLs from templates defined in engines.json
|
||||
Builds search URLs from templates (config/engines.json)
|
||||
and returns them as URL results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import quote_plus
|
||||
from typing import Any
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
|
||||
class UrlSearchEngine(SearchEngine):
|
||||
class UrlSearchEngine(BaseEngine):
|
||||
"""
|
||||
Generic search engine that builds URLs from templates.
|
||||
Generic engine that builds search URLs from templates.
|
||||
|
||||
Reads URL templates from config (engines.json) and returns
|
||||
the constructed search URL as a SearchResult.
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, engine_def: dict[str, Any]):
|
||||
super().__init__(name, engine_def.get("config", {}))
|
||||
name: str = "url"
|
||||
|
||||
def __init__(self, name: str, engine_def: dict[str, Any], timeout: float = 10.0):
|
||||
super().__init__(timeout=timeout)
|
||||
self.name = name
|
||||
self.url_template = engine_def.get("url", "")
|
||||
self.region = engine_def.get("region", "global")
|
||||
|
||||
@@ -55,6 +55,3 @@ class UrlSearchEngine(SearchEngine):
|
||||
extra={"url": url, "region": self.region},
|
||||
)
|
||||
]
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
+12
-23
@@ -1,25 +1,22 @@
|
||||
"""
|
||||
Pydantic 数据模型定义。
|
||||
Data models — re-exports from scripts.core.base for backwards compatibility.
|
||||
|
||||
从 bot-search-mcp 移植,适配 web-search-skills 的数据结构。
|
||||
提供类型安全的搜索数据模型。
|
||||
|
||||
依赖: pydantic (>=2.0)
|
||||
The canonical source of truth is scripts.core.base.SearchResult.
|
||||
This module exists so old import paths (e.g. from scripts.models import SearchQuery)
|
||||
continue to work without changes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 搜索结果模型(兼容 dataclass + pydantic 双模式)
|
||||
# ============================================================
|
||||
# Backwards-compatible thin wrappers (no functionality changes needed for now)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchQuery:
|
||||
"""搜索查询封装。"""
|
||||
"""Search query wrapper."""
|
||||
query: str
|
||||
sources: Optional[list[str]] = None
|
||||
max_results: int = 10
|
||||
@@ -31,20 +28,16 @@ class SearchQuery:
|
||||
|
||||
@dataclass
|
||||
class SourceRef:
|
||||
"""引用来源。"""
|
||||
"""Reference to a result source."""
|
||||
index: int
|
||||
title: str
|
||||
url: str
|
||||
source: str
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 性能计时模型
|
||||
# ============================================================
|
||||
|
||||
@dataclass
|
||||
class EngineTiming:
|
||||
"""单个引擎耗时记录。"""
|
||||
"""Per-engine timing record."""
|
||||
engine_name: str
|
||||
success: bool
|
||||
duration_ms: int
|
||||
@@ -54,7 +47,7 @@ class EngineTiming:
|
||||
|
||||
@dataclass
|
||||
class SearchMetadata:
|
||||
"""搜索元数据(全链路计时)。"""
|
||||
"""Search metadata (full-chain timing)."""
|
||||
total_engines: int = 0
|
||||
total_raw_results: int = 0
|
||||
engines_used: list[str] = field(default_factory=list)
|
||||
@@ -63,13 +56,9 @@ class SearchMetadata:
|
||||
timed_out_engines: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 最终响应
|
||||
# ============================================================
|
||||
|
||||
@dataclass
|
||||
class SearchResponse:
|
||||
"""完整搜索响应。"""
|
||||
"""Complete search response."""
|
||||
query: str
|
||||
results: list[Any] = field(default_factory=list)
|
||||
sources: list[SourceRef] = field(default_factory=list)
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""News search engines."""
|
||||
@@ -1,186 +0,0 @@
|
||||
"""
|
||||
财联社 (cls.cn) news search engine.
|
||||
|
||||
Uses the public telegraph API (nodeapi/telegraphList) to fetch real-time
|
||||
financial news without JS rendering. Falls back to search page URL for
|
||||
keyword-based search queries.
|
||||
|
||||
API details (discovered via frontend reverse-engineering):
|
||||
GET https://www.cls.cn/nodeapi/telegraphList
|
||||
- Parameters: app=CailianpressWeb, os=web, sv=7.7.5, rn=N, last_time=timestamp
|
||||
- Sign: SHA1(plaintext) → MD5(hex) (anti-scraping measure)
|
||||
- Returns: {"data":{"roll_data":[{"ctime":..., "content":"..."}]}}
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
API_URL = "https://www.cls.cn/nodeapi/telegraphList"
|
||||
SEARCH_URL = "https://www.cls.cn/searchPage"
|
||||
API_APP = "CailianpressWeb"
|
||||
API_OS = "web"
|
||||
API_SV = "7.7.5"
|
||||
|
||||
|
||||
class CLSNewsEngine(SearchEngine):
|
||||
"""财联社 (cls.cn) news search engine."""
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("cls", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Referer": "https://www.cls.cn/telegraph",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
return f"{SEARCH_URL}?keyword={quote_plus(query)}&type=all"
|
||||
|
||||
def _build_api_sign(self, last_time: int, rn: int) -> str:
|
||||
"""Build SHA1 + MD5 double-hashed sign for API request."""
|
||||
plain = (
|
||||
f"app={API_APP}&last_time={last_time}"
|
||||
f"&os={API_OS}&rn={rn}&sv={API_SV}"
|
||||
)
|
||||
sha1_hex = hashlib.sha1(plain.encode()).hexdigest()
|
||||
return hashlib.md5(sha1_hex.encode()).hexdigest()
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search 财联社.
|
||||
|
||||
Uses the telegraph API for latest news (ignores query).
|
||||
For keyword-specific results, falls back to the search page URL.
|
||||
"""
|
||||
# Step 1: Try to get real results from telegraph API
|
||||
api_results = await self._fetch_telegraph(max_results)
|
||||
if api_results:
|
||||
return api_results[:max_results]
|
||||
|
||||
# Step 2: Fallback to search page URL
|
||||
search_url = self.build_url(query)
|
||||
return [
|
||||
SearchResult(
|
||||
title=f"财联社: {query}",
|
||||
url=search_url,
|
||||
snippet=(
|
||||
f"Search 财联社 for '{query}' "
|
||||
"(JS-required search page)"
|
||||
),
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="news",
|
||||
extra={"search_url": search_url, "note": "JS-rendered page"},
|
||||
)
|
||||
]
|
||||
|
||||
async def _fetch_telegraph(
|
||||
self, max_results: int = 10
|
||||
) -> list[SearchResult]:
|
||||
"""Fetch real-time telegraph news via API."""
|
||||
try:
|
||||
rn = min(max(max_results, 5), 100)
|
||||
last_time = int(time.time())
|
||||
sign = self._build_api_sign(last_time, rn)
|
||||
|
||||
params = {
|
||||
"app": API_APP,
|
||||
"last_time": str(last_time),
|
||||
"os": API_OS,
|
||||
"rn": str(rn),
|
||||
"sv": API_SV,
|
||||
"sign": sign,
|
||||
}
|
||||
|
||||
response = await self._client.get(API_URL, params=params)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
items = data.get("data", {}).get("roll_data", [])
|
||||
|
||||
results = []
|
||||
for i, item in enumerate(items[:max_results]):
|
||||
content = item.get("content", "")
|
||||
ctime = item.get("ctime", 0)
|
||||
|
||||
# Extract title from content (财联社 format:
|
||||
# "【title】content" or plain text)
|
||||
title, snippet = self._parse_content(content)
|
||||
|
||||
# Use unique URL per item to avoid dedup collision
|
||||
item_url = (
|
||||
f"https://www.cls.cn/telegraph#{ctime}-{i}"
|
||||
)
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title or content[:80],
|
||||
url=item_url,
|
||||
snippet=snippet or content[:200],
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={
|
||||
"time": self._format_time(ctime),
|
||||
"timestamp": ctime,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
except (httpx.HTTPError, ValueError, KeyError) as e:
|
||||
logger.warning(f"财联社 API failed: {e}")
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _parse_content(content: str) -> tuple[str, str]:
|
||||
"""Parse 财联社 content into (title, snippet).
|
||||
|
||||
Format examples:
|
||||
"【日本地震】内容详情..." → ("日本地震", "内容详情...")
|
||||
"财联社5月20日电,普通新闻" → ("", "财联社5月20日电,普通新闻")
|
||||
"""
|
||||
if content.startswith("【"):
|
||||
end = content.find("】")
|
||||
if end != -1:
|
||||
title = content[1:end]
|
||||
snippet = content[end + 1:].strip()
|
||||
return title, snippet
|
||||
return "", content
|
||||
|
||||
@staticmethod
|
||||
def _format_time(ts: int) -> str:
|
||||
"""Format Unix timestamp to readable time."""
|
||||
import datetime
|
||||
|
||||
try:
|
||||
dt = datetime.datetime.fromtimestamp(
|
||||
ts,
|
||||
tz=datetime.timezone(datetime.timedelta(hours=8)),
|
||||
)
|
||||
return dt.strftime("%Y-%m-%d %H:%M")
|
||||
except (OSError, ValueError):
|
||||
return str(ts)
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
@@ -1,367 +0,0 @@
|
||||
"""
|
||||
RSS news aggregation engine.
|
||||
|
||||
Fetches and parses RSS/Atom feeds from trusted news sources.
|
||||
Inspired by the news-summary skill's RSS-based approach.
|
||||
|
||||
Supported sources:
|
||||
- BBC (World, Business, Technology, Top Stories)
|
||||
- Reuters (World News)
|
||||
- NPR (US news)
|
||||
- Al Jazeera (Global South perspective)
|
||||
- 财联社 (RSS if available)
|
||||
- 华尔街见闻 (RSS if available)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# RSS feed definitions
|
||||
# Core sources (general news) + AI newsletters + essays + podcasts
|
||||
RSS_FEEDS = {
|
||||
"BBC World": {
|
||||
"url": "https://feeds.bbci.co.uk/news/world/rss.xml",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
},
|
||||
"BBC Top Stories": {
|
||||
"url": "https://feeds.bbci.co.uk/news/rss.xml",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
},
|
||||
"BBC Business": {
|
||||
"url": "https://feeds.bbci.co.uk/news/business/rss.xml",
|
||||
"category": "business",
|
||||
"lang": "en",
|
||||
},
|
||||
"BBC Technology": {
|
||||
"url": "https://feeds.bbci.co.uk/news/technology/rss.xml",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
},
|
||||
"Reuters": {
|
||||
"url": "https://www.reutersagency.com/feed/?best-regions=world&post_type=best",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
},
|
||||
"NPR": {
|
||||
"url": "https://feeds.npr.org/1001/rss.xml",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
},
|
||||
"Al Jazeera": {
|
||||
"url": "https://www.aljazeera.com/xml/rss/all.xml",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
},
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# AI Newsletters (inspired by news-aggregator-skill)
|
||||
# -----------------------------------------------------------------------
|
||||
"Latent Space": {
|
||||
"url": "https://feeds.transistor.fm/latent-space-podcast",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "ai_newsletter",
|
||||
},
|
||||
"ChinAI": {
|
||||
"url": "https://chinai.substack.com/feed",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "ai_newsletter",
|
||||
},
|
||||
"Memia": {
|
||||
"url": "https://memia.substack.com/feed",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "ai_newsletter",
|
||||
},
|
||||
"Interconnects": {
|
||||
"url": "https://interconnects.substack.com/feed",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "ai_newsletter",
|
||||
},
|
||||
"KDnuggets": {
|
||||
"url": "https://www.kdnuggets.com/feed",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "ai_newsletter",
|
||||
},
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Deep Essays & Thinkers (inspired by news-aggregator-skill)
|
||||
# -----------------------------------------------------------------------
|
||||
"Paul Graham": {
|
||||
"url": "https://paulgraham.com/rss.xml",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "essay",
|
||||
},
|
||||
"James Clear": {
|
||||
"url": "https://jamesclear.com/feed",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
"subcategory": "essay",
|
||||
},
|
||||
"Wait But Why": {
|
||||
"url": "https://waitbutwhy.com/feed",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
"subcategory": "essay",
|
||||
},
|
||||
"Farnam Street": {
|
||||
"url": "https://fs.blog/feed/",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
"subcategory": "essay",
|
||||
},
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Podcasts (inspired by news-aggregator-skill)
|
||||
# -----------------------------------------------------------------------
|
||||
"Lex Fridman Podcast": {
|
||||
"url": "https://lexfridman.com/feed/podcast/",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "podcast",
|
||||
},
|
||||
"80,000 Hours": {
|
||||
"url": "https://80000hours.org/feed/podcast/",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
"subcategory": "podcast",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class RSSNewsEngine(SearchEngine):
|
||||
"""
|
||||
RSS-based news aggregation engine.
|
||||
|
||||
Fetches and parses RSS/Atom feeds from international news sources.
|
||||
Supports filtering by category, keyword, and time range.
|
||||
"""
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("RSS News", config)
|
||||
timeout = (config or {}).get("timeout", 30)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
"Accept": "application/rss+xml, application/xml, text/xml, */*",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
"""Build search URL - for RSS this is not a search URL but a feed URL."""
|
||||
feed_name = kwargs.get("feed", "BBC World")
|
||||
feed = RSS_FEEDS.get(feed_name, RSS_FEEDS["BBC World"])
|
||||
return feed["url"]
|
||||
|
||||
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
"""
|
||||
Search news across RSS feeds. If query is empty, returns latest headlines.
|
||||
|
||||
Args:
|
||||
query: Search keyword to filter articles (case-insensitive).
|
||||
If empty/blank, returns all headlines.
|
||||
max_results: Maximum articles per feed.
|
||||
feeds: Specific feeds to query (list of feed names or "all").
|
||||
category: Filter by category (news, business, technology).
|
||||
hours: Only include articles from the last N hours.
|
||||
|
||||
Returns:
|
||||
List of SearchResult objects sorted by date (newest first).
|
||||
"""
|
||||
feeds_to_query = self._select_feeds(
|
||||
feed_names=kwargs.get("feeds", "all"),
|
||||
category=kwargs.get("category"),
|
||||
)
|
||||
|
||||
results = []
|
||||
for feed_name, feed_info in feeds_to_query.items():
|
||||
try:
|
||||
response = await self._client.get(feed_info["url"])
|
||||
response.raise_for_status()
|
||||
|
||||
parsed = self._parse_feed(response.text, feed_name, feed_info)
|
||||
for article in parsed:
|
||||
# Filter by keyword if query provided
|
||||
if query:
|
||||
q = query.lower()
|
||||
if q in article.title.lower() or q in article.snippet.lower():
|
||||
results.append(article)
|
||||
else:
|
||||
results.append(article)
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
logger.warning(f"RSS feed '{feed_name}' failed: {e}")
|
||||
continue
|
||||
|
||||
# Apply time filter
|
||||
hours = kwargs.get("hours")
|
||||
if hours:
|
||||
from datetime import timedelta
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(hours=int(hours))
|
||||
results = [r for r in results if r.extra.get("parsed_date", datetime.max.replace(tzinfo=timezone.utc)) >= cutoff]
|
||||
|
||||
# Sort by date (newest first), limit results
|
||||
results.sort(key=lambda r: r.extra.get("parsed_date", datetime.min.replace(tzinfo=timezone.utc)), reverse=True)
|
||||
return results[:max_results * max(1, len(feeds_to_query))]
|
||||
|
||||
def _select_feeds(self, feed_names="all", category=None):
|
||||
"""Select feeds based on filters."""
|
||||
if feed_names != "all":
|
||||
if isinstance(feed_names, str):
|
||||
feed_names = [feed_names]
|
||||
return {k: v for k, v in RSS_FEEDS.items() if k in feed_names}
|
||||
|
||||
if category:
|
||||
return {k: v for k, v in RSS_FEEDS.items() if v.get("category") == category}
|
||||
|
||||
return RSS_FEEDS
|
||||
|
||||
def _parse_feed(self, xml_text: str, feed_name: str, feed_info: dict) -> list[SearchResult]:
|
||||
"""Parse RSS/Atom XML into SearchResult objects."""
|
||||
results = []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError as e:
|
||||
logger.error(f"Failed to parse RSS from {feed_name}: {e}")
|
||||
return []
|
||||
|
||||
# Detect RSS vs Atom format
|
||||
if root.tag == "rss":
|
||||
return self._parse_rss(root, feed_name, feed_info)
|
||||
elif root.tag.endswith("feed"):
|
||||
return self._parse_atom(root, feed_name, feed_info)
|
||||
else:
|
||||
logger.warning(f"Unknown feed format for {feed_name}: {root.tag}")
|
||||
return results
|
||||
|
||||
def _parse_rss(self, root: ET.Element, feed_name: str, feed_info: dict) -> list[SearchResult]:
|
||||
"""Parse RSS 2.0 format."""
|
||||
results = []
|
||||
namespaces = {"content": "http://purl.org/rss/1.0/modules/content/"}
|
||||
|
||||
for i, item in enumerate(root.findall(".//item")):
|
||||
title = item.findtext("title", "")
|
||||
link = item.findtext("link", "")
|
||||
description = item.findtext("description", "")
|
||||
pub_date_str = item.findtext("pubDate", "")
|
||||
creator = item.findtext("dc:creator", "") or item.findtext("author", "")
|
||||
|
||||
# Clean up description (remove HTML tags)
|
||||
import re
|
||||
snippet = re.sub(r"<[^>]+>", "", description).strip()
|
||||
# Truncate
|
||||
snippet = snippet[:300] + "..." if len(snippet) > 300 else snippet
|
||||
|
||||
# Parse date
|
||||
parsed_date = self._parse_date(pub_date_str)
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title,
|
||||
url=link,
|
||||
snippet=snippet,
|
||||
source=feed_name,
|
||||
rank=i + 1,
|
||||
category=feed_info.get("category", "news"),
|
||||
extra={
|
||||
"feed_name": feed_name,
|
||||
"author": creator,
|
||||
"published": pub_date_str,
|
||||
"parsed_date": parsed_date,
|
||||
"feed_url": feed_info["url"],
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _parse_atom(self, root: ET.Element, feed_name: str, feed_info: dict) -> list[SearchResult]:
|
||||
"""Parse Atom format."""
|
||||
results = []
|
||||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||||
|
||||
feed_title = root.findtext("a:title", "", ns)
|
||||
|
||||
for i, entry in enumerate(root.findall("a:entry", ns)):
|
||||
title = entry.findtext("a:title", "", ns)
|
||||
link_el = entry.find("a:link", ns)
|
||||
link = link_el.get("href", "") if link_el is not None else ""
|
||||
summary = entry.findtext("a:summary", "", ns) or entry.findtext("a:content", "", ns)
|
||||
published = entry.findtext("a:published", "", ns) or entry.findtext("a:updated", "", ns)
|
||||
author = entry.findtext("a:author/a:name", "", ns)
|
||||
|
||||
import re
|
||||
snippet = re.sub(r"<[^>]+>", "", summary).strip()
|
||||
snippet = snippet[:300] + "..." if len(snippet) > 300 else snippet
|
||||
|
||||
parsed_date = self._parse_date(published)
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title,
|
||||
url=link,
|
||||
snippet=snippet,
|
||||
source=feed_name,
|
||||
rank=i + 1,
|
||||
category=feed_info.get("category", "news"),
|
||||
extra={
|
||||
"feed_name": feed_name,
|
||||
"author": author,
|
||||
"published": published,
|
||||
"parsed_date": parsed_date,
|
||||
"feed_url": feed_info["url"],
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _parse_date(self, date_str: str) -> datetime:
|
||||
"""Parse various date formats to datetime."""
|
||||
if not date_str:
|
||||
return datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
formats = [
|
||||
"%a, %d %b %Y %H:%M:%S %z", # RSS: Sat, 01 Jan 2024 12:00:00 +0000
|
||||
"%a, %d %b %Y %H:%M:%S %Z", # RSS with timezone name
|
||||
"%Y-%m-%dT%H:%M:%S%z", # Atom ISO 8601
|
||||
"%Y-%m-%dT%H:%M:%S.%f%z", # Atom with microseconds
|
||||
"%Y-%m-%dT%H:%M:%SZ", # Atom UTC
|
||||
"%Y-%m-%d", # Simple date
|
||||
]
|
||||
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(date_str.strip(), fmt).replace(tzinfo=timezone.utc)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
return datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
@@ -1,461 +0,0 @@
|
||||
"""
|
||||
Tech & social news aggregation engines.
|
||||
|
||||
Implements non-Playwright news sources inspired by news-aggregator-skill:
|
||||
- Hacker News (via Algolia API)
|
||||
- GitHub Trending (HTML scrape)
|
||||
- 36氪 (HTML scrape)
|
||||
- 微博热搜 (Ajax API)
|
||||
- V2EX (Public API)
|
||||
- 腾讯新闻 (API)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
from scripts.core.thread_utils import run_in_thread
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Shared HTTP client settings
|
||||
_DEFAULT_TIMEOUT = 15
|
||||
_COMMON_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hacker News (via Algolia API)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class HackerNewsEngine(SearchEngine):
|
||||
"""Hacker News search via Algolia public API."""
|
||||
|
||||
SEARCH_URL = "https://hn.algolia.com/api/v1/search"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("Hacker News", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers={"User-Agent": _COMMON_HEADERS["User-Agent"]},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
try:
|
||||
params = {
|
||||
"query": query,
|
||||
"hitsPerPage": max_results,
|
||||
"tags": "story",
|
||||
}
|
||||
response = await self._client.get(self.SEARCH_URL, params=params)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return self._parse_results(data, max_results)
|
||||
except Exception as e:
|
||||
logger.error(f"Hacker News search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_results(self, data: dict, max_results: int) -> list[SearchResult]:
|
||||
results = []
|
||||
hits = data.get("hits", [])[:max_results]
|
||||
for i, hit in enumerate(hits):
|
||||
title = hit.get("title", "")
|
||||
url = hit.get("url") or f"https://news.ycombinator.com/item?id={hit.get('objectID', '')}"
|
||||
snippet = hit.get("story_text", "") or ""
|
||||
# Clean HTML from snippet
|
||||
snippet = re.sub(r"<[^>]+>", "", snippet)[:200]
|
||||
points = hit.get("points", 0)
|
||||
author = hit.get("author", "")
|
||||
created_at = hit.get("created_at", "")
|
||||
|
||||
results.append(SearchResult(
|
||||
title=title,
|
||||
url=url,
|
||||
snippet=snippet or f"[{points} points] by {author}",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={
|
||||
"points": points,
|
||||
"author": author,
|
||||
"time": created_at,
|
||||
"heat": f"{points} points",
|
||||
},
|
||||
))
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GitHub Trending
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class GitHubTrendingEngine(SearchEngine):
|
||||
"""GitHub Trending repositories."""
|
||||
|
||||
TRENDING_URL = "https://github.com/trending"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("GitHub Trending", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers=_COMMON_HEADERS,
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str = "", max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
"""GitHub Trending doesn't support keyword search; returns top trending repos."""
|
||||
url = self.TRENDING_URL
|
||||
if query:
|
||||
url += f"?spoken_language_code=&q={quote_plus(query)}"
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return await run_in_thread(self._parse_html, response.text, max_results)
|
||||
except Exception as e:
|
||||
logger.error(f"GitHub Trending failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_html(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
articles = soup.select("article.Box-row")[:max_results]
|
||||
for i, article in enumerate(articles):
|
||||
h2 = article.select_one("h2")
|
||||
if not h2:
|
||||
continue
|
||||
repo_path = h2.get_text(strip=True).replace(" ", "")
|
||||
repo_url = f"https://github.com/{repo_path}"
|
||||
|
||||
desc_el = article.select_one("p")
|
||||
description = desc_el.get_text(strip=True) if desc_el else ""
|
||||
|
||||
# Language
|
||||
lang_el = article.select_one("[itemprop='programmingLanguage']")
|
||||
language = lang_el.get_text(strip=True) if lang_el else ""
|
||||
|
||||
# Stars today
|
||||
stars_el = article.select_one(".float-sm-right")
|
||||
stars_today = stars_el.get_text(strip=True) if stars_el else ""
|
||||
|
||||
results.append(SearchResult(
|
||||
title=repo_path,
|
||||
url=repo_url,
|
||||
snippet=description[:200] if description else language,
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={
|
||||
"language": language,
|
||||
"heat": stars_today,
|
||||
"time": "",
|
||||
},
|
||||
))
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 36氪 (36kr.com)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class Kr36Engine(SearchEngine):
|
||||
"""36氪 tech news search."""
|
||||
|
||||
SEARCH_URL = "https://36kr.com/search/articles/{keyword}"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("36氪", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers={**_COMMON_HEADERS, "Referer": "https://36kr.com/"},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
url = self.SEARCH_URL.format(keyword=quote_plus(query))
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return await run_in_thread(self._parse_html, response.text, query, max_results)
|
||||
except Exception as e:
|
||||
logger.error(f"36氪 search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_html(self, html: str, query: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
|
||||
# Look for article cards
|
||||
for i, item in enumerate(soup.select("a[href*='/p/'], .article-item, .search-result-item, [class*='article']")[:max_results]):
|
||||
title_el = item.select_one("h3, h4, .title, [class*='title']")
|
||||
link = item.get("href", "")
|
||||
snippet_el = item.select_one(".abstract, .summary, .desc, p, [class*='desc']")
|
||||
|
||||
title = title_el.get_text(strip=True) if title_el else ""
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if not title and not link:
|
||||
continue
|
||||
|
||||
full_url = link if link.startswith("http") else f"https://36kr.com{link}"
|
||||
|
||||
results.append(SearchResult(
|
||||
title=title or f"36氪: {query}",
|
||||
url=full_url,
|
||||
snippet=snippet[:200] if snippet else "",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={"region": "cn"},
|
||||
))
|
||||
|
||||
if not results:
|
||||
search_url = f"https://36kr.com/search/articles/{quote_plus(query)}"
|
||||
results.append(SearchResult(
|
||||
title=f"36氪: {query}",
|
||||
url=search_url,
|
||||
snippet="Search 36氪 (site may require JS)",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="news",
|
||||
extra={"search_url": search_url, "note": "JS-rendered page"},
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 微博热搜 (Weibo Trending)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class WeiboEngine(SearchEngine):
|
||||
"""微博热搜 trending topics."""
|
||||
|
||||
HOT_URL = "https://weibo.com/ajax/side/hotSearch"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("微博热搜", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers={**_COMMON_HEADERS, "Referer": "https://weibo.com/"},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str = "", max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
try:
|
||||
response = await self._client.get(self.HOT_URL)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return self._parse_results(data, query, max_results)
|
||||
except Exception as e:
|
||||
logger.error(f"微博热搜 failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_results(self, data: dict, query: str, max_results: int) -> list[SearchResult]:
|
||||
results = []
|
||||
realtime = data.get("data", {}).get("realtime", [])[:max_results]
|
||||
for i, item in enumerate(realtime):
|
||||
word = item.get("word", "")
|
||||
raw_hot = item.get("raw_hot", 0)
|
||||
url = f"https://s.weibo.com/weibo?q={quote_plus(word)}"
|
||||
|
||||
# Filter by keyword if query provided
|
||||
if query and query.lower() not in word.lower():
|
||||
continue
|
||||
|
||||
results.append(SearchResult(
|
||||
title=f"#{word}#",
|
||||
url=url,
|
||||
snippet=f"热度: {raw_hot:,}" if raw_hot else "",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={
|
||||
"heat": f"{raw_hot:,}" if raw_hot else "",
|
||||
"region": "cn",
|
||||
},
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# V2EX (via public API)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class V2EXEngine(SearchEngine):
|
||||
"""V2EX tech community hot topics."""
|
||||
|
||||
HOT_API = "https://www.v2ex.com/api/v2/topics/hot"
|
||||
# Fallback to older API if v2 needs auth
|
||||
HOT_API_FALLBACK = "https://www.v2ex.com/api/topics/hot.json"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("V2EX", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers=_COMMON_HEADERS,
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str = "", max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
# Try v2 API first, fallback to old API
|
||||
for api_url in [self.HOT_API, self.HOT_API_FALLBACK]:
|
||||
try:
|
||||
response = await self._client.get(api_url)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
# v2 API returns list directly; old API also returns list
|
||||
topics = data if isinstance(data, list) else data.get("data", [])
|
||||
return self._parse_results(topics, query, max_results)
|
||||
except Exception:
|
||||
continue
|
||||
logger.error("V2EX API failed (both v2 and fallback)")
|
||||
return []
|
||||
|
||||
def _parse_results(self, topics: list, query: str, max_results: int) -> list[SearchResult]:
|
||||
results = []
|
||||
for i, topic in enumerate(topics[:max_results]):
|
||||
title = topic.get("title", "")
|
||||
node_name = ""
|
||||
if "node" in topic and isinstance(topic["node"], dict):
|
||||
node_name = topic["node"].get("title", "")
|
||||
topic_id = topic.get("id", "")
|
||||
url = f"https://www.v2ex.com/t/{topic_id}"
|
||||
replies = topic.get("replies", 0)
|
||||
created = topic.get("created", "")
|
||||
member = ""
|
||||
if "member" in topic and isinstance(topic["member"], dict):
|
||||
member = topic["member"].get("username", "")
|
||||
|
||||
# Filter by keyword
|
||||
if query and query.lower() not in title.lower():
|
||||
continue
|
||||
|
||||
# Format time
|
||||
time_str = ""
|
||||
if created:
|
||||
try:
|
||||
dt = datetime.fromtimestamp(int(created), tz=timezone.utc)
|
||||
time_str = dt.strftime("%Y-%m-%d %H:%M")
|
||||
except (ValueError, OSError):
|
||||
time_str = str(created)
|
||||
|
||||
results.append(SearchResult(
|
||||
title=title,
|
||||
url=url,
|
||||
snippet=f"[{node_name}] by {member}" if node_name and member else f"by {member}" if member else "",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={
|
||||
"heat": f"{replies} replies",
|
||||
"author": member,
|
||||
"time": time_str,
|
||||
"region": "cn",
|
||||
},
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 腾讯新闻 (news.qq.com)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TencentNewsEngine(SearchEngine):
|
||||
"""腾讯新闻 search."""
|
||||
|
||||
SEARCH_URL = "https://news.qq.com/search"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("腾讯新闻", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers={**_COMMON_HEADERS, "Referer": "https://news.qq.com/"},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
url = f"{self.SEARCH_URL}?q={quote_plus(query)}"
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return await run_in_thread(self._parse_html, response.text, query, max_results)
|
||||
except Exception as e:
|
||||
logger.error(f"腾讯新闻 search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_html(self, html: str, query: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
|
||||
for i, item in enumerate(soup.select("a[href*='news.qq.com'], .list-item, [class*='result'], article")[:max_results]):
|
||||
title_el = item.select_one("h3, h4, .title, a")
|
||||
link = item.get("href", "") or (title_el.get("href", "") if title_el else "")
|
||||
snippet_el = item.select_one(".summary, .desc, .abstract, p, .text")
|
||||
|
||||
title = title_el.get_text(strip=True) if title_el else ""
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if not title:
|
||||
continue
|
||||
|
||||
full_url = link if link.startswith("http") else f"https:{link}" if link.startswith("//") else link
|
||||
|
||||
results.append(SearchResult(
|
||||
title=title,
|
||||
url=full_url,
|
||||
snippet=snippet[:200] if snippet else "",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={"region": "cn"},
|
||||
))
|
||||
|
||||
if not results:
|
||||
search_url = f"{self.SEARCH_URL}?q={quote_plus(query)}"
|
||||
results.append(SearchResult(
|
||||
title=f"腾讯新闻: {query}",
|
||||
url=search_url,
|
||||
snippet=f"Search Tencent News for '{query}'",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="news",
|
||||
extra={"search_url": search_url, "note": "JS-rendered page"},
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
@@ -1,117 +0,0 @@
|
||||
"""
|
||||
华尔街见闻 (wallstreetcn.com) news search engine.
|
||||
|
||||
Scrapes wallstreetcn.com search results for financial news.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WallStreetCNEngine(SearchEngine):
|
||||
"""华尔街见闻 (wallstreetcn.com) news search engine."""
|
||||
|
||||
SEARCH_URL = "https://wallstreetcn.com/search"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("wallstreetcn", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
"Referer": "https://wallstreetcn.com/",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
return f"{self.SEARCH_URL}?q={quote_plus(query)}"
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search 华尔街见闻 and parse results."""
|
||||
url = self.build_url(query)
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return self._parse_results(response.text, query, max_results)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"华尔街见闻 search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_results(
|
||||
self, html: str, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Parse 华尔街见闻 search page."""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
|
||||
articles = soup.select(
|
||||
"a[href*='/article/'], .search-result-item, article, .item"
|
||||
)
|
||||
for i, article in enumerate(articles[:max_results]):
|
||||
title_el = article.select_one("h3, h4, .title, a")
|
||||
link = article.get("href", "")
|
||||
snippet_el = article.select_one(
|
||||
".summary, .desc, .abstract, p"
|
||||
)
|
||||
|
||||
title = title_el.get_text(strip=True) if title_el else ""
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if not title and not link:
|
||||
continue
|
||||
|
||||
full_url = (
|
||||
link
|
||||
if link.startswith("http")
|
||||
else f"https://wallstreetcn.com{link}"
|
||||
)
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title or f"华尔街见闻: {query}",
|
||||
url=full_url,
|
||||
snippet=snippet or f"Search result from 华尔街见闻",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={"region": "cn"},
|
||||
)
|
||||
)
|
||||
|
||||
if not results:
|
||||
search_url = self.build_url(query)
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=f"华尔街见闻: {query}",
|
||||
url=search_url,
|
||||
snippet=f"Search 华尔街见闻 for '{query}' (site may require JS)",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="news",
|
||||
extra={"search_url": search_url, "note": "JS-rendered page"},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
@@ -1 +0,0 @@
|
||||
"""Social media search implementations (Twitter/X, etc.)."""
|
||||
@@ -1,208 +0,0 @@
|
||||
"""
|
||||
Twitter/X search module.
|
||||
|
||||
Generates search URLs for X.com (formerly Twitter) search and
|
||||
optionally parses search result pages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TwitterSearchEngine(SearchEngine):
|
||||
"""
|
||||
Twitter/X search engine.
|
||||
|
||||
Provides search URL generation and optional result scraping.
|
||||
Supports basic keyword search, user search, and advanced queries.
|
||||
"""
|
||||
|
||||
SEARCH_URL = "https://twitter.com/search"
|
||||
BASE_URL = "https://twitter.com"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("Twitter", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, search_type: str = "top", **kwargs) -> str:
|
||||
"""
|
||||
Build Twitter/X search URL.
|
||||
|
||||
Args:
|
||||
query: Search keyword.
|
||||
search_type: "top", "latest", "people", "photos", "videos"
|
||||
**kwargs: Additional URL parameters.
|
||||
|
||||
Returns:
|
||||
Search URL string.
|
||||
"""
|
||||
encoded_query = quote_plus(query)
|
||||
url = f"{self.SEARCH_URL}?q={encoded_query}&src=typed_query"
|
||||
|
||||
if search_type == "latest":
|
||||
url += "&f=live"
|
||||
elif search_type == "people":
|
||||
url += "&f=user"
|
||||
elif search_type == "photos":
|
||||
url += "&f=image"
|
||||
elif search_type == "videos":
|
||||
url += "&f=video"
|
||||
|
||||
if kwargs.get("lang"):
|
||||
url += f"&l={kwargs['lang']}"
|
||||
if kwargs.get("until"):
|
||||
url += f"&until={kwargs['until']}"
|
||||
if kwargs.get("since"):
|
||||
url += f"&since={kwargs['since']}"
|
||||
|
||||
return url
|
||||
|
||||
def build_advanced_url(self, **params) -> str:
|
||||
"""
|
||||
Build advanced Twitter search URL with operators.
|
||||
|
||||
Supported operators:
|
||||
from:username - Tweets from user
|
||||
to:username - Tweets to user
|
||||
@username - Mentions
|
||||
#hashtag - Hashtag
|
||||
filter:links - Tweets with links
|
||||
filter:images - Tweets with images
|
||||
min_retweets:N - Minimum retweets
|
||||
min_faves:N - Minimum likes
|
||||
lang:en - Language
|
||||
|
||||
Returns:
|
||||
Advanced search URL.
|
||||
"""
|
||||
parts = []
|
||||
if params.get("from_user"):
|
||||
parts.append(f"from:{params['from_user']}")
|
||||
if params.get("to_user"):
|
||||
parts.append(f"to:{params['to_user']}")
|
||||
if params.get("hashtag"):
|
||||
parts.append(f"#{params['hashtag']}")
|
||||
if params.get("keyword"):
|
||||
parts.append(params["keyword"])
|
||||
if params.get("min_retweets"):
|
||||
parts.append(f"min_retweets:{params['min_retweets']}")
|
||||
if params.get("min_faves"):
|
||||
parts.append(f"min_faves:{params['min_faves']}")
|
||||
if params.get("lang"):
|
||||
parts.append(f"lang:{params['lang']}")
|
||||
if params.get("filter"):
|
||||
parts.append(f"filter:{params['filter']}")
|
||||
|
||||
query = " ".join(parts)
|
||||
return self.build_url(query, search_type=params.get("search_type", "top"))
|
||||
|
||||
async def search(self, query: str, max_results: int = 10, search_type: str = "top", **kwargs) -> list[SearchResult]:
|
||||
"""
|
||||
Search Twitter/X and optionally parse page content.
|
||||
|
||||
Args:
|
||||
query: Search keyword.
|
||||
max_results: Maximum results (used for URL generation).
|
||||
search_type: "top", "latest", "people", "photos", "videos"
|
||||
**kwargs: Additional options.
|
||||
|
||||
Returns:
|
||||
List of SearchResult objects.
|
||||
"""
|
||||
url = self.build_url(query, search_type=search_type, **kwargs)
|
||||
|
||||
# Try to scrape the search page for basic info
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
if response.status_code == 200:
|
||||
soup = BeautifulSoup(response.text, "lxml")
|
||||
results = self._parse_page(soup, max_results)
|
||||
if results:
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.debug(f"Twitter page scraping failed (expected): {e}")
|
||||
|
||||
# Fall back to URL-based result
|
||||
result_type_map = {
|
||||
"top": "Top tweets",
|
||||
"latest": "Latest tweets",
|
||||
"people": "People",
|
||||
"photos": "Photos",
|
||||
"videos": "Videos",
|
||||
}
|
||||
desc = result_type_map.get(search_type, "Tweets")
|
||||
|
||||
return [
|
||||
SearchResult(
|
||||
title=f"Twitter: {query} ({desc})",
|
||||
url=url,
|
||||
snippet=f"Search Twitter/X for '{query}'. Note: Twitter may require login.",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="social",
|
||||
extra={"search_type": search_type, "twitter_url": url},
|
||||
)
|
||||
]
|
||||
|
||||
def _parse_page(self, soup, max_results: int) -> list[SearchResult]:
|
||||
"""Attempt to extract tweet info from search page."""
|
||||
results = []
|
||||
# Twitter uses JS rendering, so HTML parsing may yield limited results
|
||||
tweets = soup.select("article[data-testid='tweet'], div.tweet, .content")
|
||||
for i, tweet in enumerate(tweets[:max_results]):
|
||||
text_el = tweet.select_one(
|
||||
"[data-testid='tweetText'], .tweet-text, p[lang]"
|
||||
)
|
||||
link_el = tweet.select_one("a[href*='/status/'], time")
|
||||
user_el = tweet.select_one(
|
||||
"[data-testid='User-Name'], .username, .fullname"
|
||||
)
|
||||
|
||||
text = text_el.get_text(strip=True) if text_el else ""
|
||||
user = user_el.get_text(strip=True) if user_el else ""
|
||||
|
||||
tweet_url = ""
|
||||
if link_el:
|
||||
href = link_el.get("href", "")
|
||||
tweet_url = f"{self.BASE_URL}{href}" if href.startswith("/") else href
|
||||
|
||||
if text:
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=f"@{user}: {text[:80]}..." if user else text[:80],
|
||||
url=tweet_url or self.build_url(text[:30]),
|
||||
snippet=text,
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="social",
|
||||
extra={"user": user} if user else {},
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
+149
-78
@@ -1,9 +1,17 @@
|
||||
"""
|
||||
Unified multi-source search — URL-based engines.
|
||||
Unified multi-source search — Python API.
|
||||
|
||||
Architecture:
|
||||
Primary: URL engines (Baidu, Bing, Google, DuckDuckGo, Sogou)
|
||||
No external service dependency.
|
||||
Provides a clean async interface for programmatic use.
|
||||
All engines run in parallel with configurable concurrency.
|
||||
|
||||
Usage:
|
||||
from scripts.unified_search import UnifiedSearch
|
||||
|
||||
us = UnifiedSearch()
|
||||
resp = await us.search("quantum computing", sources=["bing", "arxiv"], max_results=5)
|
||||
for r in resp.results:
|
||||
print(f"[{r.rank}] {r.title}")
|
||||
await us.close()
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -12,98 +20,161 @@ import asyncio
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from scripts.models import SearchQuery, SearchResponse, SourceRef, SearchMetadata, EngineTiming
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
from scripts.engines.bing_scraper import BingScraper
|
||||
from scripts.engines.sogou_scraper import SogouScraper
|
||||
from scripts.engines.baidu_scraper import BaiduScraper
|
||||
from scripts.engines.search360_scraper import Search360Scraper
|
||||
from scripts.engines.parser_engines import DuckDuckGoParser, BraveParser
|
||||
from scripts.engines.news_engine import NewsSearch
|
||||
from scripts.engines.academic_engine import AcademicSearch
|
||||
from scripts.engines.github_engine import GitHubEngine
|
||||
from scripts.core.aggregator import ResultAggregator
|
||||
from scripts.models import SearchResponse, SourceRef, SearchMetadata, EngineTiming
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
title: str
|
||||
url: str
|
||||
content: str
|
||||
source: str = ""
|
||||
engine: str = ""
|
||||
score: float = 0.0
|
||||
category: str = ""
|
||||
published: str = ""
|
||||
# Default engine registry
|
||||
_ALL_ENGINES: dict[str, BaseEngine] = {
|
||||
"bing": BingScraper(),
|
||||
"sogou": SogouScraper(),
|
||||
"baidu": BaiduScraper(),
|
||||
"360": Search360Scraper(),
|
||||
"duckduckgo": DuckDuckGoParser(),
|
||||
"brave": BraveParser(),
|
||||
"news": NewsSearch(),
|
||||
"academic": AcademicSearch(),
|
||||
"github": GitHubEngine(),
|
||||
}
|
||||
|
||||
|
||||
class UnifiedSearch:
|
||||
"""Unified search across multiple open-source engines."""
|
||||
"""Unified multi-source search — Python API entry point."""
|
||||
|
||||
def __init__(self):
|
||||
self._start_time = 0.0
|
||||
def __init__(self, timeout: float = 15.0, max_concurrent: int = 5):
|
||||
self.timeout = timeout
|
||||
self.max_concurrent = max_concurrent
|
||||
self._engines: dict[str, BaseEngine] = {
|
||||
name: eng.__class__(timeout=timeout)
|
||||
for name, eng in _ALL_ENGINES.items()
|
||||
}
|
||||
|
||||
async def search(self, query: str, max_results: int = 5,
|
||||
language: str = "zh-CN") -> SearchResponse:
|
||||
"""Search using URL-based engines."""
|
||||
self._start_time = time.time()
|
||||
metadata = SearchMetadata()
|
||||
metadata.total_engines = 1
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
sources: Optional[list[str]] = None,
|
||||
max_results: int = 10,
|
||||
region: str = "cn",
|
||||
**kwargs,
|
||||
) -> SearchResponse:
|
||||
"""
|
||||
Search across multiple sources in parallel.
|
||||
|
||||
all_results, timings = await self._search_urls(query, max_results)
|
||||
Args:
|
||||
query: Search query string
|
||||
sources: List of engine names to use (default: all)
|
||||
max_results: Max results to return (deduped)
|
||||
region: "cn" or "global"
|
||||
**kwargs: Extra params passed to engines
|
||||
|
||||
elapsed_ms = int((time.time() - self._start_time) * 1000)
|
||||
metadata.duration_ms = elapsed_ms
|
||||
metadata.engine_timings = timings
|
||||
metadata.total_raw_results = len(all_results)
|
||||
Returns:
|
||||
SearchResponse with results, sources, and metadata
|
||||
"""
|
||||
t0 = time.time()
|
||||
|
||||
sources = []
|
||||
seen_urls = set()
|
||||
for i, r in enumerate(all_results[:max_results]):
|
||||
if r.url and r.url not in seen_urls:
|
||||
seen_urls.add(r.url)
|
||||
sources.append(SourceRef(
|
||||
index=i + 1,
|
||||
title=r.title or r.content[:50],
|
||||
url=r.url or "",
|
||||
source=r.source,
|
||||
# Select engines
|
||||
if sources:
|
||||
selected = {k: v for k, v in self._engines.items() if k in sources}
|
||||
else:
|
||||
selected = dict(self._engines)
|
||||
|
||||
# Build region kwarg
|
||||
eng_kwargs = {"region": region} if region else {}
|
||||
|
||||
# Run all engines concurrently with semaphore
|
||||
semaphore = asyncio.Semaphore(self.max_concurrent)
|
||||
|
||||
async def run_engine(name: str, eng: BaseEngine) -> tuple[str, list[SearchResult], float]:
|
||||
async with semaphore:
|
||||
start = time.time()
|
||||
try:
|
||||
results = await asyncio.wait_for(
|
||||
eng.search(query, max_results=max_results, **eng_kwargs),
|
||||
timeout=eng.timeout + 2.0,
|
||||
)
|
||||
elapsed = time.time() - start
|
||||
return name, list(results), elapsed
|
||||
except asyncio.TimeoutError:
|
||||
return name, [], time.time() - start
|
||||
except Exception:
|
||||
return name, [], time.time() - start
|
||||
|
||||
tasks = [run_engine(n, e) for n, e in selected.items()]
|
||||
done = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# Collect results and timings
|
||||
all_results: list[SearchResult] = []
|
||||
engine_timings: list[EngineTiming] = []
|
||||
engines_used: set[str] = set()
|
||||
|
||||
for item in done:
|
||||
if isinstance(item, tuple):
|
||||
name, results, elapsed = item
|
||||
all_results.extend(results)
|
||||
if results:
|
||||
engines_used.add(name)
|
||||
engine_timings.append(EngineTiming(
|
||||
engine_name=name,
|
||||
success=bool(results),
|
||||
duration_ms=int(elapsed * 1000),
|
||||
result_count=len(results),
|
||||
))
|
||||
elif isinstance(item, Exception):
|
||||
# Exception in gather — log but don't crash
|
||||
pass
|
||||
|
||||
metadata.engines_used = list(set(s.source for s in sources))
|
||||
# Deduplicate and rank
|
||||
aggregator = ResultAggregator()
|
||||
merged = aggregator.merge_and_rank(all_results, max_results=max_results)
|
||||
|
||||
elapsed_ms = int((time.time() - t0) * 1000)
|
||||
|
||||
# Build SourceRef list
|
||||
sources_list = [
|
||||
SourceRef(
|
||||
index=r.rank,
|
||||
title=r.title,
|
||||
url=r.url,
|
||||
source=r.source,
|
||||
)
|
||||
for r in merged
|
||||
]
|
||||
|
||||
metadata = SearchMetadata(
|
||||
total_engines=len(selected),
|
||||
total_raw_results=len(all_results),
|
||||
engines_used=list(engines_used),
|
||||
duration_ms=elapsed_ms,
|
||||
engine_timings=engine_timings,
|
||||
)
|
||||
|
||||
return SearchResponse(
|
||||
query=query,
|
||||
results=list(all_results[:max_results]),
|
||||
sources=sources,
|
||||
results=merged,
|
||||
sources=sources_list,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
async def _search_urls(self, query: str, max_results: int
|
||||
) -> tuple[list[SearchResult], list[EngineTiming]]:
|
||||
"""Build search URLs for major engines."""
|
||||
timings = []
|
||||
engines = {
|
||||
"baidu": f"https://www.baidu.com/s?wd={query}",
|
||||
"bing": f"https://cn.bing.com/search?q={query}",
|
||||
"google": f"https://www.google.com/search?q={query}",
|
||||
"duckduckgo": f"https://html.duckduckgo.com/html/?q={query}",
|
||||
"sogou": f"https://www.sogou.com/web?query={query}",
|
||||
}
|
||||
results = []
|
||||
for name, url in engines.items():
|
||||
results.append(SearchResult(
|
||||
title=f"{name}: {query}",
|
||||
url=url,
|
||||
content=f"Search {query} on {name}",
|
||||
source=name,
|
||||
))
|
||||
|
||||
timings.append(EngineTiming(
|
||||
engine_name="url_fallback",
|
||||
success=True,
|
||||
duration_ms=1,
|
||||
result_count=len(results),
|
||||
))
|
||||
return results, timings
|
||||
|
||||
async def health(self) -> dict:
|
||||
"""Check status."""
|
||||
return {
|
||||
"status": "ok",
|
||||
"engines": ["baidu", "bing", "google", "duckduckgo", "sogou"],
|
||||
}
|
||||
async def health(self) -> dict[str, Any]:
|
||||
"""Check connectivity of all engines."""
|
||||
results = {}
|
||||
for name, eng in self._engines.items():
|
||||
try:
|
||||
health = await asyncio.wait_for(eng.health(), timeout=5.0)
|
||||
results[name] = health
|
||||
except Exception as e:
|
||||
results[name] = {"name": name, "status": "fail", "error": str(e)}
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
"""Cleanup (no-op for current implementation)."""
|
||||
pass
|
||||
@@ -1 +0,0 @@
|
||||
"""WeChat search engines."""
|
||||
@@ -1,365 +0,0 @@
|
||||
"""
|
||||
Sogou WeChat (搜狗微信) search engine.
|
||||
|
||||
Searches WeChat public account articles via weixin.sogou.com.
|
||||
|
||||
Anti-scraping strategy (inspired by Node.js wechat-article-search skill):
|
||||
- Cookie pre-fetch from v.sogou.com to get SNUID
|
||||
- User-Agent rotation with a pool of real browser UAs
|
||||
- Random delays between requests (500-1500ms)
|
||||
- Retry logic with exponential backoff
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Real browser User-Agent pool for rotation
|
||||
USER_AGENTS = [
|
||||
# Chrome
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
# Safari
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1",
|
||||
# Firefox
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0",
|
||||
# Edge
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
|
||||
]
|
||||
|
||||
|
||||
class SogouWeChatEngine(SearchEngine):
|
||||
"""搜狗微信搜索 - searches WeChat public account articles."""
|
||||
|
||||
SEARCH_URL = "https://weixin.sogou.com/weixin"
|
||||
COOKIE_URL = "https://v.sogou.com"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("weixin", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._ua = random.choice(USER_AGENTS)
|
||||
self._cookies: dict[str, str] = {}
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers=self._build_headers(),
|
||||
follow_redirects=False, # Don't auto-follow to detect anti-scraping
|
||||
)
|
||||
|
||||
def _build_headers(self) -> dict[str, str]:
|
||||
return {
|
||||
"User-Agent": self._ua,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"Referer": "https://weixin.sogou.com/",
|
||||
"DNT": "1",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
|
||||
# ---- Anti-scraping enhancements from bot-search-mcp ---- #
|
||||
|
||||
CAPTCHA_KEYWORDS = [
|
||||
"请输入验证码", "访问频率过高", "请稍后重试", "验证码",
|
||||
"captcha", "访问过于频繁", "请输入以下验证码",
|
||||
"403 Forbidden", "您的访问被拒绝",
|
||||
]
|
||||
|
||||
def _random_delay(self, min_s: float = 0.5, max_s: float = 1.5):
|
||||
"""Random delay to mimic human browsing."""
|
||||
delay = random.uniform(min_s, max_s)
|
||||
return asyncio.sleep(delay)
|
||||
|
||||
def _rotate_ua(self):
|
||||
"""Rotate to a new random User-Agent."""
|
||||
self._ua = random.choice(USER_AGENTS)
|
||||
|
||||
async def _get_cookie(self) -> dict[str, str]:
|
||||
"""Pre-fetch cookie from v.sogou.com to get SNUID/SUID/IPLOC/SUV."""
|
||||
try:
|
||||
response = await self._client.get(
|
||||
self.COOKIE_URL,
|
||||
headers={
|
||||
"User-Agent": self._ua,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
},
|
||||
)
|
||||
cookies: dict[str, str] = {}
|
||||
|
||||
# Extract from httpx Cookies
|
||||
for key, value in response.cookies.items():
|
||||
if key.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
|
||||
cookies[key] = value
|
||||
|
||||
# Also parse from Set-Cookie header (more reliable)
|
||||
set_cookie = response.headers.get("set-cookie", "")
|
||||
for part in set_cookie.split(";"):
|
||||
if "=" in part:
|
||||
k, v = part.split("=", 1)
|
||||
k = k.strip()
|
||||
if k.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
|
||||
cookies[k] = v.split(",")[-1].strip()
|
||||
|
||||
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com: {list(cookies.keys())}")
|
||||
return cookies
|
||||
except Exception as e:
|
||||
logger.warning(f"Cookie pre-fetch failed: {e}")
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def _detect_captcha(html: str) -> bool:
|
||||
"""检测是否触发了验证码或反爬。"""
|
||||
html_lower = html.lower()
|
||||
for kw in SogouWeChatEngine.CAPTCHA_KEYWORDS:
|
||||
if kw in html_lower or kw.lower() in html_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _exponential_backoff(
|
||||
self, attempt: int, base_delay: float = 2.0
|
||||
):
|
||||
"""指数退避等待(带随机 jitter)。"""
|
||||
wait = (2 ** attempt) + random.uniform(0, 1)
|
||||
await asyncio.sleep(wait)
|
||||
|
||||
def build_url(self, query: str, search_type: int = 2, page: int = 1, **kwargs) -> str:
|
||||
"""
|
||||
Build Sogou WeChat search URL.
|
||||
|
||||
Args:
|
||||
query: Search keyword.
|
||||
search_type: 2=articles, 1=accounts.
|
||||
page: Page number (starts at 1).
|
||||
"""
|
||||
return (
|
||||
f"{self.SEARCH_URL}?query={quote_plus(query)}"
|
||||
f"&s_from=input&_sug_=n&type={search_type}"
|
||||
f"&page={page}&ie=utf8"
|
||||
)
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search WeChat articles via Sogou and parse results."""
|
||||
# Step 1: Get cookie first
|
||||
if not self._cookies:
|
||||
self._cookies = await self._get_cookie()
|
||||
await self._random_delay()
|
||||
|
||||
# Step 2: Search with retry
|
||||
max_pages = min(5, (max_results + 9) // 10)
|
||||
all_results = []
|
||||
|
||||
for page in range(1, max_pages + 1):
|
||||
url = self.build_url(query, page=page, **kwargs)
|
||||
results = await self._search_page(url, query, max_results - len(all_results))
|
||||
all_results.extend(results)
|
||||
|
||||
if len(all_results) >= max_results:
|
||||
break
|
||||
|
||||
# Delay between pages
|
||||
await self._random_delay()
|
||||
|
||||
if not all_results:
|
||||
# Fallback: try legacy wx.sogou.com
|
||||
return await self._legacy_search(query, max_results, **kwargs)
|
||||
|
||||
return all_results[:max_results]
|
||||
|
||||
async def _search_page(
|
||||
self, url: str, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Search a single page with enhanced retry + anti-scraping logic."""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
# Rotate UA on each retry
|
||||
if attempt > 0:
|
||||
self._rotate_ua()
|
||||
self._client.headers.update({"User-Agent": self._ua})
|
||||
|
||||
response = await self._client.get(
|
||||
url,
|
||||
cookies=self._cookies,
|
||||
follow_redirects=False,
|
||||
)
|
||||
|
||||
# 302 = anti-scraping redirect
|
||||
if response.status_code == 302:
|
||||
logger.warning(
|
||||
f"Sogou anti-scraping triggered (302), "
|
||||
f"attempt {attempt + 1}"
|
||||
)
|
||||
self._cookies = await self._get_cookie()
|
||||
await self._exponential_backoff(attempt)
|
||||
continue
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.warning(
|
||||
f"Sogou returned {response.status_code}, "
|
||||
f"attempt {attempt + 1}"
|
||||
)
|
||||
await self._exponential_backoff(attempt)
|
||||
continue
|
||||
|
||||
html = response.text
|
||||
|
||||
# Detect captcha / block page
|
||||
if self._detect_captcha(html):
|
||||
logger.warning(
|
||||
f"Sogou captcha/block detected, "
|
||||
f"attempt {attempt + 1}"
|
||||
)
|
||||
await self._exponential_backoff(attempt)
|
||||
self._cookies = await self._get_cookie()
|
||||
continue
|
||||
|
||||
return self._parse_results(html, query, max_results)
|
||||
|
||||
except (httpx.HTTPError, httpx.TimeoutException) as e:
|
||||
logger.warning(
|
||||
f"Sogou search attempt {attempt + 1} failed: {e}"
|
||||
)
|
||||
await self._exponential_backoff(attempt)
|
||||
continue
|
||||
|
||||
return []
|
||||
|
||||
async def _legacy_search(
|
||||
self, query: str, max_results: int, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Fallback: try the legacy wx.sogou.com domain."""
|
||||
legacy_url = f"https://wx.sogou.com/weixin?type=2&query={quote_plus(query)}"
|
||||
try:
|
||||
response = await self._client.get(
|
||||
legacy_url,
|
||||
cookies=self._cookies,
|
||||
follow_redirects=True,
|
||||
)
|
||||
if response.status_code == 200:
|
||||
return self._parse_results(response.text, query, max_results)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Ultimate fallback
|
||||
search_url = self.build_url(query)
|
||||
return [
|
||||
SearchResult(
|
||||
title=f"微信搜索: {query}",
|
||||
url=search_url,
|
||||
snippet=f"Search WeChat for '{query}' via Sogou",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="wechat",
|
||||
extra={"search_url": search_url},
|
||||
)
|
||||
]
|
||||
|
||||
def _parse_results(
|
||||
self, html: str, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Parse Sogou WeChat search results."""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
|
||||
# Main selector: ul.news-list > li (standard Sogou WeChat structure)
|
||||
items = soup.select("ul.news-list > li")
|
||||
|
||||
if not items:
|
||||
# Fallback selectors
|
||||
items = soup.select(
|
||||
".news-list2 li, .wx-rb-item, .results .result, "
|
||||
".item, .news-box .news-list li"
|
||||
)
|
||||
|
||||
for i, item in enumerate(items[:max_results]):
|
||||
title_el = item.select_one("h3 a")
|
||||
if not title_el:
|
||||
title_el = item.select_one(".tit a, .title a, h3")
|
||||
|
||||
snippet_el = item.select_one("p.txt-info, .txt-info, .summary, .des, p.info")
|
||||
source_el = item.select_one(
|
||||
".all-time-y2 a.account, a.account, "
|
||||
".account, .source, .s-p, .from"
|
||||
)
|
||||
|
||||
title = title_el.get_text(strip=True) if title_el else ""
|
||||
link = title_el.get("href", "") if title_el and hasattr(title_el, "get") else ""
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
account = source_el.get_text(strip=True) if source_el else ""
|
||||
|
||||
# Parse time from inline scripts or text
|
||||
time_text = self._parse_time(item)
|
||||
|
||||
if not title:
|
||||
continue
|
||||
|
||||
# Make sure URL is absolute
|
||||
if link and not link.startswith("http"):
|
||||
link = f"https://weixin.sogou.com{link}"
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title,
|
||||
url=link,
|
||||
snippet=snippet or f"WeChat article from {account or 'unknown'}",
|
||||
source=self.name,
|
||||
rank=(i + 1),
|
||||
category="wechat",
|
||||
extra={
|
||||
"account": account,
|
||||
"time": time_text,
|
||||
"search_url": self.build_url(query),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _parse_time(self, item) -> str:
|
||||
"""Extract publish time from search result item."""
|
||||
# Try script tag with Unix timestamp (Sogou encodes dates in scripts)
|
||||
for script in item.select("script"):
|
||||
text = script.string or ""
|
||||
# Look for Unix timestamp pattern
|
||||
match = re.search(r'\d{10}', text)
|
||||
if match:
|
||||
import datetime
|
||||
ts = int(match.group())
|
||||
try:
|
||||
dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone(datetime.timedelta(hours=8)))
|
||||
return dt.strftime("%Y-%m-%d %H:%M")
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
# Try inline text with relative time like "2小时前", "3天前"
|
||||
time_text = item.get_text()
|
||||
relative_match = re.search(r'(\d+)\s*(小时|分钟|天|周|月)前', time_text)
|
||||
if relative_match:
|
||||
return f"{relative_match.group(1)}{relative_match.group(2)}前"
|
||||
|
||||
# Try standard date patterns
|
||||
date_match = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', time_text)
|
||||
if date_match:
|
||||
return date_match.group(1)
|
||||
|
||||
return ""
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
Reference in new issue
Block a user