refactor: restructure project with CLI tool, multi-engine support, and skill-builder compliant SKILL.md

- Replace setup.py with pyproject.toml (uv-compatible)
- Add CLI entry point (web-search) with 9 subcommands
- Add multi-engine architecture (14 web + 7 special sources)
- Add WeChat/CLS/WallStreetCN/ArXiv/Twitter search engines
- Remove old src/ and examples/ directories
- Remove redundant requirements.txt and setup.py
- Update README.md and SKILL.md with full documentation
- SKILL.md now compliant with skill-builder standards (frontmatter, triggers, structure)
This commit is contained in:
vex1023 committed 2026-05-14 16:31:54 +08:00
1 parent 80ac14491e
commit 87935b59ce
29 files changed
+2364 -354

No files matched your search

+1
View File
@@ -0,0 +1 @@
"""Web Search Skills package."""
+1
View File
@@ -0,0 +1 @@
"""Academic search engines."""
+130
View File
@@ -0,0 +1,130 @@
"""
ArXiv academic paper search engine.
Uses the ArXiv API (export.arxiv.org/api/query) to search for papers.
"""
from __future__ import annotations
import logging
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from typing import Optional
from urllib.parse import quote_plus
import httpx
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
class ArXivEngine(SearchEngine):
"""ArXiv academic paper search engine via official API."""
API_URL = "https://export.arxiv.org/api/query"
def __init__(self, config: Optional[dict] = None):
super().__init__("ArXiv", config)
timeout = (config or {}).get("timeout", 30)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"User-Agent": (config or {}).get(
"user_agent",
"ArXivSearchSkill/1.0",
),
"Accept": "application/atom+xml",
},
follow_redirects=True,
)
def build_url(self, query: str, max_results: int = 10, **kwargs) -> str:
search_field = kwargs.get("search_field", "all")
return (
f"{self.API_URL}?search_query={search_field}:{quote_plus(query)}"
f"&max_results={max_results}&sortBy=submittedDate&sortOrder=descending"
)
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search ArXiv papers."""
url = self.build_url(query, max_results=max_results, **kwargs)
try:
response = await self._client.get(url)
response.raise_for_status()
return self._parse_response(response.text, max_results)
except httpx.HTTPError as e:
logger.error(f"ArXiv search failed: {e}")
return []
def _parse_response(self, xml_text: str, max_results: int) -> list[SearchResult]:
"""Parse ArXiv Atom XML response."""
ns = {
"atom": "http://www.w3.org/2005/Atom",
"arxiv": "http://arxiv.org/schemas/atom",
}
results = []
try:
root = ET.fromstring(xml_text)
except ET.ParseError as e:
logger.error(f"ArXiv XML parse error: {e}")
return []
for i, entry in enumerate(root.findall("atom:entry", ns)[:max_results]):
title = entry.findtext("atom:title", "", ns)
summary = entry.findtext("atom:summary", "", ns)
published = entry.findtext("atom:published", "", ns)
updated = entry.findtext("atom:updated", "", ns)
link_el = entry.find("atom:link", ns)
link = link_el.get("href", "") if link_el is not None else ""
authors = []
for author in entry.findall("atom:author", ns):
name = author.findtext("atom:name", "", ns)
if name:
authors.append(name)
# Clean title and summary
title = title.replace("\n", " ").strip()
snippet = summary.replace("\n", " ").strip()[:300]
parsed_date = self._parse_date(published) if published else (
self._parse_date(updated) if updated else
datetime.min.replace(tzinfo=timezone.utc)
)
results.append(
SearchResult(
title=title or f"ArXiv: {summary[:60]}",
url=link,
snippet=snippet or "No abstract available",
source=self.name,
rank=i + 1,
category="academic",
extra={
"published": published or "",
"updated": updated or "",
"authors": ", ".join(authors),
"parsed_date": parsed_date,
},
)
)
return results
def _parse_date(self, date_str: str) -> datetime:
"""Parse ArXiv date format."""
try:
# ArXiv format: 2024-01-01T00:00:00Z
return datetime.strptime(
date_str.strip(), "%Y-%m-%dT%H:%M:%SZ"
).replace(tzinfo=timezone.utc)
except (ValueError, TypeError):
return datetime.min.replace(tzinfo=timezone.utc)
async def close(self):
await self._client.aclose()
+458
View File
@@ -0,0 +1,458 @@
#!/usr/bin/env python3
"""
Web Search Skills CLI — search across 20+ sources from the command line.
Usage:
web-search <command> <query> [options]
Commands:
search Search all sources (default)
web Search web search engines (Google, Baidu, DuckDuckGo, etc.)
news Search news (财联社, 华尔街见闻, RSS)
wechat Search WeChat public account articles
academic Search academic papers (ArXiv)
social Search social media (Twitter/X)
rss Search RSS news feeds
sources List all available sources
urls Get search URLs without executing
Options:
-n, --max-results N Max results per source (default: 10)
-j, --json Output as JSON
-o, --output FILE Save results to file
-s, --source NAME Specific source name
-r, --region REGION Region filter (cn/global)
-v, --verbose Show debug logs
-h, --help Show this help
Examples:
web-search wechat 蒙面财经
web-search news "人工智能" -n 5
web-search web "Python async" -r global -j
web-search search "量子计算" -o results.json
web-search sources
web-search urls "机器学习"
"""
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import os
import sys
from datetime import datetime
from typing import Any
# Ensure scripts package is importable
_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR)
if _PROJECT_ROOT not in sys.path:
sys.path.insert(0, _PROJECT_ROOT)
logging.basicConfig(
level=logging.WARNING,
format="%(levelname)s: %(message)s",
)
logger = logging.getLogger("cli")
# ---------------------------------------------------------------------------
# Lazy engine imports (import on demand to keep startup fast)
# ---------------------------------------------------------------------------
def _get_unified():
from scripts.unified_search import UnifiedSearch
return UnifiedSearch
def _get_sogou():
from scripts.wechat.sogou_weixin import SogouWeChatEngine
return SogouWeChatEngine
def _get_rss():
from scripts.news.rss import RSSNewsEngine
return RSSNewsEngine
def _get_twitter():
from scripts.social.twitter import TwitterSearchEngine
return TwitterSearchEngine
def _get_ddg():
from scripts.engines.parser_engines import DuckDuckGoEngine
return DuckDuckGoEngine
def _get_config_loader():
from scripts.core.config_loader import (
get_search_engines,
get_news_sources,
get_academic_sources,
get_wechat_sources,
get_social_sources,
get_special_engines,
get_all_sources,
list_source_categories,
)
return {
"get_search_engines": get_search_engines,
"get_news_sources": get_news_sources,
"get_academic_sources": get_academic_sources,
"get_wechat_sources": get_wechat_sources,
"get_social_sources": get_social_sources,
"get_special_engines": get_special_engines,
"get_all_sources": get_all_sources,
"list_source_categories": list_source_categories,
}
# ---------------------------------------------------------------------------
# Result formatting
# ---------------------------------------------------------------------------
def _format_result(r, fmt: str = "text") -> str:
"""Format a single SearchResult."""
if fmt == "json":
return json.dumps({
"rank": r.rank,
"title": r.title,
"url": r.url,
"snippet": r.snippet[:300],
"source": r.source,
"category": r.category,
"extra": dict(r.extra) if r.extra else {},
}, ensure_ascii=False, indent=2)
lines = [f"[{r.rank}] {r.title}"]
lines.append(f" URL: {r.url}")
if r.snippet:
lines.append(f" {r.snippet[:200]}")
if r.extra:
extra_parts = []
for k, v in r.extra.items():
if k == "account" and v:
extra_parts.append(f"公众号: {v}")
elif k == "time" and v:
extra_parts.append(f"时间: {v}")
elif k == "search_url":
continue
elif v and k not in ("parsed_date", "note", "feed_url"):
extra_parts.append(f"{k}: {v}")
if extra_parts:
lines.append(f" ({'; '.join(extra_parts)})")
return "\n".join(lines)
def _print_results(results, fmt: str = "text"):
"""Print search results."""
if not results:
print(" 未找到结果")
return
if fmt == "json":
output = json.dumps(
[{
"rank": r.rank,
"title": r.title,
"url": r.url,
"snippet": r.snippet[:300],
"source": r.source,
"category": r.category,
"extra": dict(r.extra) if r.extra else {},
} for r in results],
ensure_ascii=False,
indent=2,
)
print(output)
else:
for r in results:
print(_format_result(r))
print()
# ---------------------------------------------------------------------------
# Async searchers
# ---------------------------------------------------------------------------
async def _search_all(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()()
try:
return await us.search(query, max_results=max_results, **kwargs)
finally:
await us.close()
async def _search_web(query: str, max_results: int, region: str = None, **kwargs) -> list[Any]:
us = _get_unified()()
try:
return await us.search_engines(query, region=region, max_results=max_results)
finally:
await us.close()
async def _search_news(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()()
try:
return await us.search_news(query, max_results=max_results, **kwargs)
finally:
await us.close()
async def _search_wechat(query: str, max_results: int) -> list[Any]:
engine = _get_sogou()()
try:
return await engine.search(query, max_results=max_results)
finally:
await engine.close()
async def _search_academic(query: str, max_results: int) -> list[Any]:
us = _get_unified()()
try:
return await us.search_academic(query, max_results=max_results)
finally:
await us.close()
async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()()
try:
return await us.search_social(query, max_results=max_results, **kwargs)
finally:
await us.close()
async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()()
try:
return await us.search_rss(query, max_results=max_results, **kwargs)
finally:
await us.close()
async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]:
us = _get_unified()()
try:
return await us.get_search_urls(query, sources=sources)
finally:
await us.close()
# ---------------------------------------------------------------------------
# CLI actions
# ---------------------------------------------------------------------------
def cmd_sources(args):
"""List all available search sources."""
cl = _get_config_loader()
categories = cl["list_source_categories"]()
print("可用搜索源:\n")
for cat_key, cat_name in [
("search_engines", "Web 搜索引擎"),
("news_sources", "新闻源"),
("academic_sources", "学术源"),
("wechat_sources", "微信源"),
("social_sources", "社交源"),
("special_engines", "特殊源"),
]:
count = categories.get(cat_key, 0)
if count == 0:
continue
print(f" {cat_name} ({count} 个)")
print(f"\n总计: {cl['get_all_sources']().__len__()} 个源")
print("\n详细列表:")
for s in cl["get_all_sources"]():
print(f" - {s['name']:20s} ({s.get('region', '?')}) [{s.get('type', 'url')}]")
def cmd_urls(args):
"""Get search URLs without executing."""
urls = asyncio.run(_get_urls(args.query, sources=args.source))
for name, url in urls.items():
print(f"{name}:")
print(f" {url}")
print()
async def _run_search(args):
"""Route search to the right engine(s)."""
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)
method_map = {
"search": _search_all,
"web": _search_web,
"news": _search_news,
"wechat": _search_wechat,
"academic": _search_academic,
"social": _search_social,
"rss": _search_rss,
}
search_fn = method_map.get(args.command, _search_all)
kwargs = {}
if args.command == "web" and args.region:
kwargs["region"] = args.region
try:
results = await search_fn(
args.query,
max_results=args.max_results,
**kwargs,
)
except Exception as e:
print(f"搜索失败: {e}", file=sys.stderr)
if args.verbose:
import traceback
traceback.print_exc()
sys.exit(1)
# Print header
cmd_names = {
"search": "全源搜索",
"web": "网页搜索",
"news": "新闻搜索",
"wechat": "微信公众号搜索",
"academic": "学术搜索",
"social": "社交搜索",
"rss": "RSS 新闻搜索",
}
header = cmd_names.get(args.command, "搜索")
print(f"{header}: {args.query}")
print(f"共 {len(results)} 条结果\n")
_print_results(results, fmt="json" if args.json else "text")
# Save to file if requested
if args.output:
output_data = json.dumps(
[{
"query": args.query,
"command": args.command,
"timestamp": datetime.now().isoformat(),
"total": len(results),
"results": [
{
"rank": r.rank,
"title": r.title,
"url": r.url,
"snippet": r.snippet[:500],
"source": r.source,
"category": r.category,
"extra": dict(r.extra) if r.extra else {},
}
for r in results
],
}],
ensure_ascii=False,
indent=2,
)
output_path = os.path.abspath(args.output)
with open(output_path, "w", encoding="utf-8") as f:
f.write(output_data)
print(f"\n结果已保存至: {output_path}")
# ---------------------------------------------------------------------------
# Argument parser
# ---------------------------------------------------------------------------
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="web-search",
description="Web Search Skills CLI — multi-source search tool",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
parser.add_argument(
"command",
nargs="?",
default="search",
choices=["search", "web", "news", "wechat", "academic", "social",
"rss", "sources", "urls"],
help="Search type (default: search)",
)
parser.add_argument(
"query",
nargs="?",
default="",
help="Search keyword",
)
parser.add_argument(
"-n", "--max-results",
type=int,
default=10,
help="Max results per source (default: 10)",
)
parser.add_argument(
"-j", "--json",
action="store_true",
help="Output as JSON",
)
parser.add_argument(
"-o", "--output",
type=str,
help="Save results to JSON file",
)
parser.add_argument(
"-s", "--source",
type=str,
nargs="*",
help="Specific source name(s)",
)
parser.add_argument(
"-r", "--region",
type=str,
choices=["cn", "global"],
help="Region filter (web search only)",
)
parser.add_argument(
"-v", "--verbose",
action="store_true",
help="Show debug logs",
)
return parser
# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
def main():
parser = build_parser()
args = parser.parse_args()
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)
# sources and urls commands don't need a query
if args.command == "sources":
cmd_sources(args)
return
if args.command == "urls":
if not args.query:
print("错误: urls 命令需要提供搜索关键词", file=sys.stderr)
sys.exit(1)
cmd_urls(args)
return
# Other commands need a query
if not args.query:
print(f"错误: {args.command} 命令需要提供搜索关键词", file=sys.stderr)
sys.exit(1)
asyncio.run(_run_search(args))
if __name__ == "__main__":
main()
+1
View File
@@ -0,0 +1 @@
"""Core data types and base classes."""
+48
View File
@@ -0,0 +1,48 @@
"""
Core data types and base classes for search engines.
Provides:
- SearchResult: Standard result dataclass used across all engines
- SearchEngine: Abstract base class for all search engine implementations
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Optional
@dataclass
class SearchResult:
"""Represents a single search result."""
title: str = ""
url: str = ""
snippet: str = ""
source: str = ""
rank: int = 0
category: str = ""
extra: dict[str, Any] = field(default_factory=dict)
class SearchEngine(ABC):
"""Abstract base class for search engines."""
def __init__(self, name: str, config: Optional[dict[str, Any]] = None):
self.name = name
self.config = config or {}
@abstractmethod
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Execute a search and return results."""
...
def build_url(self, query: str, **kwargs) -> str:
"""Build a search URL for this engine (optional)."""
return ""
async def close(self):
"""Clean up resources."""
pass
+70
View File
@@ -0,0 +1,70 @@
"""Load and manage engine configuration from engines.json."""
from __future__ import annotations
import json
import os
from typing import Any, Optional
_CONFIG_PATH = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
"config", "engines.json",
)
def _load_config(path: Optional[str] = None) -> dict[str, Any]:
config_path = path or _CONFIG_PATH
with open(config_path, "r", encoding="utf-8") as f:
return json.load(f)
def get_search_engines(path: Optional[str] = None) -> list[dict[str, Any]]:
return _load_config(path).get("search_engines", [])
def get_news_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
return _load_config(path).get("news_sources", [])
def get_academic_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
return _load_config(path).get("academic_sources", [])
def get_wechat_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
return _load_config(path).get("wechat_sources", [])
def get_social_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
return _load_config(path).get("social_sources", [])
def get_special_engines(path: Optional[str] = None) -> list[dict[str, Any]]:
return _load_config(path).get("special_engines", [])
def get_engine(name: str, path: Optional[str] = None):
config = _load_config(path)
for category in ["search_engines", "news_sources", "academic_sources",
"wechat_sources", "social_sources", "special_engines"]:
for engine in config.get(category, []):
if engine["name"].lower() == name.lower():
return engine
return None
def get_all_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
config = _load_config(path)
all_sources = []
for category in ["search_engines", "news_sources", "academic_sources",
"wechat_sources", "social_sources", "special_engines"]:
all_sources.extend(config.get(category, []))
return all_sources
def list_source_categories(path: Optional[str] = None) -> dict[str, int]:
"""Return a dict mapping category name to count of sources."""
config = _load_config(path)
categories = [
"search_engines", "news_sources", "academic_sources",
"wechat_sources", "social_sources", "special_engines",
]
return {cat: len(config.get(cat, [])) for cat in categories}
+1
View File
@@ -0,0 +1 @@
"""Search engine implementations."""
+80
View File
@@ -0,0 +1,80 @@
"""
Parsing-based search engines (e.g. DuckDuckGo HTML scraper).
"""
from __future__ import annotations
import logging
from typing import Optional
import httpx
from bs4 import BeautifulSoup
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
class DuckDuckGoEngine(SearchEngine):
"""DuckDuckGo HTML search engine (no API key required)."""
SEARCH_URL = "https://html.duckduckgo.com/html/"
def __init__(self, config: Optional[dict] = None):
super().__init__("DuckDuckGo", config)
timeout = (config or {}).get("timeout", 15)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"User-Agent": (config or {}).get(
"user_agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36",
),
},
follow_redirects=True,
)
def build_url(self, query: str, **kwargs) -> str:
"""Build DuckDuckGo search URL."""
from urllib.parse import quote_plus
return f"https://duckduckgo.com/?q={quote_plus(query)}"
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search DuckDuckGo and parse HTML results."""
params = {"q": query}
try:
response = await self._client.post(self.SEARCH_URL, data=params)
response.raise_for_status()
return self._parse_results(response.text, max_results)
except httpx.HTTPError as e:
logger.error(f"DuckDuckGo search failed: {e}")
return []
def _parse_results(self, html: str, max_results: int) -> list[SearchResult]:
"""Parse DuckDuckGo HTML search results."""
soup = BeautifulSoup(html, "lxml")
results = []
for i, result in enumerate(soup.select(".result")[:max_results]):
title_el = result.select_one(".result__title a")
snippet_el = result.select_one(".result__snippet")
if title_el:
results.append(
SearchResult(
title=title_el.get_text(strip=True),
url=title_el.get("href", ""),
snippet=snippet_el.get_text(strip=True)
if snippet_el else "",
source=self.name,
rank=i + 1,
category="web",
)
)
return results
async def close(self):
await self._client.aclose()
+60
View File
@@ -0,0 +1,60 @@
"""
Generic URL-based search engine.
Builds search URLs from templates defined in engines.json
and returns them as URL results.
"""
from __future__ import annotations
import logging
from typing import Any, Optional
from urllib.parse import quote_plus
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
class UrlSearchEngine(SearchEngine):
"""
Generic search engine that builds URLs from templates.
Reads URL templates from config (engines.json) and returns
the constructed search URL as a SearchResult.
"""
def __init__(self, name: str, engine_def: dict[str, Any]):
super().__init__(name, engine_def.get("config", {}))
self.url_template = engine_def.get("url", "")
self.region = engine_def.get("region", "global")
def build_url(self, query: str, **kwargs) -> str:
"""Build search URL by replacing {keyword} with encoded query."""
if not self.url_template:
return ""
encoded = quote_plus(query)
return self.url_template.replace("{keyword}", encoded)
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Return the search URL as a single result."""
url = self.build_url(query, **kwargs)
if not url:
return []
return [
SearchResult(
title=f"{self.name}: {query}",
url=url,
snippet=f"Search {self.name} for '{query}'",
source=self.name,
rank=1,
category="web",
extra={"url": url, "region": self.region},
)
]
async def close(self):
pass
+1
View File
@@ -0,0 +1 @@
"""News search engines."""
+115
View File
@@ -0,0 +1,115 @@
"""
财联社 (cls.cn) news search engine.
Scrapes cls.cn search results for financial news.
"""
from __future__ import annotations
import logging
from typing import Optional
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
class CLSNewsEngine(SearchEngine):
"""财联社 (cls.cn) news search engine."""
SEARCH_URL = "https://www.cls.cn/searchPage"
def __init__(self, config: Optional[dict] = None):
super().__init__("cls", config)
timeout = (config or {}).get("timeout", 15)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"User-Agent": (config or {}).get(
"user_agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36",
),
"Referer": "https://www.cls.cn/",
},
follow_redirects=True,
)
def build_url(self, query: str, **kwargs) -> str:
return f"{self.SEARCH_URL}?keyword={quote_plus(query)}&type=all"
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search 财联社 and parse results."""
url = self.build_url(query)
try:
response = await self._client.get(url)
response.raise_for_status()
return self._parse_results(response.text, query, max_results)
except httpx.HTTPError as e:
logger.error(f"财联社 search failed: {e}")
return []
def _parse_results(
self, html: str, query: str, max_results: int
) -> list[SearchResult]:
"""Parse 财联社 search page."""
soup = BeautifulSoup(html, "lxml")
results = []
# cls.cn typically uses dynamic rendering, so parsing may be limited
articles = soup.select(
"a[href*='/detail'], .news-item, .search-result-item, article"
)
for i, article in enumerate(articles[:max_results]):
title_el = article.select_one("h3, h4, .title, .news-title")
link = article.get("href", "")
snippet_el = article.select_one(
".summary, .desc, .abstract, p"
)
title = title_el.get_text(strip=True) if title_el else ""
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if not title and not link:
continue
full_url = link if link.startswith("http") else f"https://www.cls.cn{link}"
results.append(
SearchResult(
title=title or f"财联社: {query}",
url=full_url,
snippet=snippet or f"Search result from 财联社",
source=self.name,
rank=i + 1,
category="news",
extra={"region": "cn"},
)
)
if not results:
# Fallback: return URL for manual browsing
search_url = self.build_url(query)
results.append(
SearchResult(
title=f"财联社: {query}",
url=search_url,
snippet=f"Search 财联社 for '{query}' (site requires JS)",
source=self.name,
rank=1,
category="news",
extra={"search_url": search_url, "note": "JS-rendered page"},
)
)
return results
async def close(self):
await self._client.aclose()
+288
View File
@@ -0,0 +1,288 @@
"""
RSS news aggregation engine.
Fetches and parses RSS/Atom feeds from trusted news sources.
Inspired by the news-summary skill's RSS-based approach.
Supported sources:
- BBC (World, Business, Technology, Top Stories)
- Reuters (World News)
- NPR (US news)
- Al Jazeera (Global South perspective)
- 财联社 (RSS if available)
- 华尔街见闻 (RSS if available)
"""
from __future__ import annotations
import logging
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from typing import Any, Optional
from urllib.parse import urlparse
import httpx
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
# RSS feed definitions (from news-summary skill + additional sources)
RSS_FEEDS = {
"BBC World": {
"url": "https://feeds.bbci.co.uk/news/world/rss.xml",
"category": "news",
"lang": "en",
},
"BBC Top Stories": {
"url": "https://feeds.bbci.co.uk/news/rss.xml",
"category": "news",
"lang": "en",
},
"BBC Business": {
"url": "https://feeds.bbci.co.uk/news/business/rss.xml",
"category": "business",
"lang": "en",
},
"BBC Technology": {
"url": "https://feeds.bbci.co.uk/news/technology/rss.xml",
"category": "technology",
"lang": "en",
},
"Reuters": {
"url": "https://www.reutersagency.com/feed/?best-regions=world&post_type=best",
"category": "news",
"lang": "en",
},
"NPR": {
"url": "https://feeds.npr.org/1001/rss.xml",
"category": "news",
"lang": "en",
},
"Al Jazeera": {
"url": "https://www.aljazeera.com/xml/rss/all.xml",
"category": "news",
"lang": "en",
},
}
class RSSNewsEngine(SearchEngine):
"""
RSS-based news aggregation engine.
Fetches and parses RSS/Atom feeds from international news sources.
Supports filtering by category, keyword, and time range.
"""
def __init__(self, config: Optional[dict] = None):
super().__init__("RSS News", config)
timeout = (config or {}).get("timeout", 30)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"User-Agent": (config or {}).get(
"user_agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36",
),
"Accept": "application/rss+xml, application/xml, text/xml, */*",
},
follow_redirects=True,
)
def build_url(self, query: str, **kwargs) -> str:
"""Build search URL - for RSS this is not a search URL but a feed URL."""
feed_name = kwargs.get("feed", "BBC World")
feed = RSS_FEEDS.get(feed_name, RSS_FEEDS["BBC World"])
return feed["url"]
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
"""
Search news across RSS feeds. If query is empty, returns latest headlines.
Args:
query: Search keyword to filter articles (case-insensitive).
If empty/blank, returns all headlines.
max_results: Maximum articles per feed.
feeds: Specific feeds to query (list of feed names or "all").
category: Filter by category (news, business, technology).
hours: Only include articles from the last N hours.
Returns:
List of SearchResult objects sorted by date (newest first).
"""
feeds_to_query = self._select_feeds(
feed_names=kwargs.get("feeds", "all"),
category=kwargs.get("category"),
)
results = []
for feed_name, feed_info in feeds_to_query.items():
try:
response = await self._client.get(feed_info["url"])
response.raise_for_status()
parsed = self._parse_feed(response.text, feed_name, feed_info)
for article in parsed:
# Filter by keyword if query provided
if query:
q = query.lower()
if q in article.title.lower() or q in article.snippet.lower():
results.append(article)
else:
results.append(article)
except httpx.HTTPError as e:
logger.warning(f"RSS feed '{feed_name}' failed: {e}")
continue
# Apply time filter
hours = kwargs.get("hours")
if hours:
from datetime import timedelta
cutoff = datetime.now(timezone.utc) - timedelta(hours=int(hours))
results = [r for r in results if r.extra.get("parsed_date", datetime.max.replace(tzinfo=timezone.utc)) >= cutoff]
# Sort by date (newest first), limit results
results.sort(key=lambda r: r.extra.get("parsed_date", datetime.min.replace(tzinfo=timezone.utc)), reverse=True)
return results[:max_results * max(1, len(feeds_to_query))]
def _select_feeds(self, feed_names="all", category=None):
"""Select feeds based on filters."""
if feed_names != "all":
if isinstance(feed_names, str):
feed_names = [feed_names]
return {k: v for k, v in RSS_FEEDS.items() if k in feed_names}
if category:
return {k: v for k, v in RSS_FEEDS.items() if v.get("category") == category}
return RSS_FEEDS
def _parse_feed(self, xml_text: str, feed_name: str, feed_info: dict) -> list[SearchResult]:
"""Parse RSS/Atom XML into SearchResult objects."""
results = []
try:
root = ET.fromstring(xml_text)
except ET.ParseError as e:
logger.error(f"Failed to parse RSS from {feed_name}: {e}")
return []
# Detect RSS vs Atom format
if root.tag == "rss":
return self._parse_rss(root, feed_name, feed_info)
elif root.tag.endswith("feed"):
return self._parse_atom(root, feed_name, feed_info)
else:
logger.warning(f"Unknown feed format for {feed_name}: {root.tag}")
return results
def _parse_rss(self, root: ET.Element, feed_name: str, feed_info: dict) -> list[SearchResult]:
"""Parse RSS 2.0 format."""
results = []
namespaces = {"content": "http://purl.org/rss/1.0/modules/content/"}
for i, item in enumerate(root.findall(".//item")):
title = item.findtext("title", "")
link = item.findtext("link", "")
description = item.findtext("description", "")
pub_date_str = item.findtext("pubDate", "")
creator = item.findtext("dc:creator", "") or item.findtext("author", "")
# Clean up description (remove HTML tags)
import re
snippet = re.sub(r"<[^>]+>", "", description).strip()
# Truncate
snippet = snippet[:300] + "..." if len(snippet) > 300 else snippet
# Parse date
parsed_date = self._parse_date(pub_date_str)
results.append(
SearchResult(
title=title,
url=link,
snippet=snippet,
source=feed_name,
rank=i + 1,
category=feed_info.get("category", "news"),
extra={
"feed_name": feed_name,
"author": creator,
"published": pub_date_str,
"parsed_date": parsed_date,
"feed_url": feed_info["url"],
},
)
)
return results
def _parse_atom(self, root: ET.Element, feed_name: str, feed_info: dict) -> list[SearchResult]:
"""Parse Atom format."""
results = []
ns = {"a": "http://www.w3.org/2005/Atom"}
feed_title = root.findtext("a:title", "", ns)
for i, entry in enumerate(root.findall("a:entry", ns)):
title = entry.findtext("a:title", "", ns)
link_el = entry.find("a:link", ns)
link = link_el.get("href", "") if link_el is not None else ""
summary = entry.findtext("a:summary", "", ns) or entry.findtext("a:content", "", ns)
published = entry.findtext("a:published", "", ns) or entry.findtext("a:updated", "", ns)
author = entry.findtext("a:author/a:name", "", ns)
import re
snippet = re.sub(r"<[^>]+>", "", summary).strip()
snippet = snippet[:300] + "..." if len(snippet) > 300 else snippet
parsed_date = self._parse_date(published)
results.append(
SearchResult(
title=title,
url=link,
snippet=snippet,
source=feed_name,
rank=i + 1,
category=feed_info.get("category", "news"),
extra={
"feed_name": feed_name,
"author": author,
"published": published,
"parsed_date": parsed_date,
"feed_url": feed_info["url"],
},
)
)
return results
def _parse_date(self, date_str: str) -> datetime:
"""Parse various date formats to datetime."""
if not date_str:
return datetime.min.replace(tzinfo=timezone.utc)
formats = [
"%a, %d %b %Y %H:%M:%S %z", # RSS: Sat, 01 Jan 2024 12:00:00 +0000
"%a, %d %b %Y %H:%M:%S %Z", # RSS with timezone name
"%Y-%m-%dT%H:%M:%S%z", # Atom ISO 8601
"%Y-%m-%dT%H:%M:%S.%f%z", # Atom with microseconds
"%Y-%m-%dT%H:%M:%SZ", # Atom UTC
"%Y-%m-%d", # Simple date
]
for fmt in formats:
try:
return datetime.strptime(date_str.strip(), fmt).replace(tzinfo=timezone.utc)
except (ValueError, TypeError):
continue
return datetime.min.replace(tzinfo=timezone.utc)
async def close(self):
await self._client.aclose()
+117
View File
@@ -0,0 +1,117 @@
"""
华尔街见闻 (wallstreetcn.com) news search engine.
Scrapes wallstreetcn.com search results for financial news.
"""
from __future__ import annotations
import logging
from typing import Optional
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
class WallStreetCNEngine(SearchEngine):
"""华尔街见闻 (wallstreetcn.com) news search engine."""
SEARCH_URL = "https://wallstreetcn.com/search"
def __init__(self, config: Optional[dict] = None):
super().__init__("wallstreetcn", config)
timeout = (config or {}).get("timeout", 15)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"User-Agent": (config or {}).get(
"user_agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36",
),
"Referer": "https://wallstreetcn.com/",
},
follow_redirects=True,
)
def build_url(self, query: str, **kwargs) -> str:
return f"{self.SEARCH_URL}?q={quote_plus(query)}"
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search 华尔街见闻 and parse results."""
url = self.build_url(query)
try:
response = await self._client.get(url)
response.raise_for_status()
return self._parse_results(response.text, query, max_results)
except httpx.HTTPError as e:
logger.error(f"华尔街见闻 search failed: {e}")
return []
def _parse_results(
self, html: str, query: str, max_results: int
) -> list[SearchResult]:
"""Parse 华尔街见闻 search page."""
soup = BeautifulSoup(html, "lxml")
results = []
articles = soup.select(
"a[href*='/article/'], .search-result-item, article, .item"
)
for i, article in enumerate(articles[:max_results]):
title_el = article.select_one("h3, h4, .title, a")
link = article.get("href", "")
snippet_el = article.select_one(
".summary, .desc, .abstract, p"
)
title = title_el.get_text(strip=True) if title_el else ""
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if not title and not link:
continue
full_url = (
link
if link.startswith("http")
else f"https://wallstreetcn.com{link}"
)
results.append(
SearchResult(
title=title or f"华尔街见闻: {query}",
url=full_url,
snippet=snippet or f"Search result from 华尔街见闻",
source=self.name,
rank=i + 1,
category="news",
extra={"region": "cn"},
)
)
if not results:
search_url = self.build_url(query)
results.append(
SearchResult(
title=f"华尔街见闻: {query}",
url=search_url,
snippet=f"Search 华尔街见闻 for '{query}' (site may require JS)",
source=self.name,
rank=1,
category="news",
extra={"search_url": search_url, "note": "JS-rendered page"},
)
)
return results
async def close(self):
await self._client.aclose()
+1
View File
@@ -0,0 +1 @@
"""Social media search implementations (Twitter/X, etc.)."""
+208
View File
@@ -0,0 +1,208 @@
"""
Twitter/X search module.
Generates search URLs for X.com (formerly Twitter) search and
optionally parses search result pages.
"""
from __future__ import annotations
import logging
from typing import Optional
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
class TwitterSearchEngine(SearchEngine):
"""
Twitter/X search engine.
Provides search URL generation and optional result scraping.
Supports basic keyword search, user search, and advanced queries.
"""
SEARCH_URL = "https://twitter.com/search"
BASE_URL = "https://twitter.com"
def __init__(self, config: Optional[dict] = None):
super().__init__("Twitter", config)
timeout = (config or {}).get("timeout", 15)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"User-Agent": (config or {}).get(
"user_agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36",
),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
},
follow_redirects=True,
)
def build_url(self, query: str, search_type: str = "top", **kwargs) -> str:
"""
Build Twitter/X search URL.
Args:
query: Search keyword.
search_type: "top", "latest", "people", "photos", "videos"
**kwargs: Additional URL parameters.
Returns:
Search URL string.
"""
encoded_query = quote_plus(query)
url = f"{self.SEARCH_URL}?q={encoded_query}&src=typed_query"
if search_type == "latest":
url += "&f=live"
elif search_type == "people":
url += "&f=user"
elif search_type == "photos":
url += "&f=image"
elif search_type == "videos":
url += "&f=video"
if kwargs.get("lang"):
url += f"&l={kwargs['lang']}"
if kwargs.get("until"):
url += f"&until={kwargs['until']}"
if kwargs.get("since"):
url += f"&since={kwargs['since']}"
return url
def build_advanced_url(self, **params) -> str:
"""
Build advanced Twitter search URL with operators.
Supported operators:
from:username - Tweets from user
to:username - Tweets to user
@username - Mentions
#hashtag - Hashtag
filter:links - Tweets with links
filter:images - Tweets with images
min_retweets:N - Minimum retweets
min_faves:N - Minimum likes
lang:en - Language
Returns:
Advanced search URL.
"""
parts = []
if params.get("from_user"):
parts.append(f"from:{params['from_user']}")
if params.get("to_user"):
parts.append(f"to:{params['to_user']}")
if params.get("hashtag"):
parts.append(f"#{params['hashtag']}")
if params.get("keyword"):
parts.append(params["keyword"])
if params.get("min_retweets"):
parts.append(f"min_retweets:{params['min_retweets']}")
if params.get("min_faves"):
parts.append(f"min_faves:{params['min_faves']}")
if params.get("lang"):
parts.append(f"lang:{params['lang']}")
if params.get("filter"):
parts.append(f"filter:{params['filter']}")
query = " ".join(parts)
return self.build_url(query, search_type=params.get("search_type", "top"))
async def search(self, query: str, max_results: int = 10, search_type: str = "top", **kwargs) -> list[SearchResult]:
"""
Search Twitter/X and optionally parse page content.
Args:
query: Search keyword.
max_results: Maximum results (used for URL generation).
search_type: "top", "latest", "people", "photos", "videos"
**kwargs: Additional options.
Returns:
List of SearchResult objects.
"""
url = self.build_url(query, search_type=search_type, **kwargs)
# Try to scrape the search page for basic info
try:
response = await self._client.get(url)
if response.status_code == 200:
soup = BeautifulSoup(response.text, "lxml")
results = self._parse_page(soup, max_results)
if results:
return results
except Exception as e:
logger.debug(f"Twitter page scraping failed (expected): {e}")
# Fall back to URL-based result
result_type_map = {
"top": "Top tweets",
"latest": "Latest tweets",
"people": "People",
"photos": "Photos",
"videos": "Videos",
}
desc = result_type_map.get(search_type, "Tweets")
return [
SearchResult(
title=f"Twitter: {query} ({desc})",
url=url,
snippet=f"Search Twitter/X for '{query}'. Note: Twitter may require login.",
source=self.name,
rank=1,
category="social",
extra={"search_type": search_type, "twitter_url": url},
)
]
def _parse_page(self, soup, max_results: int) -> list[SearchResult]:
"""Attempt to extract tweet info from search page."""
results = []
# Twitter uses JS rendering, so HTML parsing may yield limited results
tweets = soup.select("article[data-testid='tweet'], div.tweet, .content")
for i, tweet in enumerate(tweets[:max_results]):
text_el = tweet.select_one(
"[data-testid='tweetText'], .tweet-text, p[lang]"
)
link_el = tweet.select_one("a[href*='/status/'], time")
user_el = tweet.select_one(
"[data-testid='User-Name'], .username, .fullname"
)
text = text_el.get_text(strip=True) if text_el else ""
user = user_el.get_text(strip=True) if user_el else ""
tweet_url = ""
if link_el:
href = link_el.get("href", "")
tweet_url = f"{self.BASE_URL}{href}" if href.startswith("/") else href
if text:
results.append(
SearchResult(
title=f"@{user}: {text[:80]}..." if user else text[:80],
url=tweet_url or self.build_url(text[:30]),
snippet=text,
source=self.name,
rank=i + 1,
category="social",
extra={"user": user} if user else {},
)
)
return results
async def close(self):
await self._client.aclose()
+183
View File
@@ -0,0 +1,183 @@
"""
Unified search entry point.
Provides a single interface to search across all configured sources.
"""
from __future__ import annotations
import asyncio
import logging
from typing import Any, Optional
from scripts.core.base import SearchResult
from scripts.core.config_loader import (
get_search_engines, get_news_sources, get_academic_sources,
get_wechat_sources, get_special_engines, get_social_sources,
get_all_sources, get_engine,
)
from scripts.engines.url_engine import UrlSearchEngine
from scripts.engines.parser_engines import DuckDuckGoEngine
from scripts.news.cls import CLSNewsEngine
from scripts.news.wallstreetcn import WallStreetCNEngine
from scripts.news.rss import RSSNewsEngine
from scripts.academic.arxiv import ArXivEngine
from scripts.wechat.sogou_weixin import SogouWeChatEngine
from scripts.social.twitter import TwitterSearchEngine
logger = logging.getLogger(__name__)
SOURCE_CATEGORIES = {
"web": get_search_engines,
"news": get_news_sources,
"academic": get_academic_sources,
"wechat": get_wechat_sources,
"social": get_social_sources,
"special": get_special_engines,
}
class UnifiedSearch:
"""Unified search client that aggregates results from multiple sources."""
def __init__(self, config: Optional[dict[str, Any]] = None):
self.config = config or {}
self._cache: dict[str, list[SearchResult]] = {}
self._engines: dict[str, Any] = {}
async def __aenter__(self):
return self
async def __aexit__(self, *args):
await self.close()
async def search(
self, query: str, sources: Optional[list[str]] = None,
max_results: int = 10, force_refresh: bool = False, **kwargs,
) -> list[SearchResult]:
"""Search across selected sources and aggregate results."""
if not sources or "all" in sources:
sources = list(SOURCE_CATEGORIES.keys())
cache_key = f"{sorted(sources) if isinstance(sources, list) else sources}:{query}"
if not force_refresh and cache_key in self._cache:
return self._cache[cache_key]
tasks = []
for source_spec in sources:
if source_spec in SOURCE_CATEGORIES:
for engine_def in SOURCE_CATEGORIES[source_spec]():
engine = self._get_engine(engine_def)
if engine:
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
else:
engine_def = get_engine(source_spec)
if engine_def:
engine = self._get_engine(engine_def)
if engine:
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
if not tasks:
return []
results_lists = await asyncio.gather(*tasks, return_exceptions=True)
all_results: list[SearchResult] = []
for results in results_lists:
if isinstance(results, Exception):
logger.error(f"Search task failed: {results}")
continue
all_results.extend(results)
self._cache[cache_key] = all_results
return all_results
def _get_engine(self, engine_def: dict[str, Any]):
"""Get or create an engine instance by definition."""
name = engine_def.get("name", "")
engine_type = engine_def.get("type", "url")
if name in self._engines:
return self._engines[name]
if name == "DuckDuckGo":
engine = DuckDuckGoEngine(self.config)
elif "cls" in name.lower():
engine = CLSNewsEngine(self.config)
elif "wallstreet" in name.lower() or "华尔街" in name:
engine = WallStreetCNEngine(self.config)
elif name == "ArXiv" or engine_type == "api":
engine = ArXivEngine(self.config)
elif "微信" in name or "wechat" in name.lower() or "weixin" in name.lower():
engine = SogouWeChatEngine(self.config)
elif "twitter" in name.lower():
engine = TwitterSearchEngine(self.config)
elif engine_type == "news" and name not in ("cls", "wallstreetcn"):
engine = RSSNewsEngine(self.config)
elif engine_type == "social":
engine = TwitterSearchEngine(self.config) if "twitter" in name.lower() else UrlSearchEngine(name, engine_def)
else:
engine = UrlSearchEngine(name, engine_def)
self._engines[name] = engine
return engine
async def _safe_search(self, engine, query, max_results, **kwargs):
try:
return await engine.search(query, max_results=max_results, **kwargs)
except Exception as e:
logger.error(f"Engine {engine.name} failed: {e}")
return []
async def search_engines(self, query, region=None, max_results=10):
"""Quick search across web search engines."""
engines = get_search_engines()
if region:
engines = [e for e in engines if e.get("region") == region]
return await self.search(query, sources=[e["name"] for e in engines], max_results=max_results)
async def search_news(self, query, max_results=10, **kwargs):
"""Quick search news sources (CN news + RSS feeds)."""
return await self.search(query, sources=["news"], max_results=max_results, **kwargs)
async def search_academic(self, query, max_results=10, **kwargs):
"""Quick search academic papers."""
return await self.search(query, sources=["academic"], max_results=max_results, **kwargs)
async def search_wechat(self, query, max_results=10):
"""Quick search WeChat articles."""
return await self.search(query, sources=["wechat"], max_results=max_results)
async def search_social(self, query, max_results=10, **kwargs):
"""Quick search social media (Twitter/X)."""
return await self.search(query, sources=["social"], max_results=max_results, **kwargs)
async def search_rss(self, query="", max_results=10, **kwargs):
"""Quick search RSS news feeds."""
engine = RSSNewsEngine(self.config)
return await engine.search(query, max_results=max_results, **kwargs)
async def get_search_urls(self, query, sources=None):
"""Get search URLs without executing searches."""
if not sources:
sources = [e["name"] for e in get_search_engines()]
urls = {}
for name in sources:
engine_def = get_engine(name)
if engine_def:
urls[name] = UrlSearchEngine(name, engine_def).build_url(query)
return urls
async def get_twitter_search_url(self, query, search_type="top"):
"""Get Twitter/X search URL for browser/web_fetch usage."""
engine = TwitterSearchEngine(self.config)
return engine.build_url(query, search_type=search_type)
async def close(self):
"""Close all engine resources."""
for engine in self._engines.values():
try:
await engine.close()
except Exception:
pass
+1
View File
@@ -0,0 +1 @@
"""WeChat search engines."""
+325
View File
@@ -0,0 +1,325 @@
"""
Sogou WeChat (搜狗微信) search engine.
Searches WeChat public account articles via weixin.sogou.com.
Anti-scraping strategy (inspired by Node.js wechat-article-search skill):
- Cookie pre-fetch from v.sogou.com to get SNUID
- User-Agent rotation with a pool of real browser UAs
- Random delays between requests (500-1500ms)
- Retry logic with exponential backoff
"""
from __future__ import annotations
import asyncio
import logging
import random
import re
from typing import Optional
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
# Real browser User-Agent pool for rotation
USER_AGENTS = [
# Chrome
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
# Safari
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1",
# Firefox
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0",
# Edge
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
]
class SogouWeChatEngine(SearchEngine):
"""搜狗微信搜索 - searches WeChat public account articles."""
SEARCH_URL = "https://weixin.sogou.com/weixin"
COOKIE_URL = "https://v.sogou.com"
def __init__(self, config: Optional[dict] = None):
super().__init__("weixin", config)
timeout = (config or {}).get("timeout", 15)
self._ua = random.choice(USER_AGENTS)
self._cookies: dict[str, str] = {}
self._client = httpx.AsyncClient(
timeout=timeout,
headers=self._build_headers(),
follow_redirects=False, # Don't auto-follow to detect anti-scraping
)
def _build_headers(self) -> dict[str, str]:
return {
"User-Agent": self._ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Accept-Encoding": "gzip, deflate",
"Referer": "https://weixin.sogou.com/",
"DNT": "1",
"Connection": "keep-alive",
}
def _random_delay(self):
"""Random delay between 500-1500ms to mimic human browsing."""
delay = random.uniform(0.5, 1.5)
return asyncio.sleep(delay)
def _rotate_ua(self):
"""Rotate to a new random User-Agent."""
self._ua = random.choice(USER_AGENTS)
async def _get_cookie(self) -> dict[str, str]:
"""Pre-fetch cookie from v.sogou.com to get SNUID."""
try:
response = await self._client.get(
self.COOKIE_URL,
headers={
"User-Agent": self._ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9",
},
)
cookies = {}
for cookie in response.cookies.jar:
if isinstance(cookie, tuple):
# SimpleCookie format: (key, value)
cookies[cookie[0]] = cookie[1]
else:
cookies[cookie.name] = cookie.value
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com")
return cookies
except Exception as e:
logger.warning(f"Cookie pre-fetch failed: {e}")
return {}
def build_url(self, query: str, search_type: int = 2, page: int = 1, **kwargs) -> str:
"""
Build Sogou WeChat search URL.
Args:
query: Search keyword.
search_type: 2=articles, 1=accounts.
page: Page number (starts at 1).
"""
return (
f"{self.SEARCH_URL}?query={quote_plus(query)}"
f"&s_from=input&_sug_=n&type={search_type}"
f"&page={page}&ie=utf8"
)
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search WeChat articles via Sogou and parse results."""
# Step 1: Get cookie first
if not self._cookies:
self._cookies = await self._get_cookie()
await self._random_delay()
# Step 2: Search with retry
max_pages = min(5, (max_results + 9) // 10)
all_results = []
for page in range(1, max_pages + 1):
url = self.build_url(query, page=page, **kwargs)
results = await self._search_page(url, query, max_results - len(all_results))
all_results.extend(results)
if len(all_results) >= max_results:
break
# Delay between pages
await self._random_delay()
if not all_results:
# Fallback: try legacy wx.sogou.com
return await self._legacy_search(query, max_results, **kwargs)
return all_results[:max_results]
async def _search_page(
self, url: str, query: str, max_results: int
) -> list[SearchResult]:
"""Search a single page with retry logic."""
for attempt in range(3):
try:
# Rotate UA on each retry
if attempt > 0:
self._rotate_ua()
self._client.headers.update({"User-Agent": self._ua})
if self._cookies:
pass # Keep existing cookies
response = await self._client.get(
url,
cookies=self._cookies,
follow_redirects=False,
)
# Check for anti-scraping page
if response.status_code == 302:
# Got redirected - likely hit anti-scraping
logger.warning(f"Sogou anti-scraping triggered (302), attempt {attempt + 1}")
# Try to get new cookie
self._cookies = await self._get_cookie()
await self._random_delay()
continue
if response.status_code != 200:
logger.warning(f"Sogou returned {response.status_code}, attempt {attempt + 1}")
await self._random_delay()
continue
html = response.text
# Check for captcha/block page
if "请输入验证码" in html or "访问频率过高" in html:
logger.warning(f"Sogou captcha triggered, attempt {attempt + 1}")
await asyncio.sleep(random.uniform(2, 4))
self._cookies = await self._get_cookie()
continue
return self._parse_results(html, query, max_results)
except (httpx.HTTPError, httpx.TimeoutException) as e:
logger.warning(f"Sogou search attempt {attempt + 1} failed: {e}")
await self._random_delay()
continue
return []
async def _legacy_search(
self, query: str, max_results: int, **kwargs
) -> list[SearchResult]:
"""Fallback: try the legacy wx.sogou.com domain."""
legacy_url = f"https://wx.sogou.com/weixin?type=2&query={quote_plus(query)}"
try:
response = await self._client.get(
legacy_url,
cookies=self._cookies,
follow_redirects=True,
)
if response.status_code == 200:
return self._parse_results(response.text, query, max_results)
except Exception:
pass
# Ultimate fallback
search_url = self.build_url(query)
return [
SearchResult(
title=f"微信搜索: {query}",
url=search_url,
snippet=f"Search WeChat for '{query}' via Sogou",
source=self.name,
rank=1,
category="wechat",
extra={"search_url": search_url},
)
]
def _parse_results(
self, html: str, query: str, max_results: int
) -> list[SearchResult]:
"""Parse Sogou WeChat search results."""
soup = BeautifulSoup(html, "lxml")
results = []
# Main selector: ul.news-list > li (standard Sogou WeChat structure)
items = soup.select("ul.news-list > li")
if not items:
# Fallback selectors
items = soup.select(
".news-list2 li, .wx-rb-item, .results .result, "
".item, .news-box .news-list li"
)
for i, item in enumerate(items[:max_results]):
title_el = item.select_one("h3 a")
if not title_el:
title_el = item.select_one(".tit a, .title a, h3")
snippet_el = item.select_one("p.txt-info, .txt-info, .summary, .des, p.info")
source_el = item.select_one(
".all-time-y2 a.account, a.account, "
".account, .source, .s-p, .from"
)
title = title_el.get_text(strip=True) if title_el else ""
link = title_el.get("href", "") if title_el and hasattr(title_el, "get") else ""
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
account = source_el.get_text(strip=True) if source_el else ""
# Parse time from inline scripts or text
time_text = self._parse_time(item)
if not title:
continue
# Make sure URL is absolute
if link and not link.startswith("http"):
link = f"https://weixin.sogou.com{link}"
results.append(
SearchResult(
title=title,
url=link,
snippet=snippet or f"WeChat article from {account or 'unknown'}",
source=self.name,
rank=(i + 1),
category="wechat",
extra={
"account": account,
"time": time_text,
"search_url": self.build_url(query),
},
)
)
return results
def _parse_time(self, item) -> str:
"""Extract publish time from search result item."""
# Try script tag with Unix timestamp (Sogou encodes dates in scripts)
for script in item.select("script"):
text = script.string or ""
# Look for Unix timestamp pattern
match = re.search(r'\d{10}', text)
if match:
import datetime
ts = int(match.group())
try:
dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone(datetime.timedelta(hours=8)))
return dt.strftime("%Y-%m-%d %H:%M")
except (OSError, ValueError):
pass
# Try inline text with relative time like "2小时前", "3天前"
time_text = item.get_text()
relative_match = re.search(r'(\d+)\s*(小时|分钟|天|周|月)前', time_text)
if relative_match:
return f"{relative_match.group(1)}{relative_match.group(2)}前"
# Try standard date patterns
date_match = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', time_text)
if date_match:
return date_match.group(1)
return ""
async def close(self):
await self._client.aclose()