refactor: restructure project with CLI tool, multi-engine support, and skill-builder compliant SKILL.md
- Replace setup.py with pyproject.toml (uv-compatible) - Add CLI entry point (web-search) with 9 subcommands - Add multi-engine architecture (14 web + 7 special sources) - Add WeChat/CLS/WallStreetCN/ArXiv/Twitter search engines - Remove old src/ and examples/ directories - Remove redundant requirements.txt and setup.py - Update README.md and SKILL.md with full documentation - SKILL.md now compliant with skill-builder standards (frontmatter, triggers, structure)
This commit is contained in:
1 parent
80ac14491e
commit
87935b59ce
29 files changed
+2364
-354
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Web Search Skills package."""
|
||||
@@ -0,0 +1 @@
|
||||
"""Academic search engines."""
|
||||
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
ArXiv academic paper search engine.
|
||||
|
||||
Uses the ArXiv API (export.arxiv.org/api/query) to search for papers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ArXivEngine(SearchEngine):
|
||||
"""ArXiv academic paper search engine via official API."""
|
||||
|
||||
API_URL = "https://export.arxiv.org/api/query"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("ArXiv", config)
|
||||
timeout = (config or {}).get("timeout", 30)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"ArXivSearchSkill/1.0",
|
||||
),
|
||||
"Accept": "application/atom+xml",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, max_results: int = 10, **kwargs) -> str:
|
||||
search_field = kwargs.get("search_field", "all")
|
||||
return (
|
||||
f"{self.API_URL}?search_query={search_field}:{quote_plus(query)}"
|
||||
f"&max_results={max_results}&sortBy=submittedDate&sortOrder=descending"
|
||||
)
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search ArXiv papers."""
|
||||
url = self.build_url(query, max_results=max_results, **kwargs)
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return self._parse_response(response.text, max_results)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"ArXiv search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_response(self, xml_text: str, max_results: int) -> list[SearchResult]:
|
||||
"""Parse ArXiv Atom XML response."""
|
||||
ns = {
|
||||
"atom": "http://www.w3.org/2005/Atom",
|
||||
"arxiv": "http://arxiv.org/schemas/atom",
|
||||
}
|
||||
results = []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError as e:
|
||||
logger.error(f"ArXiv XML parse error: {e}")
|
||||
return []
|
||||
|
||||
for i, entry in enumerate(root.findall("atom:entry", ns)[:max_results]):
|
||||
title = entry.findtext("atom:title", "", ns)
|
||||
summary = entry.findtext("atom:summary", "", ns)
|
||||
published = entry.findtext("atom:published", "", ns)
|
||||
updated = entry.findtext("atom:updated", "", ns)
|
||||
|
||||
link_el = entry.find("atom:link", ns)
|
||||
link = link_el.get("href", "") if link_el is not None else ""
|
||||
|
||||
authors = []
|
||||
for author in entry.findall("atom:author", ns):
|
||||
name = author.findtext("atom:name", "", ns)
|
||||
if name:
|
||||
authors.append(name)
|
||||
|
||||
# Clean title and summary
|
||||
title = title.replace("\n", " ").strip()
|
||||
snippet = summary.replace("\n", " ").strip()[:300]
|
||||
|
||||
parsed_date = self._parse_date(published) if published else (
|
||||
self._parse_date(updated) if updated else
|
||||
datetime.min.replace(tzinfo=timezone.utc)
|
||||
)
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title or f"ArXiv: {summary[:60]}",
|
||||
url=link,
|
||||
snippet=snippet or "No abstract available",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="academic",
|
||||
extra={
|
||||
"published": published or "",
|
||||
"updated": updated or "",
|
||||
"authors": ", ".join(authors),
|
||||
"parsed_date": parsed_date,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _parse_date(self, date_str: str) -> datetime:
|
||||
"""Parse ArXiv date format."""
|
||||
try:
|
||||
# ArXiv format: 2024-01-01T00:00:00Z
|
||||
return datetime.strptime(
|
||||
date_str.strip(), "%Y-%m-%dT%H:%M:%SZ"
|
||||
).replace(tzinfo=timezone.utc)
|
||||
except (ValueError, TypeError):
|
||||
return datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
+458
@@ -0,0 +1,458 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Web Search Skills CLI — search across 20+ sources from the command line.
|
||||
|
||||
Usage:
|
||||
web-search <command> <query> [options]
|
||||
|
||||
Commands:
|
||||
search Search all sources (default)
|
||||
web Search web search engines (Google, Baidu, DuckDuckGo, etc.)
|
||||
news Search news (财联社, 华尔街见闻, RSS)
|
||||
wechat Search WeChat public account articles
|
||||
academic Search academic papers (ArXiv)
|
||||
social Search social media (Twitter/X)
|
||||
rss Search RSS news feeds
|
||||
sources List all available sources
|
||||
urls Get search URLs without executing
|
||||
|
||||
Options:
|
||||
-n, --max-results N Max results per source (default: 10)
|
||||
-j, --json Output as JSON
|
||||
-o, --output FILE Save results to file
|
||||
-s, --source NAME Specific source name
|
||||
-r, --region REGION Region filter (cn/global)
|
||||
-v, --verbose Show debug logs
|
||||
-h, --help Show this help
|
||||
|
||||
Examples:
|
||||
web-search wechat 蒙面财经
|
||||
web-search news "人工智能" -n 5
|
||||
web-search web "Python async" -r global -j
|
||||
web-search search "量子计算" -o results.json
|
||||
web-search sources
|
||||
web-search urls "机器学习"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
# Ensure scripts package is importable
|
||||
_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR)
|
||||
if _PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, _PROJECT_ROOT)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.WARNING,
|
||||
format="%(levelname)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("cli")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lazy engine imports (import on demand to keep startup fast)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_unified():
|
||||
from scripts.unified_search import UnifiedSearch
|
||||
return UnifiedSearch
|
||||
|
||||
|
||||
def _get_sogou():
|
||||
from scripts.wechat.sogou_weixin import SogouWeChatEngine
|
||||
return SogouWeChatEngine
|
||||
|
||||
|
||||
def _get_rss():
|
||||
from scripts.news.rss import RSSNewsEngine
|
||||
return RSSNewsEngine
|
||||
|
||||
|
||||
def _get_twitter():
|
||||
from scripts.social.twitter import TwitterSearchEngine
|
||||
return TwitterSearchEngine
|
||||
|
||||
|
||||
def _get_ddg():
|
||||
from scripts.engines.parser_engines import DuckDuckGoEngine
|
||||
return DuckDuckGoEngine
|
||||
|
||||
|
||||
def _get_config_loader():
|
||||
from scripts.core.config_loader import (
|
||||
get_search_engines,
|
||||
get_news_sources,
|
||||
get_academic_sources,
|
||||
get_wechat_sources,
|
||||
get_social_sources,
|
||||
get_special_engines,
|
||||
get_all_sources,
|
||||
list_source_categories,
|
||||
)
|
||||
return {
|
||||
"get_search_engines": get_search_engines,
|
||||
"get_news_sources": get_news_sources,
|
||||
"get_academic_sources": get_academic_sources,
|
||||
"get_wechat_sources": get_wechat_sources,
|
||||
"get_social_sources": get_social_sources,
|
||||
"get_special_engines": get_special_engines,
|
||||
"get_all_sources": get_all_sources,
|
||||
"list_source_categories": list_source_categories,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result formatting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _format_result(r, fmt: str = "text") -> str:
|
||||
"""Format a single SearchResult."""
|
||||
if fmt == "json":
|
||||
return json.dumps({
|
||||
"rank": r.rank,
|
||||
"title": r.title,
|
||||
"url": r.url,
|
||||
"snippet": r.snippet[:300],
|
||||
"source": r.source,
|
||||
"category": r.category,
|
||||
"extra": dict(r.extra) if r.extra else {},
|
||||
}, ensure_ascii=False, indent=2)
|
||||
|
||||
lines = [f"[{r.rank}] {r.title}"]
|
||||
lines.append(f" URL: {r.url}")
|
||||
if r.snippet:
|
||||
lines.append(f" {r.snippet[:200]}")
|
||||
if r.extra:
|
||||
extra_parts = []
|
||||
for k, v in r.extra.items():
|
||||
if k == "account" and v:
|
||||
extra_parts.append(f"公众号: {v}")
|
||||
elif k == "time" and v:
|
||||
extra_parts.append(f"时间: {v}")
|
||||
elif k == "search_url":
|
||||
continue
|
||||
elif v and k not in ("parsed_date", "note", "feed_url"):
|
||||
extra_parts.append(f"{k}: {v}")
|
||||
if extra_parts:
|
||||
lines.append(f" ({'; '.join(extra_parts)})")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _print_results(results, fmt: str = "text"):
|
||||
"""Print search results."""
|
||||
if not results:
|
||||
print(" 未找到结果")
|
||||
return
|
||||
|
||||
if fmt == "json":
|
||||
output = json.dumps(
|
||||
[{
|
||||
"rank": r.rank,
|
||||
"title": r.title,
|
||||
"url": r.url,
|
||||
"snippet": r.snippet[:300],
|
||||
"source": r.source,
|
||||
"category": r.category,
|
||||
"extra": dict(r.extra) if r.extra else {},
|
||||
} for r in results],
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
print(output)
|
||||
else:
|
||||
for r in results:
|
||||
print(_format_result(r))
|
||||
print()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async searchers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _search_all(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search(query, max_results=max_results, **kwargs)
|
||||
finally:
|
||||
await us.close()
|
||||
|
||||
|
||||
async def _search_web(query: str, max_results: int, region: str = None, **kwargs) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search_engines(query, region=region, max_results=max_results)
|
||||
finally:
|
||||
await us.close()
|
||||
|
||||
|
||||
async def _search_news(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search_news(query, max_results=max_results, **kwargs)
|
||||
finally:
|
||||
await us.close()
|
||||
|
||||
|
||||
async def _search_wechat(query: str, max_results: int) -> list[Any]:
|
||||
engine = _get_sogou()()
|
||||
try:
|
||||
return await engine.search(query, max_results=max_results)
|
||||
finally:
|
||||
await engine.close()
|
||||
|
||||
|
||||
async def _search_academic(query: str, max_results: int) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search_academic(query, max_results=max_results)
|
||||
finally:
|
||||
await us.close()
|
||||
|
||||
|
||||
async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search_social(query, max_results=max_results, **kwargs)
|
||||
finally:
|
||||
await us.close()
|
||||
|
||||
|
||||
async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search_rss(query, max_results=max_results, **kwargs)
|
||||
finally:
|
||||
await us.close()
|
||||
|
||||
|
||||
async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.get_search_urls(query, sources=sources)
|
||||
finally:
|
||||
await us.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI actions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def cmd_sources(args):
|
||||
"""List all available search sources."""
|
||||
cl = _get_config_loader()
|
||||
categories = cl["list_source_categories"]()
|
||||
|
||||
print("可用搜索源:\n")
|
||||
for cat_key, cat_name in [
|
||||
("search_engines", "Web 搜索引擎"),
|
||||
("news_sources", "新闻源"),
|
||||
("academic_sources", "学术源"),
|
||||
("wechat_sources", "微信源"),
|
||||
("social_sources", "社交源"),
|
||||
("special_engines", "特殊源"),
|
||||
]:
|
||||
count = categories.get(cat_key, 0)
|
||||
if count == 0:
|
||||
continue
|
||||
print(f" {cat_name} ({count} 个)")
|
||||
|
||||
print(f"\n总计: {cl['get_all_sources']().__len__()} 个源")
|
||||
print("\n详细列表:")
|
||||
for s in cl["get_all_sources"]():
|
||||
print(f" - {s['name']:20s} ({s.get('region', '?')}) [{s.get('type', 'url')}]")
|
||||
|
||||
|
||||
def cmd_urls(args):
|
||||
"""Get search URLs without executing."""
|
||||
urls = asyncio.run(_get_urls(args.query, sources=args.source))
|
||||
for name, url in urls.items():
|
||||
print(f"{name}:")
|
||||
print(f" {url}")
|
||||
print()
|
||||
|
||||
|
||||
async def _run_search(args):
|
||||
"""Route search to the right engine(s)."""
|
||||
if args.verbose:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
method_map = {
|
||||
"search": _search_all,
|
||||
"web": _search_web,
|
||||
"news": _search_news,
|
||||
"wechat": _search_wechat,
|
||||
"academic": _search_academic,
|
||||
"social": _search_social,
|
||||
"rss": _search_rss,
|
||||
}
|
||||
|
||||
search_fn = method_map.get(args.command, _search_all)
|
||||
|
||||
kwargs = {}
|
||||
if args.command == "web" and args.region:
|
||||
kwargs["region"] = args.region
|
||||
|
||||
try:
|
||||
results = await search_fn(
|
||||
args.query,
|
||||
max_results=args.max_results,
|
||||
**kwargs,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"搜索失败: {e}", file=sys.stderr)
|
||||
if args.verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
# Print header
|
||||
cmd_names = {
|
||||
"search": "全源搜索",
|
||||
"web": "网页搜索",
|
||||
"news": "新闻搜索",
|
||||
"wechat": "微信公众号搜索",
|
||||
"academic": "学术搜索",
|
||||
"social": "社交搜索",
|
||||
"rss": "RSS 新闻搜索",
|
||||
}
|
||||
header = cmd_names.get(args.command, "搜索")
|
||||
print(f"{header}: {args.query}")
|
||||
print(f"共 {len(results)} 条结果\n")
|
||||
|
||||
_print_results(results, fmt="json" if args.json else "text")
|
||||
|
||||
# Save to file if requested
|
||||
if args.output:
|
||||
output_data = json.dumps(
|
||||
[{
|
||||
"query": args.query,
|
||||
"command": args.command,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"total": len(results),
|
||||
"results": [
|
||||
{
|
||||
"rank": r.rank,
|
||||
"title": r.title,
|
||||
"url": r.url,
|
||||
"snippet": r.snippet[:500],
|
||||
"source": r.source,
|
||||
"category": r.category,
|
||||
"extra": dict(r.extra) if r.extra else {},
|
||||
}
|
||||
for r in results
|
||||
],
|
||||
}],
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
output_path = os.path.abspath(args.output)
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
f.write(output_data)
|
||||
print(f"\n结果已保存至: {output_path}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="web-search",
|
||||
description="Web Search Skills CLI — multi-source search tool",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"command",
|
||||
nargs="?",
|
||||
default="search",
|
||||
choices=["search", "web", "news", "wechat", "academic", "social",
|
||||
"rss", "sources", "urls"],
|
||||
help="Search type (default: search)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"query",
|
||||
nargs="?",
|
||||
default="",
|
||||
help="Search keyword",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-n", "--max-results",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Max results per source (default: 10)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-j", "--json",
|
||||
action="store_true",
|
||||
help="Output as JSON",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o", "--output",
|
||||
type=str,
|
||||
help="Save results to JSON file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-s", "--source",
|
||||
type=str,
|
||||
nargs="*",
|
||||
help="Specific source name(s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-r", "--region",
|
||||
type=str,
|
||||
choices=["cn", "global"],
|
||||
help="Region filter (web search only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v", "--verbose",
|
||||
action="store_true",
|
||||
help="Show debug logs",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
# sources and urls commands don't need a query
|
||||
if args.command == "sources":
|
||||
cmd_sources(args)
|
||||
return
|
||||
|
||||
if args.command == "urls":
|
||||
if not args.query:
|
||||
print("错误: urls 命令需要提供搜索关键词", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
cmd_urls(args)
|
||||
return
|
||||
|
||||
# Other commands need a query
|
||||
if not args.query:
|
||||
print(f"错误: {args.command} 命令需要提供搜索关键词", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
asyncio.run(_run_search(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""Core data types and base classes."""
|
||||
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
Core data types and base classes for search engines.
|
||||
|
||||
Provides:
|
||||
- SearchResult: Standard result dataclass used across all engines
|
||||
- SearchEngine: Abstract base class for all search engine implementations
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""Represents a single search result."""
|
||||
title: str = ""
|
||||
url: str = ""
|
||||
snippet: str = ""
|
||||
source: str = ""
|
||||
rank: int = 0
|
||||
category: str = ""
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class SearchEngine(ABC):
|
||||
"""Abstract base class for search engines."""
|
||||
|
||||
def __init__(self, name: str, config: Optional[dict[str, Any]] = None):
|
||||
self.name = name
|
||||
self.config = config or {}
|
||||
|
||||
@abstractmethod
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Execute a search and return results."""
|
||||
...
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
"""Build a search URL for this engine (optional)."""
|
||||
return ""
|
||||
|
||||
async def close(self):
|
||||
"""Clean up resources."""
|
||||
pass
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Load and manage engine configuration from engines.json."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Optional
|
||||
|
||||
_CONFIG_PATH = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
|
||||
"config", "engines.json",
|
||||
)
|
||||
|
||||
|
||||
def _load_config(path: Optional[str] = None) -> dict[str, Any]:
|
||||
config_path = path or _CONFIG_PATH
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def get_search_engines(path: Optional[str] = None) -> list[dict[str, Any]]:
|
||||
return _load_config(path).get("search_engines", [])
|
||||
|
||||
|
||||
def get_news_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
|
||||
return _load_config(path).get("news_sources", [])
|
||||
|
||||
|
||||
def get_academic_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
|
||||
return _load_config(path).get("academic_sources", [])
|
||||
|
||||
|
||||
def get_wechat_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
|
||||
return _load_config(path).get("wechat_sources", [])
|
||||
|
||||
|
||||
def get_social_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
|
||||
return _load_config(path).get("social_sources", [])
|
||||
|
||||
|
||||
def get_special_engines(path: Optional[str] = None) -> list[dict[str, Any]]:
|
||||
return _load_config(path).get("special_engines", [])
|
||||
|
||||
|
||||
def get_engine(name: str, path: Optional[str] = None):
|
||||
config = _load_config(path)
|
||||
for category in ["search_engines", "news_sources", "academic_sources",
|
||||
"wechat_sources", "social_sources", "special_engines"]:
|
||||
for engine in config.get(category, []):
|
||||
if engine["name"].lower() == name.lower():
|
||||
return engine
|
||||
return None
|
||||
|
||||
|
||||
def get_all_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
|
||||
config = _load_config(path)
|
||||
all_sources = []
|
||||
for category in ["search_engines", "news_sources", "academic_sources",
|
||||
"wechat_sources", "social_sources", "special_engines"]:
|
||||
all_sources.extend(config.get(category, []))
|
||||
return all_sources
|
||||
|
||||
def list_source_categories(path: Optional[str] = None) -> dict[str, int]:
|
||||
"""Return a dict mapping category name to count of sources."""
|
||||
config = _load_config(path)
|
||||
categories = [
|
||||
"search_engines", "news_sources", "academic_sources",
|
||||
"wechat_sources", "social_sources", "special_engines",
|
||||
]
|
||||
return {cat: len(config.get(cat, [])) for cat in categories}
|
||||
@@ -0,0 +1 @@
|
||||
"""Search engine implementations."""
|
||||
@@ -0,0 +1,80 @@
|
||||
"""
|
||||
Parsing-based search engines (e.g. DuckDuckGo HTML scraper).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DuckDuckGoEngine(SearchEngine):
|
||||
"""DuckDuckGo HTML search engine (no API key required)."""
|
||||
|
||||
SEARCH_URL = "https://html.duckduckgo.com/html/"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("DuckDuckGo", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
"""Build DuckDuckGo search URL."""
|
||||
from urllib.parse import quote_plus
|
||||
return f"https://duckduckgo.com/?q={quote_plus(query)}"
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search DuckDuckGo and parse HTML results."""
|
||||
params = {"q": query}
|
||||
try:
|
||||
response = await self._client.post(self.SEARCH_URL, data=params)
|
||||
response.raise_for_status()
|
||||
return self._parse_results(response.text, max_results)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"DuckDuckGo search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_results(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
"""Parse DuckDuckGo HTML search results."""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
|
||||
for i, result in enumerate(soup.select(".result")[:max_results]):
|
||||
title_el = result.select_one(".result__title a")
|
||||
snippet_el = result.select_one(".result__snippet")
|
||||
if title_el:
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title_el.get_text(strip=True),
|
||||
url=title_el.get("href", ""),
|
||||
snippet=snippet_el.get_text(strip=True)
|
||||
if snippet_el else "",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="web",
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
@@ -0,0 +1,60 @@
|
||||
"""
|
||||
Generic URL-based search engine.
|
||||
|
||||
Builds search URLs from templates defined in engines.json
|
||||
and returns them as URL results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UrlSearchEngine(SearchEngine):
|
||||
"""
|
||||
Generic search engine that builds URLs from templates.
|
||||
|
||||
Reads URL templates from config (engines.json) and returns
|
||||
the constructed search URL as a SearchResult.
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, engine_def: dict[str, Any]):
|
||||
super().__init__(name, engine_def.get("config", {}))
|
||||
self.url_template = engine_def.get("url", "")
|
||||
self.region = engine_def.get("region", "global")
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
"""Build search URL by replacing {keyword} with encoded query."""
|
||||
if not self.url_template:
|
||||
return ""
|
||||
encoded = quote_plus(query)
|
||||
return self.url_template.replace("{keyword}", encoded)
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Return the search URL as a single result."""
|
||||
url = self.build_url(query, **kwargs)
|
||||
if not url:
|
||||
return []
|
||||
|
||||
return [
|
||||
SearchResult(
|
||||
title=f"{self.name}: {query}",
|
||||
url=url,
|
||||
snippet=f"Search {self.name} for '{query}'",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="web",
|
||||
extra={"url": url, "region": self.region},
|
||||
)
|
||||
]
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
@@ -0,0 +1 @@
|
||||
"""News search engines."""
|
||||
@@ -0,0 +1,115 @@
|
||||
"""
|
||||
财联社 (cls.cn) news search engine.
|
||||
|
||||
Scrapes cls.cn search results for financial news.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CLSNewsEngine(SearchEngine):
|
||||
"""财联社 (cls.cn) news search engine."""
|
||||
|
||||
SEARCH_URL = "https://www.cls.cn/searchPage"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("cls", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
"Referer": "https://www.cls.cn/",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
return f"{self.SEARCH_URL}?keyword={quote_plus(query)}&type=all"
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search 财联社 and parse results."""
|
||||
url = self.build_url(query)
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return self._parse_results(response.text, query, max_results)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"财联社 search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_results(
|
||||
self, html: str, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Parse 财联社 search page."""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
|
||||
# cls.cn typically uses dynamic rendering, so parsing may be limited
|
||||
articles = soup.select(
|
||||
"a[href*='/detail'], .news-item, .search-result-item, article"
|
||||
)
|
||||
for i, article in enumerate(articles[:max_results]):
|
||||
title_el = article.select_one("h3, h4, .title, .news-title")
|
||||
link = article.get("href", "")
|
||||
snippet_el = article.select_one(
|
||||
".summary, .desc, .abstract, p"
|
||||
)
|
||||
|
||||
title = title_el.get_text(strip=True) if title_el else ""
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if not title and not link:
|
||||
continue
|
||||
|
||||
full_url = link if link.startswith("http") else f"https://www.cls.cn{link}"
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title or f"财联社: {query}",
|
||||
url=full_url,
|
||||
snippet=snippet or f"Search result from 财联社",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={"region": "cn"},
|
||||
)
|
||||
)
|
||||
|
||||
if not results:
|
||||
# Fallback: return URL for manual browsing
|
||||
search_url = self.build_url(query)
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=f"财联社: {query}",
|
||||
url=search_url,
|
||||
snippet=f"Search 财联社 for '{query}' (site requires JS)",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="news",
|
||||
extra={"search_url": search_url, "note": "JS-rendered page"},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
@@ -0,0 +1,288 @@
|
||||
"""
|
||||
RSS news aggregation engine.
|
||||
|
||||
Fetches and parses RSS/Atom feeds from trusted news sources.
|
||||
Inspired by the news-summary skill's RSS-based approach.
|
||||
|
||||
Supported sources:
|
||||
- BBC (World, Business, Technology, Top Stories)
|
||||
- Reuters (World News)
|
||||
- NPR (US news)
|
||||
- Al Jazeera (Global South perspective)
|
||||
- 财联社 (RSS if available)
|
||||
- 华尔街见闻 (RSS if available)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# RSS feed definitions (from news-summary skill + additional sources)
|
||||
RSS_FEEDS = {
|
||||
"BBC World": {
|
||||
"url": "https://feeds.bbci.co.uk/news/world/rss.xml",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
},
|
||||
"BBC Top Stories": {
|
||||
"url": "https://feeds.bbci.co.uk/news/rss.xml",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
},
|
||||
"BBC Business": {
|
||||
"url": "https://feeds.bbci.co.uk/news/business/rss.xml",
|
||||
"category": "business",
|
||||
"lang": "en",
|
||||
},
|
||||
"BBC Technology": {
|
||||
"url": "https://feeds.bbci.co.uk/news/technology/rss.xml",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
},
|
||||
"Reuters": {
|
||||
"url": "https://www.reutersagency.com/feed/?best-regions=world&post_type=best",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
},
|
||||
"NPR": {
|
||||
"url": "https://feeds.npr.org/1001/rss.xml",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
},
|
||||
"Al Jazeera": {
|
||||
"url": "https://www.aljazeera.com/xml/rss/all.xml",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class RSSNewsEngine(SearchEngine):
|
||||
"""
|
||||
RSS-based news aggregation engine.
|
||||
|
||||
Fetches and parses RSS/Atom feeds from international news sources.
|
||||
Supports filtering by category, keyword, and time range.
|
||||
"""
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("RSS News", config)
|
||||
timeout = (config or {}).get("timeout", 30)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
"Accept": "application/rss+xml, application/xml, text/xml, */*",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
"""Build search URL - for RSS this is not a search URL but a feed URL."""
|
||||
feed_name = kwargs.get("feed", "BBC World")
|
||||
feed = RSS_FEEDS.get(feed_name, RSS_FEEDS["BBC World"])
|
||||
return feed["url"]
|
||||
|
||||
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
"""
|
||||
Search news across RSS feeds. If query is empty, returns latest headlines.
|
||||
|
||||
Args:
|
||||
query: Search keyword to filter articles (case-insensitive).
|
||||
If empty/blank, returns all headlines.
|
||||
max_results: Maximum articles per feed.
|
||||
feeds: Specific feeds to query (list of feed names or "all").
|
||||
category: Filter by category (news, business, technology).
|
||||
hours: Only include articles from the last N hours.
|
||||
|
||||
Returns:
|
||||
List of SearchResult objects sorted by date (newest first).
|
||||
"""
|
||||
feeds_to_query = self._select_feeds(
|
||||
feed_names=kwargs.get("feeds", "all"),
|
||||
category=kwargs.get("category"),
|
||||
)
|
||||
|
||||
results = []
|
||||
for feed_name, feed_info in feeds_to_query.items():
|
||||
try:
|
||||
response = await self._client.get(feed_info["url"])
|
||||
response.raise_for_status()
|
||||
|
||||
parsed = self._parse_feed(response.text, feed_name, feed_info)
|
||||
for article in parsed:
|
||||
# Filter by keyword if query provided
|
||||
if query:
|
||||
q = query.lower()
|
||||
if q in article.title.lower() or q in article.snippet.lower():
|
||||
results.append(article)
|
||||
else:
|
||||
results.append(article)
|
||||
|
||||
except httpx.HTTPError as e:
|
||||
logger.warning(f"RSS feed '{feed_name}' failed: {e}")
|
||||
continue
|
||||
|
||||
# Apply time filter
|
||||
hours = kwargs.get("hours")
|
||||
if hours:
|
||||
from datetime import timedelta
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(hours=int(hours))
|
||||
results = [r for r in results if r.extra.get("parsed_date", datetime.max.replace(tzinfo=timezone.utc)) >= cutoff]
|
||||
|
||||
# Sort by date (newest first), limit results
|
||||
results.sort(key=lambda r: r.extra.get("parsed_date", datetime.min.replace(tzinfo=timezone.utc)), reverse=True)
|
||||
return results[:max_results * max(1, len(feeds_to_query))]
|
||||
|
||||
def _select_feeds(self, feed_names="all", category=None):
|
||||
"""Select feeds based on filters."""
|
||||
if feed_names != "all":
|
||||
if isinstance(feed_names, str):
|
||||
feed_names = [feed_names]
|
||||
return {k: v for k, v in RSS_FEEDS.items() if k in feed_names}
|
||||
|
||||
if category:
|
||||
return {k: v for k, v in RSS_FEEDS.items() if v.get("category") == category}
|
||||
|
||||
return RSS_FEEDS
|
||||
|
||||
def _parse_feed(self, xml_text: str, feed_name: str, feed_info: dict) -> list[SearchResult]:
|
||||
"""Parse RSS/Atom XML into SearchResult objects."""
|
||||
results = []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError as e:
|
||||
logger.error(f"Failed to parse RSS from {feed_name}: {e}")
|
||||
return []
|
||||
|
||||
# Detect RSS vs Atom format
|
||||
if root.tag == "rss":
|
||||
return self._parse_rss(root, feed_name, feed_info)
|
||||
elif root.tag.endswith("feed"):
|
||||
return self._parse_atom(root, feed_name, feed_info)
|
||||
else:
|
||||
logger.warning(f"Unknown feed format for {feed_name}: {root.tag}")
|
||||
return results
|
||||
|
||||
def _parse_rss(self, root: ET.Element, feed_name: str, feed_info: dict) -> list[SearchResult]:
|
||||
"""Parse RSS 2.0 format."""
|
||||
results = []
|
||||
namespaces = {"content": "http://purl.org/rss/1.0/modules/content/"}
|
||||
|
||||
for i, item in enumerate(root.findall(".//item")):
|
||||
title = item.findtext("title", "")
|
||||
link = item.findtext("link", "")
|
||||
description = item.findtext("description", "")
|
||||
pub_date_str = item.findtext("pubDate", "")
|
||||
creator = item.findtext("dc:creator", "") or item.findtext("author", "")
|
||||
|
||||
# Clean up description (remove HTML tags)
|
||||
import re
|
||||
snippet = re.sub(r"<[^>]+>", "", description).strip()
|
||||
# Truncate
|
||||
snippet = snippet[:300] + "..." if len(snippet) > 300 else snippet
|
||||
|
||||
# Parse date
|
||||
parsed_date = self._parse_date(pub_date_str)
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title,
|
||||
url=link,
|
||||
snippet=snippet,
|
||||
source=feed_name,
|
||||
rank=i + 1,
|
||||
category=feed_info.get("category", "news"),
|
||||
extra={
|
||||
"feed_name": feed_name,
|
||||
"author": creator,
|
||||
"published": pub_date_str,
|
||||
"parsed_date": parsed_date,
|
||||
"feed_url": feed_info["url"],
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _parse_atom(self, root: ET.Element, feed_name: str, feed_info: dict) -> list[SearchResult]:
|
||||
"""Parse Atom format."""
|
||||
results = []
|
||||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||||
|
||||
feed_title = root.findtext("a:title", "", ns)
|
||||
|
||||
for i, entry in enumerate(root.findall("a:entry", ns)):
|
||||
title = entry.findtext("a:title", "", ns)
|
||||
link_el = entry.find("a:link", ns)
|
||||
link = link_el.get("href", "") if link_el is not None else ""
|
||||
summary = entry.findtext("a:summary", "", ns) or entry.findtext("a:content", "", ns)
|
||||
published = entry.findtext("a:published", "", ns) or entry.findtext("a:updated", "", ns)
|
||||
author = entry.findtext("a:author/a:name", "", ns)
|
||||
|
||||
import re
|
||||
snippet = re.sub(r"<[^>]+>", "", summary).strip()
|
||||
snippet = snippet[:300] + "..." if len(snippet) > 300 else snippet
|
||||
|
||||
parsed_date = self._parse_date(published)
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title,
|
||||
url=link,
|
||||
snippet=snippet,
|
||||
source=feed_name,
|
||||
rank=i + 1,
|
||||
category=feed_info.get("category", "news"),
|
||||
extra={
|
||||
"feed_name": feed_name,
|
||||
"author": author,
|
||||
"published": published,
|
||||
"parsed_date": parsed_date,
|
||||
"feed_url": feed_info["url"],
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _parse_date(self, date_str: str) -> datetime:
|
||||
"""Parse various date formats to datetime."""
|
||||
if not date_str:
|
||||
return datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
formats = [
|
||||
"%a, %d %b %Y %H:%M:%S %z", # RSS: Sat, 01 Jan 2024 12:00:00 +0000
|
||||
"%a, %d %b %Y %H:%M:%S %Z", # RSS with timezone name
|
||||
"%Y-%m-%dT%H:%M:%S%z", # Atom ISO 8601
|
||||
"%Y-%m-%dT%H:%M:%S.%f%z", # Atom with microseconds
|
||||
"%Y-%m-%dT%H:%M:%SZ", # Atom UTC
|
||||
"%Y-%m-%d", # Simple date
|
||||
]
|
||||
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(date_str.strip(), fmt).replace(tzinfo=timezone.utc)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
return datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
@@ -0,0 +1,117 @@
|
||||
"""
|
||||
华尔街见闻 (wallstreetcn.com) news search engine.
|
||||
|
||||
Scrapes wallstreetcn.com search results for financial news.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WallStreetCNEngine(SearchEngine):
|
||||
"""华尔街见闻 (wallstreetcn.com) news search engine."""
|
||||
|
||||
SEARCH_URL = "https://wallstreetcn.com/search"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("wallstreetcn", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
"Referer": "https://wallstreetcn.com/",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
return f"{self.SEARCH_URL}?q={quote_plus(query)}"
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search 华尔街见闻 and parse results."""
|
||||
url = self.build_url(query)
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return self._parse_results(response.text, query, max_results)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"华尔街见闻 search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_results(
|
||||
self, html: str, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Parse 华尔街见闻 search page."""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
|
||||
articles = soup.select(
|
||||
"a[href*='/article/'], .search-result-item, article, .item"
|
||||
)
|
||||
for i, article in enumerate(articles[:max_results]):
|
||||
title_el = article.select_one("h3, h4, .title, a")
|
||||
link = article.get("href", "")
|
||||
snippet_el = article.select_one(
|
||||
".summary, .desc, .abstract, p"
|
||||
)
|
||||
|
||||
title = title_el.get_text(strip=True) if title_el else ""
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if not title and not link:
|
||||
continue
|
||||
|
||||
full_url = (
|
||||
link
|
||||
if link.startswith("http")
|
||||
else f"https://wallstreetcn.com{link}"
|
||||
)
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title or f"华尔街见闻: {query}",
|
||||
url=full_url,
|
||||
snippet=snippet or f"Search result from 华尔街见闻",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={"region": "cn"},
|
||||
)
|
||||
)
|
||||
|
||||
if not results:
|
||||
search_url = self.build_url(query)
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=f"华尔街见闻: {query}",
|
||||
url=search_url,
|
||||
snippet=f"Search 华尔街见闻 for '{query}' (site may require JS)",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="news",
|
||||
extra={"search_url": search_url, "note": "JS-rendered page"},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
@@ -0,0 +1 @@
|
||||
"""Social media search implementations (Twitter/X, etc.)."""
|
||||
@@ -0,0 +1,208 @@
|
||||
"""
|
||||
Twitter/X search module.
|
||||
|
||||
Generates search URLs for X.com (formerly Twitter) search and
|
||||
optionally parses search result pages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TwitterSearchEngine(SearchEngine):
|
||||
"""
|
||||
Twitter/X search engine.
|
||||
|
||||
Provides search URL generation and optional result scraping.
|
||||
Supports basic keyword search, user search, and advanced queries.
|
||||
"""
|
||||
|
||||
SEARCH_URL = "https://twitter.com/search"
|
||||
BASE_URL = "https://twitter.com"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("Twitter", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, search_type: str = "top", **kwargs) -> str:
|
||||
"""
|
||||
Build Twitter/X search URL.
|
||||
|
||||
Args:
|
||||
query: Search keyword.
|
||||
search_type: "top", "latest", "people", "photos", "videos"
|
||||
**kwargs: Additional URL parameters.
|
||||
|
||||
Returns:
|
||||
Search URL string.
|
||||
"""
|
||||
encoded_query = quote_plus(query)
|
||||
url = f"{self.SEARCH_URL}?q={encoded_query}&src=typed_query"
|
||||
|
||||
if search_type == "latest":
|
||||
url += "&f=live"
|
||||
elif search_type == "people":
|
||||
url += "&f=user"
|
||||
elif search_type == "photos":
|
||||
url += "&f=image"
|
||||
elif search_type == "videos":
|
||||
url += "&f=video"
|
||||
|
||||
if kwargs.get("lang"):
|
||||
url += f"&l={kwargs['lang']}"
|
||||
if kwargs.get("until"):
|
||||
url += f"&until={kwargs['until']}"
|
||||
if kwargs.get("since"):
|
||||
url += f"&since={kwargs['since']}"
|
||||
|
||||
return url
|
||||
|
||||
def build_advanced_url(self, **params) -> str:
|
||||
"""
|
||||
Build advanced Twitter search URL with operators.
|
||||
|
||||
Supported operators:
|
||||
from:username - Tweets from user
|
||||
to:username - Tweets to user
|
||||
@username - Mentions
|
||||
#hashtag - Hashtag
|
||||
filter:links - Tweets with links
|
||||
filter:images - Tweets with images
|
||||
min_retweets:N - Minimum retweets
|
||||
min_faves:N - Minimum likes
|
||||
lang:en - Language
|
||||
|
||||
Returns:
|
||||
Advanced search URL.
|
||||
"""
|
||||
parts = []
|
||||
if params.get("from_user"):
|
||||
parts.append(f"from:{params['from_user']}")
|
||||
if params.get("to_user"):
|
||||
parts.append(f"to:{params['to_user']}")
|
||||
if params.get("hashtag"):
|
||||
parts.append(f"#{params['hashtag']}")
|
||||
if params.get("keyword"):
|
||||
parts.append(params["keyword"])
|
||||
if params.get("min_retweets"):
|
||||
parts.append(f"min_retweets:{params['min_retweets']}")
|
||||
if params.get("min_faves"):
|
||||
parts.append(f"min_faves:{params['min_faves']}")
|
||||
if params.get("lang"):
|
||||
parts.append(f"lang:{params['lang']}")
|
||||
if params.get("filter"):
|
||||
parts.append(f"filter:{params['filter']}")
|
||||
|
||||
query = " ".join(parts)
|
||||
return self.build_url(query, search_type=params.get("search_type", "top"))
|
||||
|
||||
async def search(self, query: str, max_results: int = 10, search_type: str = "top", **kwargs) -> list[SearchResult]:
|
||||
"""
|
||||
Search Twitter/X and optionally parse page content.
|
||||
|
||||
Args:
|
||||
query: Search keyword.
|
||||
max_results: Maximum results (used for URL generation).
|
||||
search_type: "top", "latest", "people", "photos", "videos"
|
||||
**kwargs: Additional options.
|
||||
|
||||
Returns:
|
||||
List of SearchResult objects.
|
||||
"""
|
||||
url = self.build_url(query, search_type=search_type, **kwargs)
|
||||
|
||||
# Try to scrape the search page for basic info
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
if response.status_code == 200:
|
||||
soup = BeautifulSoup(response.text, "lxml")
|
||||
results = self._parse_page(soup, max_results)
|
||||
if results:
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.debug(f"Twitter page scraping failed (expected): {e}")
|
||||
|
||||
# Fall back to URL-based result
|
||||
result_type_map = {
|
||||
"top": "Top tweets",
|
||||
"latest": "Latest tweets",
|
||||
"people": "People",
|
||||
"photos": "Photos",
|
||||
"videos": "Videos",
|
||||
}
|
||||
desc = result_type_map.get(search_type, "Tweets")
|
||||
|
||||
return [
|
||||
SearchResult(
|
||||
title=f"Twitter: {query} ({desc})",
|
||||
url=url,
|
||||
snippet=f"Search Twitter/X for '{query}'. Note: Twitter may require login.",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="social",
|
||||
extra={"search_type": search_type, "twitter_url": url},
|
||||
)
|
||||
]
|
||||
|
||||
def _parse_page(self, soup, max_results: int) -> list[SearchResult]:
|
||||
"""Attempt to extract tweet info from search page."""
|
||||
results = []
|
||||
# Twitter uses JS rendering, so HTML parsing may yield limited results
|
||||
tweets = soup.select("article[data-testid='tweet'], div.tweet, .content")
|
||||
for i, tweet in enumerate(tweets[:max_results]):
|
||||
text_el = tweet.select_one(
|
||||
"[data-testid='tweetText'], .tweet-text, p[lang]"
|
||||
)
|
||||
link_el = tweet.select_one("a[href*='/status/'], time")
|
||||
user_el = tweet.select_one(
|
||||
"[data-testid='User-Name'], .username, .fullname"
|
||||
)
|
||||
|
||||
text = text_el.get_text(strip=True) if text_el else ""
|
||||
user = user_el.get_text(strip=True) if user_el else ""
|
||||
|
||||
tweet_url = ""
|
||||
if link_el:
|
||||
href = link_el.get("href", "")
|
||||
tweet_url = f"{self.BASE_URL}{href}" if href.startswith("/") else href
|
||||
|
||||
if text:
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=f"@{user}: {text[:80]}..." if user else text[:80],
|
||||
url=tweet_url or self.build_url(text[:30]),
|
||||
snippet=text,
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="social",
|
||||
extra={"user": user} if user else {},
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
@@ -0,0 +1,183 @@
|
||||
"""
|
||||
Unified search entry point.
|
||||
|
||||
Provides a single interface to search across all configured sources.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
from scripts.core.base import SearchResult
|
||||
from scripts.core.config_loader import (
|
||||
get_search_engines, get_news_sources, get_academic_sources,
|
||||
get_wechat_sources, get_special_engines, get_social_sources,
|
||||
get_all_sources, get_engine,
|
||||
)
|
||||
from scripts.engines.url_engine import UrlSearchEngine
|
||||
from scripts.engines.parser_engines import DuckDuckGoEngine
|
||||
from scripts.news.cls import CLSNewsEngine
|
||||
from scripts.news.wallstreetcn import WallStreetCNEngine
|
||||
from scripts.news.rss import RSSNewsEngine
|
||||
from scripts.academic.arxiv import ArXivEngine
|
||||
from scripts.wechat.sogou_weixin import SogouWeChatEngine
|
||||
from scripts.social.twitter import TwitterSearchEngine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SOURCE_CATEGORIES = {
|
||||
"web": get_search_engines,
|
||||
"news": get_news_sources,
|
||||
"academic": get_academic_sources,
|
||||
"wechat": get_wechat_sources,
|
||||
"social": get_social_sources,
|
||||
"special": get_special_engines,
|
||||
}
|
||||
|
||||
|
||||
class UnifiedSearch:
|
||||
"""Unified search client that aggregates results from multiple sources."""
|
||||
|
||||
def __init__(self, config: Optional[dict[str, Any]] = None):
|
||||
self.config = config or {}
|
||||
self._cache: dict[str, list[SearchResult]] = {}
|
||||
self._engines: dict[str, Any] = {}
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
await self.close()
|
||||
|
||||
async def search(
|
||||
self, query: str, sources: Optional[list[str]] = None,
|
||||
max_results: int = 10, force_refresh: bool = False, **kwargs,
|
||||
) -> list[SearchResult]:
|
||||
"""Search across selected sources and aggregate results."""
|
||||
if not sources or "all" in sources:
|
||||
sources = list(SOURCE_CATEGORIES.keys())
|
||||
|
||||
cache_key = f"{sorted(sources) if isinstance(sources, list) else sources}:{query}"
|
||||
if not force_refresh and cache_key in self._cache:
|
||||
return self._cache[cache_key]
|
||||
|
||||
tasks = []
|
||||
for source_spec in sources:
|
||||
if source_spec in SOURCE_CATEGORIES:
|
||||
for engine_def in SOURCE_CATEGORIES[source_spec]():
|
||||
engine = self._get_engine(engine_def)
|
||||
if engine:
|
||||
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
|
||||
else:
|
||||
engine_def = get_engine(source_spec)
|
||||
if engine_def:
|
||||
engine = self._get_engine(engine_def)
|
||||
if engine:
|
||||
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
|
||||
|
||||
if not tasks:
|
||||
return []
|
||||
|
||||
results_lists = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
all_results: list[SearchResult] = []
|
||||
for results in results_lists:
|
||||
if isinstance(results, Exception):
|
||||
logger.error(f"Search task failed: {results}")
|
||||
continue
|
||||
all_results.extend(results)
|
||||
|
||||
self._cache[cache_key] = all_results
|
||||
return all_results
|
||||
|
||||
def _get_engine(self, engine_def: dict[str, Any]):
|
||||
"""Get or create an engine instance by definition."""
|
||||
name = engine_def.get("name", "")
|
||||
engine_type = engine_def.get("type", "url")
|
||||
|
||||
if name in self._engines:
|
||||
return self._engines[name]
|
||||
|
||||
if name == "DuckDuckGo":
|
||||
engine = DuckDuckGoEngine(self.config)
|
||||
elif "cls" in name.lower():
|
||||
engine = CLSNewsEngine(self.config)
|
||||
elif "wallstreet" in name.lower() or "华尔街" in name:
|
||||
engine = WallStreetCNEngine(self.config)
|
||||
elif name == "ArXiv" or engine_type == "api":
|
||||
engine = ArXivEngine(self.config)
|
||||
elif "微信" in name or "wechat" in name.lower() or "weixin" in name.lower():
|
||||
engine = SogouWeChatEngine(self.config)
|
||||
elif "twitter" in name.lower():
|
||||
engine = TwitterSearchEngine(self.config)
|
||||
elif engine_type == "news" and name not in ("cls", "wallstreetcn"):
|
||||
engine = RSSNewsEngine(self.config)
|
||||
elif engine_type == "social":
|
||||
engine = TwitterSearchEngine(self.config) if "twitter" in name.lower() else UrlSearchEngine(name, engine_def)
|
||||
else:
|
||||
engine = UrlSearchEngine(name, engine_def)
|
||||
|
||||
self._engines[name] = engine
|
||||
return engine
|
||||
|
||||
async def _safe_search(self, engine, query, max_results, **kwargs):
|
||||
try:
|
||||
return await engine.search(query, max_results=max_results, **kwargs)
|
||||
except Exception as e:
|
||||
logger.error(f"Engine {engine.name} failed: {e}")
|
||||
return []
|
||||
|
||||
async def search_engines(self, query, region=None, max_results=10):
|
||||
"""Quick search across web search engines."""
|
||||
engines = get_search_engines()
|
||||
if region:
|
||||
engines = [e for e in engines if e.get("region") == region]
|
||||
return await self.search(query, sources=[e["name"] for e in engines], max_results=max_results)
|
||||
|
||||
async def search_news(self, query, max_results=10, **kwargs):
|
||||
"""Quick search news sources (CN news + RSS feeds)."""
|
||||
return await self.search(query, sources=["news"], max_results=max_results, **kwargs)
|
||||
|
||||
async def search_academic(self, query, max_results=10, **kwargs):
|
||||
"""Quick search academic papers."""
|
||||
return await self.search(query, sources=["academic"], max_results=max_results, **kwargs)
|
||||
|
||||
async def search_wechat(self, query, max_results=10):
|
||||
"""Quick search WeChat articles."""
|
||||
return await self.search(query, sources=["wechat"], max_results=max_results)
|
||||
|
||||
async def search_social(self, query, max_results=10, **kwargs):
|
||||
"""Quick search social media (Twitter/X)."""
|
||||
return await self.search(query, sources=["social"], max_results=max_results, **kwargs)
|
||||
|
||||
async def search_rss(self, query="", max_results=10, **kwargs):
|
||||
"""Quick search RSS news feeds."""
|
||||
engine = RSSNewsEngine(self.config)
|
||||
return await engine.search(query, max_results=max_results, **kwargs)
|
||||
|
||||
async def get_search_urls(self, query, sources=None):
|
||||
"""Get search URLs without executing searches."""
|
||||
if not sources:
|
||||
sources = [e["name"] for e in get_search_engines()]
|
||||
|
||||
urls = {}
|
||||
for name in sources:
|
||||
engine_def = get_engine(name)
|
||||
if engine_def:
|
||||
urls[name] = UrlSearchEngine(name, engine_def).build_url(query)
|
||||
|
||||
return urls
|
||||
|
||||
async def get_twitter_search_url(self, query, search_type="top"):
|
||||
"""Get Twitter/X search URL for browser/web_fetch usage."""
|
||||
engine = TwitterSearchEngine(self.config)
|
||||
return engine.build_url(query, search_type=search_type)
|
||||
|
||||
async def close(self):
|
||||
"""Close all engine resources."""
|
||||
for engine in self._engines.values():
|
||||
try:
|
||||
await engine.close()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1 @@
|
||||
"""WeChat search engines."""
|
||||
@@ -0,0 +1,325 @@
|
||||
"""
|
||||
Sogou WeChat (搜狗微信) search engine.
|
||||
|
||||
Searches WeChat public account articles via weixin.sogou.com.
|
||||
|
||||
Anti-scraping strategy (inspired by Node.js wechat-article-search skill):
|
||||
- Cookie pre-fetch from v.sogou.com to get SNUID
|
||||
- User-Agent rotation with a pool of real browser UAs
|
||||
- Random delays between requests (500-1500ms)
|
||||
- Retry logic with exponential backoff
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Real browser User-Agent pool for rotation
|
||||
USER_AGENTS = [
|
||||
# Chrome
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
# Safari
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1",
|
||||
# Firefox
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0",
|
||||
# Edge
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
|
||||
]
|
||||
|
||||
|
||||
class SogouWeChatEngine(SearchEngine):
|
||||
"""搜狗微信搜索 - searches WeChat public account articles."""
|
||||
|
||||
SEARCH_URL = "https://weixin.sogou.com/weixin"
|
||||
COOKIE_URL = "https://v.sogou.com"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("weixin", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._ua = random.choice(USER_AGENTS)
|
||||
self._cookies: dict[str, str] = {}
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers=self._build_headers(),
|
||||
follow_redirects=False, # Don't auto-follow to detect anti-scraping
|
||||
)
|
||||
|
||||
def _build_headers(self) -> dict[str, str]:
|
||||
return {
|
||||
"User-Agent": self._ua,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"Referer": "https://weixin.sogou.com/",
|
||||
"DNT": "1",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
|
||||
def _random_delay(self):
|
||||
"""Random delay between 500-1500ms to mimic human browsing."""
|
||||
delay = random.uniform(0.5, 1.5)
|
||||
return asyncio.sleep(delay)
|
||||
|
||||
def _rotate_ua(self):
|
||||
"""Rotate to a new random User-Agent."""
|
||||
self._ua = random.choice(USER_AGENTS)
|
||||
|
||||
async def _get_cookie(self) -> dict[str, str]:
|
||||
"""Pre-fetch cookie from v.sogou.com to get SNUID."""
|
||||
try:
|
||||
response = await self._client.get(
|
||||
self.COOKIE_URL,
|
||||
headers={
|
||||
"User-Agent": self._ua,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
},
|
||||
)
|
||||
cookies = {}
|
||||
for cookie in response.cookies.jar:
|
||||
if isinstance(cookie, tuple):
|
||||
# SimpleCookie format: (key, value)
|
||||
cookies[cookie[0]] = cookie[1]
|
||||
else:
|
||||
cookies[cookie.name] = cookie.value
|
||||
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com")
|
||||
return cookies
|
||||
except Exception as e:
|
||||
logger.warning(f"Cookie pre-fetch failed: {e}")
|
||||
return {}
|
||||
|
||||
def build_url(self, query: str, search_type: int = 2, page: int = 1, **kwargs) -> str:
|
||||
"""
|
||||
Build Sogou WeChat search URL.
|
||||
|
||||
Args:
|
||||
query: Search keyword.
|
||||
search_type: 2=articles, 1=accounts.
|
||||
page: Page number (starts at 1).
|
||||
"""
|
||||
return (
|
||||
f"{self.SEARCH_URL}?query={quote_plus(query)}"
|
||||
f"&s_from=input&_sug_=n&type={search_type}"
|
||||
f"&page={page}&ie=utf8"
|
||||
)
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search WeChat articles via Sogou and parse results."""
|
||||
# Step 1: Get cookie first
|
||||
if not self._cookies:
|
||||
self._cookies = await self._get_cookie()
|
||||
await self._random_delay()
|
||||
|
||||
# Step 2: Search with retry
|
||||
max_pages = min(5, (max_results + 9) // 10)
|
||||
all_results = []
|
||||
|
||||
for page in range(1, max_pages + 1):
|
||||
url = self.build_url(query, page=page, **kwargs)
|
||||
results = await self._search_page(url, query, max_results - len(all_results))
|
||||
all_results.extend(results)
|
||||
|
||||
if len(all_results) >= max_results:
|
||||
break
|
||||
|
||||
# Delay between pages
|
||||
await self._random_delay()
|
||||
|
||||
if not all_results:
|
||||
# Fallback: try legacy wx.sogou.com
|
||||
return await self._legacy_search(query, max_results, **kwargs)
|
||||
|
||||
return all_results[:max_results]
|
||||
|
||||
async def _search_page(
|
||||
self, url: str, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Search a single page with retry logic."""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
# Rotate UA on each retry
|
||||
if attempt > 0:
|
||||
self._rotate_ua()
|
||||
self._client.headers.update({"User-Agent": self._ua})
|
||||
if self._cookies:
|
||||
pass # Keep existing cookies
|
||||
|
||||
response = await self._client.get(
|
||||
url,
|
||||
cookies=self._cookies,
|
||||
follow_redirects=False,
|
||||
)
|
||||
|
||||
# Check for anti-scraping page
|
||||
if response.status_code == 302:
|
||||
# Got redirected - likely hit anti-scraping
|
||||
logger.warning(f"Sogou anti-scraping triggered (302), attempt {attempt + 1}")
|
||||
# Try to get new cookie
|
||||
self._cookies = await self._get_cookie()
|
||||
await self._random_delay()
|
||||
continue
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.warning(f"Sogou returned {response.status_code}, attempt {attempt + 1}")
|
||||
await self._random_delay()
|
||||
continue
|
||||
|
||||
html = response.text
|
||||
|
||||
# Check for captcha/block page
|
||||
if "请输入验证码" in html or "访问频率过高" in html:
|
||||
logger.warning(f"Sogou captcha triggered, attempt {attempt + 1}")
|
||||
await asyncio.sleep(random.uniform(2, 4))
|
||||
self._cookies = await self._get_cookie()
|
||||
continue
|
||||
|
||||
return self._parse_results(html, query, max_results)
|
||||
|
||||
except (httpx.HTTPError, httpx.TimeoutException) as e:
|
||||
logger.warning(f"Sogou search attempt {attempt + 1} failed: {e}")
|
||||
await self._random_delay()
|
||||
continue
|
||||
|
||||
return []
|
||||
|
||||
async def _legacy_search(
|
||||
self, query: str, max_results: int, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Fallback: try the legacy wx.sogou.com domain."""
|
||||
legacy_url = f"https://wx.sogou.com/weixin?type=2&query={quote_plus(query)}"
|
||||
try:
|
||||
response = await self._client.get(
|
||||
legacy_url,
|
||||
cookies=self._cookies,
|
||||
follow_redirects=True,
|
||||
)
|
||||
if response.status_code == 200:
|
||||
return self._parse_results(response.text, query, max_results)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Ultimate fallback
|
||||
search_url = self.build_url(query)
|
||||
return [
|
||||
SearchResult(
|
||||
title=f"微信搜索: {query}",
|
||||
url=search_url,
|
||||
snippet=f"Search WeChat for '{query}' via Sogou",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="wechat",
|
||||
extra={"search_url": search_url},
|
||||
)
|
||||
]
|
||||
|
||||
def _parse_results(
|
||||
self, html: str, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Parse Sogou WeChat search results."""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
|
||||
# Main selector: ul.news-list > li (standard Sogou WeChat structure)
|
||||
items = soup.select("ul.news-list > li")
|
||||
|
||||
if not items:
|
||||
# Fallback selectors
|
||||
items = soup.select(
|
||||
".news-list2 li, .wx-rb-item, .results .result, "
|
||||
".item, .news-box .news-list li"
|
||||
)
|
||||
|
||||
for i, item in enumerate(items[:max_results]):
|
||||
title_el = item.select_one("h3 a")
|
||||
if not title_el:
|
||||
title_el = item.select_one(".tit a, .title a, h3")
|
||||
|
||||
snippet_el = item.select_one("p.txt-info, .txt-info, .summary, .des, p.info")
|
||||
source_el = item.select_one(
|
||||
".all-time-y2 a.account, a.account, "
|
||||
".account, .source, .s-p, .from"
|
||||
)
|
||||
|
||||
title = title_el.get_text(strip=True) if title_el else ""
|
||||
link = title_el.get("href", "") if title_el and hasattr(title_el, "get") else ""
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
account = source_el.get_text(strip=True) if source_el else ""
|
||||
|
||||
# Parse time from inline scripts or text
|
||||
time_text = self._parse_time(item)
|
||||
|
||||
if not title:
|
||||
continue
|
||||
|
||||
# Make sure URL is absolute
|
||||
if link and not link.startswith("http"):
|
||||
link = f"https://weixin.sogou.com{link}"
|
||||
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title,
|
||||
url=link,
|
||||
snippet=snippet or f"WeChat article from {account or 'unknown'}",
|
||||
source=self.name,
|
||||
rank=(i + 1),
|
||||
category="wechat",
|
||||
extra={
|
||||
"account": account,
|
||||
"time": time_text,
|
||||
"search_url": self.build_url(query),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _parse_time(self, item) -> str:
|
||||
"""Extract publish time from search result item."""
|
||||
# Try script tag with Unix timestamp (Sogou encodes dates in scripts)
|
||||
for script in item.select("script"):
|
||||
text = script.string or ""
|
||||
# Look for Unix timestamp pattern
|
||||
match = re.search(r'\d{10}', text)
|
||||
if match:
|
||||
import datetime
|
||||
ts = int(match.group())
|
||||
try:
|
||||
dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone(datetime.timedelta(hours=8)))
|
||||
return dt.strftime("%Y-%m-%d %H:%M")
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
# Try inline text with relative time like "2小时前", "3天前"
|
||||
time_text = item.get_text()
|
||||
relative_match = re.search(r'(\d+)\s*(小时|分钟|天|周|月)前', time_text)
|
||||
if relative_match:
|
||||
return f"{relative_match.group(1)}{relative_match.group(2)}前"
|
||||
|
||||
# Try standard date patterns
|
||||
date_match = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', time_text)
|
||||
if date_match:
|
||||
return date_match.group(1)
|
||||
|
||||
return ""
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
Reference in new issue
Block a user