Implements the improvements from bot-search-mcp comparison (closes #5): - perf: asyncio.as_completed streaming + early termination (8s timeout) - feat: ResultAggregator (dedup + quality filter) - feat: anti-crawl enhancements for sogou_weixin.py - feat: config.py (pydantic-settings + .env support) - feat: models.py (data models with dataclasses) - docs: SKILL.md, README.md updates
366 lines
13 KiB
Python
366 lines
13 KiB
Python
"""
|
|
Sogou WeChat (搜狗微信) search engine.
|
|
|
|
Searches WeChat public account articles via weixin.sogou.com.
|
|
|
|
Anti-scraping strategy (inspired by Node.js wechat-article-search skill):
|
|
- Cookie pre-fetch from v.sogou.com to get SNUID
|
|
- User-Agent rotation with a pool of real browser UAs
|
|
- Random delays between requests (500-1500ms)
|
|
- Retry logic with exponential backoff
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
import re
|
|
from typing import Optional
|
|
from urllib.parse import quote_plus
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
from scripts.core.base import SearchEngine, SearchResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Real browser User-Agent pool for rotation
|
|
USER_AGENTS = [
|
|
# Chrome
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
# Safari
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
|
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1",
|
|
# Firefox
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0",
|
|
# Edge
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
|
|
]
|
|
|
|
|
|
class SogouWeChatEngine(SearchEngine):
|
|
"""搜狗微信搜索 - searches WeChat public account articles."""
|
|
|
|
SEARCH_URL = "https://weixin.sogou.com/weixin"
|
|
COOKIE_URL = "https://v.sogou.com"
|
|
|
|
def __init__(self, config: Optional[dict] = None):
|
|
super().__init__("weixin", config)
|
|
timeout = (config or {}).get("timeout", 15)
|
|
self._ua = random.choice(USER_AGENTS)
|
|
self._cookies: dict[str, str] = {}
|
|
self._client = httpx.AsyncClient(
|
|
timeout=timeout,
|
|
headers=self._build_headers(),
|
|
follow_redirects=False, # Don't auto-follow to detect anti-scraping
|
|
)
|
|
|
|
def _build_headers(self) -> dict[str, str]:
|
|
return {
|
|
"User-Agent": self._ua,
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
"Accept-Encoding": "gzip, deflate",
|
|
"Referer": "https://weixin.sogou.com/",
|
|
"DNT": "1",
|
|
"Connection": "keep-alive",
|
|
}
|
|
|
|
# ---- Anti-scraping enhancements from bot-search-mcp ---- #
|
|
|
|
CAPTCHA_KEYWORDS = [
|
|
"请输入验证码", "访问频率过高", "请稍后重试", "验证码",
|
|
"captcha", "访问过于频繁", "请输入以下验证码",
|
|
"403 Forbidden", "您的访问被拒绝",
|
|
]
|
|
|
|
def _random_delay(self, min_s: float = 0.5, max_s: float = 1.5):
|
|
"""Random delay to mimic human browsing."""
|
|
delay = random.uniform(min_s, max_s)
|
|
return asyncio.sleep(delay)
|
|
|
|
def _rotate_ua(self):
|
|
"""Rotate to a new random User-Agent."""
|
|
self._ua = random.choice(USER_AGENTS)
|
|
|
|
async def _get_cookie(self) -> dict[str, str]:
|
|
"""Pre-fetch cookie from v.sogou.com to get SNUID/SUID/IPLOC/SUV."""
|
|
try:
|
|
response = await self._client.get(
|
|
self.COOKIE_URL,
|
|
headers={
|
|
"User-Agent": self._ua,
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
|
},
|
|
)
|
|
cookies: dict[str, str] = {}
|
|
|
|
# Extract from httpx Cookies
|
|
for key, value in response.cookies.items():
|
|
if key.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
|
|
cookies[key] = value
|
|
|
|
# Also parse from Set-Cookie header (more reliable)
|
|
set_cookie = response.headers.get("set-cookie", "")
|
|
for part in set_cookie.split(";"):
|
|
if "=" in part:
|
|
k, v = part.split("=", 1)
|
|
k = k.strip()
|
|
if k.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
|
|
cookies[k] = v.split(",")[-1].strip()
|
|
|
|
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com: {list(cookies.keys())}")
|
|
return cookies
|
|
except Exception as e:
|
|
logger.warning(f"Cookie pre-fetch failed: {e}")
|
|
return {}
|
|
|
|
@staticmethod
|
|
def _detect_captcha(html: str) -> bool:
|
|
"""检测是否触发了验证码或反爬。"""
|
|
html_lower = html.lower()
|
|
for kw in SogouWeChatEngine.CAPTCHA_KEYWORDS:
|
|
if kw in html_lower or kw.lower() in html_lower:
|
|
return True
|
|
return False
|
|
|
|
async def _exponential_backoff(
|
|
self, attempt: int, base_delay: float = 2.0
|
|
):
|
|
"""指数退避等待(带随机 jitter)。"""
|
|
wait = (2 ** attempt) + random.uniform(0, 1)
|
|
await asyncio.sleep(wait)
|
|
|
|
def build_url(self, query: str, search_type: int = 2, page: int = 1, **kwargs) -> str:
|
|
"""
|
|
Build Sogou WeChat search URL.
|
|
|
|
Args:
|
|
query: Search keyword.
|
|
search_type: 2=articles, 1=accounts.
|
|
page: Page number (starts at 1).
|
|
"""
|
|
return (
|
|
f"{self.SEARCH_URL}?query={quote_plus(query)}"
|
|
f"&s_from=input&_sug_=n&type={search_type}"
|
|
f"&page={page}&ie=utf8"
|
|
)
|
|
|
|
async def search(
|
|
self, query: str, max_results: int = 10, **kwargs
|
|
) -> list[SearchResult]:
|
|
"""Search WeChat articles via Sogou and parse results."""
|
|
# Step 1: Get cookie first
|
|
if not self._cookies:
|
|
self._cookies = await self._get_cookie()
|
|
await self._random_delay()
|
|
|
|
# Step 2: Search with retry
|
|
max_pages = min(5, (max_results + 9) // 10)
|
|
all_results = []
|
|
|
|
for page in range(1, max_pages + 1):
|
|
url = self.build_url(query, page=page, **kwargs)
|
|
results = await self._search_page(url, query, max_results - len(all_results))
|
|
all_results.extend(results)
|
|
|
|
if len(all_results) >= max_results:
|
|
break
|
|
|
|
# Delay between pages
|
|
await self._random_delay()
|
|
|
|
if not all_results:
|
|
# Fallback: try legacy wx.sogou.com
|
|
return await self._legacy_search(query, max_results, **kwargs)
|
|
|
|
return all_results[:max_results]
|
|
|
|
async def _search_page(
|
|
self, url: str, query: str, max_results: int
|
|
) -> list[SearchResult]:
|
|
"""Search a single page with enhanced retry + anti-scraping logic."""
|
|
for attempt in range(3):
|
|
try:
|
|
# Rotate UA on each retry
|
|
if attempt > 0:
|
|
self._rotate_ua()
|
|
self._client.headers.update({"User-Agent": self._ua})
|
|
|
|
response = await self._client.get(
|
|
url,
|
|
cookies=self._cookies,
|
|
follow_redirects=False,
|
|
)
|
|
|
|
# 302 = anti-scraping redirect
|
|
if response.status_code == 302:
|
|
logger.warning(
|
|
f"Sogou anti-scraping triggered (302), "
|
|
f"attempt {attempt + 1}"
|
|
)
|
|
self._cookies = await self._get_cookie()
|
|
await self._exponential_backoff(attempt)
|
|
continue
|
|
|
|
if response.status_code != 200:
|
|
logger.warning(
|
|
f"Sogou returned {response.status_code}, "
|
|
f"attempt {attempt + 1}"
|
|
)
|
|
await self._exponential_backoff(attempt)
|
|
continue
|
|
|
|
html = response.text
|
|
|
|
# Detect captcha / block page
|
|
if self._detect_captcha(html):
|
|
logger.warning(
|
|
f"Sogou captcha/block detected, "
|
|
f"attempt {attempt + 1}"
|
|
)
|
|
await self._exponential_backoff(attempt)
|
|
self._cookies = await self._get_cookie()
|
|
continue
|
|
|
|
return self._parse_results(html, query, max_results)
|
|
|
|
except (httpx.HTTPError, httpx.TimeoutException) as e:
|
|
logger.warning(
|
|
f"Sogou search attempt {attempt + 1} failed: {e}"
|
|
)
|
|
await self._exponential_backoff(attempt)
|
|
continue
|
|
|
|
return []
|
|
|
|
async def _legacy_search(
|
|
self, query: str, max_results: int, **kwargs
|
|
) -> list[SearchResult]:
|
|
"""Fallback: try the legacy wx.sogou.com domain."""
|
|
legacy_url = f"https://wx.sogou.com/weixin?type=2&query={quote_plus(query)}"
|
|
try:
|
|
response = await self._client.get(
|
|
legacy_url,
|
|
cookies=self._cookies,
|
|
follow_redirects=True,
|
|
)
|
|
if response.status_code == 200:
|
|
return self._parse_results(response.text, query, max_results)
|
|
except Exception:
|
|
pass
|
|
|
|
# Ultimate fallback
|
|
search_url = self.build_url(query)
|
|
return [
|
|
SearchResult(
|
|
title=f"微信搜索: {query}",
|
|
url=search_url,
|
|
snippet=f"Search WeChat for '{query}' via Sogou",
|
|
source=self.name,
|
|
rank=1,
|
|
category="wechat",
|
|
extra={"search_url": search_url},
|
|
)
|
|
]
|
|
|
|
def _parse_results(
|
|
self, html: str, query: str, max_results: int
|
|
) -> list[SearchResult]:
|
|
"""Parse Sogou WeChat search results."""
|
|
soup = BeautifulSoup(html, "lxml")
|
|
results = []
|
|
|
|
# Main selector: ul.news-list > li (standard Sogou WeChat structure)
|
|
items = soup.select("ul.news-list > li")
|
|
|
|
if not items:
|
|
# Fallback selectors
|
|
items = soup.select(
|
|
".news-list2 li, .wx-rb-item, .results .result, "
|
|
".item, .news-box .news-list li"
|
|
)
|
|
|
|
for i, item in enumerate(items[:max_results]):
|
|
title_el = item.select_one("h3 a")
|
|
if not title_el:
|
|
title_el = item.select_one(".tit a, .title a, h3")
|
|
|
|
snippet_el = item.select_one("p.txt-info, .txt-info, .summary, .des, p.info")
|
|
source_el = item.select_one(
|
|
".all-time-y2 a.account, a.account, "
|
|
".account, .source, .s-p, .from"
|
|
)
|
|
|
|
title = title_el.get_text(strip=True) if title_el else ""
|
|
link = title_el.get("href", "") if title_el and hasattr(title_el, "get") else ""
|
|
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
|
account = source_el.get_text(strip=True) if source_el else ""
|
|
|
|
# Parse time from inline scripts or text
|
|
time_text = self._parse_time(item)
|
|
|
|
if not title:
|
|
continue
|
|
|
|
# Make sure URL is absolute
|
|
if link and not link.startswith("http"):
|
|
link = f"https://weixin.sogou.com{link}"
|
|
|
|
results.append(
|
|
SearchResult(
|
|
title=title,
|
|
url=link,
|
|
snippet=snippet or f"WeChat article from {account or 'unknown'}",
|
|
source=self.name,
|
|
rank=(i + 1),
|
|
category="wechat",
|
|
extra={
|
|
"account": account,
|
|
"time": time_text,
|
|
"search_url": self.build_url(query),
|
|
},
|
|
)
|
|
)
|
|
|
|
return results
|
|
|
|
def _parse_time(self, item) -> str:
|
|
"""Extract publish time from search result item."""
|
|
# Try script tag with Unix timestamp (Sogou encodes dates in scripts)
|
|
for script in item.select("script"):
|
|
text = script.string or ""
|
|
# Look for Unix timestamp pattern
|
|
match = re.search(r'\d{10}', text)
|
|
if match:
|
|
import datetime
|
|
ts = int(match.group())
|
|
try:
|
|
dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone(datetime.timedelta(hours=8)))
|
|
return dt.strftime("%Y-%m-%d %H:%M")
|
|
except (OSError, ValueError):
|
|
pass
|
|
|
|
# Try inline text with relative time like "2小时前", "3天前"
|
|
time_text = item.get_text()
|
|
relative_match = re.search(r'(\d+)\s*(小时|分钟|天|周|月)前', time_text)
|
|
if relative_match:
|
|
return f"{relative_match.group(1)}{relative_match.group(2)}前"
|
|
|
|
# Try standard date patterns
|
|
date_match = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', time_text)
|
|
if date_match:
|
|
return date_match.group(1)
|
|
|
|
return ""
|
|
|
|
async def close(self):
|
|
await self._client.aclose()
|