Files
vex1023 30158d8deb feat: stream search, anti-crawl, aggregator, config, models
Implements the improvements from bot-search-mcp comparison (closes #5):

- perf: asyncio.as_completed streaming + early termination (8s timeout)
- feat: ResultAggregator (dedup + quality filter)
- feat: anti-crawl enhancements for sogou_weixin.py
- feat: config.py (pydantic-settings + .env support)
- feat: models.py (data models with dataclasses)
- docs: SKILL.md, README.md updates
2026-05-20 12:23:21 +08:00

366 lines
13 KiB
Python

"""
Sogou WeChat (搜狗微信) search engine.
Searches WeChat public account articles via weixin.sogou.com.
Anti-scraping strategy (inspired by Node.js wechat-article-search skill):
- Cookie pre-fetch from v.sogou.com to get SNUID
- User-Agent rotation with a pool of real browser UAs
- Random delays between requests (500-1500ms)
- Retry logic with exponential backoff
"""
from __future__ import annotations
import asyncio
import logging
import random
import re
from typing import Optional
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
# Real browser User-Agent pool for rotation
USER_AGENTS = [
# Chrome
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
# Safari
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1",
# Firefox
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0",
# Edge
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
]
class SogouWeChatEngine(SearchEngine):
"""搜狗微信搜索 - searches WeChat public account articles."""
SEARCH_URL = "https://weixin.sogou.com/weixin"
COOKIE_URL = "https://v.sogou.com"
def __init__(self, config: Optional[dict] = None):
super().__init__("weixin", config)
timeout = (config or {}).get("timeout", 15)
self._ua = random.choice(USER_AGENTS)
self._cookies: dict[str, str] = {}
self._client = httpx.AsyncClient(
timeout=timeout,
headers=self._build_headers(),
follow_redirects=False, # Don't auto-follow to detect anti-scraping
)
def _build_headers(self) -> dict[str, str]:
return {
"User-Agent": self._ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Accept-Encoding": "gzip, deflate",
"Referer": "https://weixin.sogou.com/",
"DNT": "1",
"Connection": "keep-alive",
}
# ---- Anti-scraping enhancements from bot-search-mcp ---- #
CAPTCHA_KEYWORDS = [
"请输入验证码", "访问频率过高", "请稍后重试", "验证码",
"captcha", "访问过于频繁", "请输入以下验证码",
"403 Forbidden", "您的访问被拒绝",
]
def _random_delay(self, min_s: float = 0.5, max_s: float = 1.5):
"""Random delay to mimic human browsing."""
delay = random.uniform(min_s, max_s)
return asyncio.sleep(delay)
def _rotate_ua(self):
"""Rotate to a new random User-Agent."""
self._ua = random.choice(USER_AGENTS)
async def _get_cookie(self) -> dict[str, str]:
"""Pre-fetch cookie from v.sogou.com to get SNUID/SUID/IPLOC/SUV."""
try:
response = await self._client.get(
self.COOKIE_URL,
headers={
"User-Agent": self._ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9",
},
)
cookies: dict[str, str] = {}
# Extract from httpx Cookies
for key, value in response.cookies.items():
if key.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
cookies[key] = value
# Also parse from Set-Cookie header (more reliable)
set_cookie = response.headers.get("set-cookie", "")
for part in set_cookie.split(";"):
if "=" in part:
k, v = part.split("=", 1)
k = k.strip()
if k.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
cookies[k] = v.split(",")[-1].strip()
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com: {list(cookies.keys())}")
return cookies
except Exception as e:
logger.warning(f"Cookie pre-fetch failed: {e}")
return {}
@staticmethod
def _detect_captcha(html: str) -> bool:
"""检测是否触发了验证码或反爬。"""
html_lower = html.lower()
for kw in SogouWeChatEngine.CAPTCHA_KEYWORDS:
if kw in html_lower or kw.lower() in html_lower:
return True
return False
async def _exponential_backoff(
self, attempt: int, base_delay: float = 2.0
):
"""指数退避等待(带随机 jitter)。"""
wait = (2 ** attempt) + random.uniform(0, 1)
await asyncio.sleep(wait)
def build_url(self, query: str, search_type: int = 2, page: int = 1, **kwargs) -> str:
"""
Build Sogou WeChat search URL.
Args:
query: Search keyword.
search_type: 2=articles, 1=accounts.
page: Page number (starts at 1).
"""
return (
f"{self.SEARCH_URL}?query={quote_plus(query)}"
f"&s_from=input&_sug_=n&type={search_type}"
f"&page={page}&ie=utf8"
)
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search WeChat articles via Sogou and parse results."""
# Step 1: Get cookie first
if not self._cookies:
self._cookies = await self._get_cookie()
await self._random_delay()
# Step 2: Search with retry
max_pages = min(5, (max_results + 9) // 10)
all_results = []
for page in range(1, max_pages + 1):
url = self.build_url(query, page=page, **kwargs)
results = await self._search_page(url, query, max_results - len(all_results))
all_results.extend(results)
if len(all_results) >= max_results:
break
# Delay between pages
await self._random_delay()
if not all_results:
# Fallback: try legacy wx.sogou.com
return await self._legacy_search(query, max_results, **kwargs)
return all_results[:max_results]
async def _search_page(
self, url: str, query: str, max_results: int
) -> list[SearchResult]:
"""Search a single page with enhanced retry + anti-scraping logic."""
for attempt in range(3):
try:
# Rotate UA on each retry
if attempt > 0:
self._rotate_ua()
self._client.headers.update({"User-Agent": self._ua})
response = await self._client.get(
url,
cookies=self._cookies,
follow_redirects=False,
)
# 302 = anti-scraping redirect
if response.status_code == 302:
logger.warning(
f"Sogou anti-scraping triggered (302), "
f"attempt {attempt + 1}"
)
self._cookies = await self._get_cookie()
await self._exponential_backoff(attempt)
continue
if response.status_code != 200:
logger.warning(
f"Sogou returned {response.status_code}, "
f"attempt {attempt + 1}"
)
await self._exponential_backoff(attempt)
continue
html = response.text
# Detect captcha / block page
if self._detect_captcha(html):
logger.warning(
f"Sogou captcha/block detected, "
f"attempt {attempt + 1}"
)
await self._exponential_backoff(attempt)
self._cookies = await self._get_cookie()
continue
return self._parse_results(html, query, max_results)
except (httpx.HTTPError, httpx.TimeoutException) as e:
logger.warning(
f"Sogou search attempt {attempt + 1} failed: {e}"
)
await self._exponential_backoff(attempt)
continue
return []
async def _legacy_search(
self, query: str, max_results: int, **kwargs
) -> list[SearchResult]:
"""Fallback: try the legacy wx.sogou.com domain."""
legacy_url = f"https://wx.sogou.com/weixin?type=2&query={quote_plus(query)}"
try:
response = await self._client.get(
legacy_url,
cookies=self._cookies,
follow_redirects=True,
)
if response.status_code == 200:
return self._parse_results(response.text, query, max_results)
except Exception:
pass
# Ultimate fallback
search_url = self.build_url(query)
return [
SearchResult(
title=f"微信搜索: {query}",
url=search_url,
snippet=f"Search WeChat for '{query}' via Sogou",
source=self.name,
rank=1,
category="wechat",
extra={"search_url": search_url},
)
]
def _parse_results(
self, html: str, query: str, max_results: int
) -> list[SearchResult]:
"""Parse Sogou WeChat search results."""
soup = BeautifulSoup(html, "lxml")
results = []
# Main selector: ul.news-list > li (standard Sogou WeChat structure)
items = soup.select("ul.news-list > li")
if not items:
# Fallback selectors
items = soup.select(
".news-list2 li, .wx-rb-item, .results .result, "
".item, .news-box .news-list li"
)
for i, item in enumerate(items[:max_results]):
title_el = item.select_one("h3 a")
if not title_el:
title_el = item.select_one(".tit a, .title a, h3")
snippet_el = item.select_one("p.txt-info, .txt-info, .summary, .des, p.info")
source_el = item.select_one(
".all-time-y2 a.account, a.account, "
".account, .source, .s-p, .from"
)
title = title_el.get_text(strip=True) if title_el else ""
link = title_el.get("href", "") if title_el and hasattr(title_el, "get") else ""
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
account = source_el.get_text(strip=True) if source_el else ""
# Parse time from inline scripts or text
time_text = self._parse_time(item)
if not title:
continue
# Make sure URL is absolute
if link and not link.startswith("http"):
link = f"https://weixin.sogou.com{link}"
results.append(
SearchResult(
title=title,
url=link,
snippet=snippet or f"WeChat article from {account or 'unknown'}",
source=self.name,
rank=(i + 1),
category="wechat",
extra={
"account": account,
"time": time_text,
"search_url": self.build_url(query),
},
)
)
return results
def _parse_time(self, item) -> str:
"""Extract publish time from search result item."""
# Try script tag with Unix timestamp (Sogou encodes dates in scripts)
for script in item.select("script"):
text = script.string or ""
# Look for Unix timestamp pattern
match = re.search(r'\d{10}', text)
if match:
import datetime
ts = int(match.group())
try:
dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone(datetime.timedelta(hours=8)))
return dt.strftime("%Y-%m-%d %H:%M")
except (OSError, ValueError):
pass
# Try inline text with relative time like "2小时前", "3天前"
time_text = item.get_text()
relative_match = re.search(r'(\d+)\s*(小时|分钟|天|周|月)前', time_text)
if relative_match:
return f"{relative_match.group(1)}{relative_match.group(2)}前"
# Try standard date patterns
date_match = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', time_text)
if date_match:
return date_match.group(1)
return ""
async def close(self):
await self._client.aclose()