feat: stream search, anti-crawl, aggregator, config, models

Implements the improvements from bot-search-mcp comparison (closes #5):

- perf: asyncio.as_completed streaming + early termination (8s timeout)
- feat: ResultAggregator (dedup + quality filter)
- feat: anti-crawl enhancements for sogou_weixin.py
- feat: config.py (pydantic-settings + .env support)
- feat: models.py (data models with dataclasses)
- docs: SKILL.md, README.md updates
This commit is contained in:
vex1023 committed 2026-05-20 12:23:21 +08:00
1 parent 06e3c6bdb9
commit 30158d8deb
5 files changed
+380 -57

No files matched your search

+68 -28
View File
@@ -72,9 +72,17 @@ class SogouWeChatEngine(SearchEngine):
"Connection": "keep-alive",
}
def _random_delay(self):
"""Random delay between 500-1500ms to mimic human browsing."""
delay = random.uniform(0.5, 1.5)
# ---- Anti-scraping enhancements from bot-search-mcp ---- #
CAPTCHA_KEYWORDS = [
"请输入验证码", "访问频率过高", "请稍后重试", "验证码",
"captcha", "访问过于频繁", "请输入以下验证码",
"403 Forbidden", "您的访问被拒绝",
]
def _random_delay(self, min_s: float = 0.5, max_s: float = 1.5):
"""Random delay to mimic human browsing."""
delay = random.uniform(min_s, max_s)
return asyncio.sleep(delay)
def _rotate_ua(self):
@@ -82,7 +90,7 @@ class SogouWeChatEngine(SearchEngine):
self._ua = random.choice(USER_AGENTS)
async def _get_cookie(self) -> dict[str, str]:
"""Pre-fetch cookie from v.sogou.com to get SNUID."""
"""Pre-fetch cookie from v.sogou.com to get SNUID/SUID/IPLOC/SUV."""
try:
response = await self._client.get(
self.COOKIE_URL,
@@ -92,19 +100,44 @@ class SogouWeChatEngine(SearchEngine):
"Accept-Language": "zh-CN,zh;q=0.9",
},
)
cookies = {}
for cookie in response.cookies.jar:
if isinstance(cookie, tuple):
# SimpleCookie format: (key, value)
cookies[cookie[0]] = cookie[1]
else:
cookies[cookie.name] = cookie.value
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com")
cookies: dict[str, str] = {}
# Extract from httpx Cookies
for key, value in response.cookies.items():
if key.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
cookies[key] = value
# Also parse from Set-Cookie header (more reliable)
set_cookie = response.headers.get("set-cookie", "")
for part in set_cookie.split(";"):
if "=" in part:
k, v = part.split("=", 1)
k = k.strip()
if k.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
cookies[k] = v.split(",")[-1].strip()
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com: {list(cookies.keys())}")
return cookies
except Exception as e:
logger.warning(f"Cookie pre-fetch failed: {e}")
return {}
@staticmethod
def _detect_captcha(html: str) -> bool:
"""检测是否触发了验证码或反爬。"""
html_lower = html.lower()
for kw in SogouWeChatEngine.CAPTCHA_KEYWORDS:
if kw in html_lower or kw.lower() in html_lower:
return True
return False
async def _exponential_backoff(
self, attempt: int, base_delay: float = 2.0
):
"""指数退避等待(带随机 jitter)。"""
wait = (2 ** attempt) + random.uniform(0, 1)
await asyncio.sleep(wait)
def build_url(self, query: str, search_type: int = 2, page: int = 1, **kwargs) -> str:
"""
Build Sogou WeChat search URL.
@@ -153,15 +186,13 @@ class SogouWeChatEngine(SearchEngine):
async def _search_page(
self, url: str, query: str, max_results: int
) -> list[SearchResult]:
"""Search a single page with retry logic."""
"""Search a single page with enhanced retry + anti-scraping logic."""
for attempt in range(3):
try:
# Rotate UA on each retry
if attempt > 0:
self._rotate_ua()
self._client.headers.update({"User-Agent": self._ua})
if self._cookies:
pass # Keep existing cookies
response = await self._client.get(
url,
@@ -169,34 +200,43 @@ class SogouWeChatEngine(SearchEngine):
follow_redirects=False,
)
# Check for anti-scraping page
# 302 = anti-scraping redirect
if response.status_code == 302:
# Got redirected - likely hit anti-scraping
logger.warning(f"Sogou anti-scraping triggered (302), attempt {attempt + 1}")
# Try to get new cookie
logger.warning(
f"Sogou anti-scraping triggered (302), "
f"attempt {attempt + 1}"
)
self._cookies = await self._get_cookie()
await self._random_delay()
await self._exponential_backoff(attempt)
continue
if response.status_code != 200:
logger.warning(f"Sogou returned {response.status_code}, attempt {attempt + 1}")
await self._random_delay()
logger.warning(
f"Sogou returned {response.status_code}, "
f"attempt {attempt + 1}"
)
await self._exponential_backoff(attempt)
continue
html = response.text
# Check for captcha/block page
if "请输入验证码" in html or "访问频率过高" in html:
logger.warning(f"Sogou captcha triggered, attempt {attempt + 1}")
await asyncio.sleep(random.uniform(2, 4))
# Detect captcha / block page
if self._detect_captcha(html):
logger.warning(
f"Sogou captcha/block detected, "
f"attempt {attempt + 1}"
)
await self._exponential_backoff(attempt)
self._cookies = await self._get_cookie()
continue
return self._parse_results(html, query, max_results)
except (httpx.HTTPError, httpx.TimeoutException) as e:
logger.warning(f"Sogou search attempt {attempt + 1} failed: {e}")
await self._random_delay()
logger.warning(
f"Sogou search attempt {attempt + 1} failed: {e}"
)
await self._exponential_backoff(attempt)
continue
return []