feat: stream search, anti-crawl, aggregator, config, models
Implements the improvements from bot-search-mcp comparison (closes #5): - perf: asyncio.as_completed streaming + early termination (8s timeout) - feat: ResultAggregator (dedup + quality filter) - feat: anti-crawl enhancements for sogou_weixin.py - feat: config.py (pydantic-settings + .env support) - feat: models.py (data models with dataclasses) - docs: SKILL.md, README.md updates
This commit is contained in:
1 parent
06e3c6bdb9
commit
30158d8deb
5 files changed
+380
-57
No files matched your search
@@ -72,9 +72,17 @@ class SogouWeChatEngine(SearchEngine):
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
|
||||
def _random_delay(self):
|
||||
"""Random delay between 500-1500ms to mimic human browsing."""
|
||||
delay = random.uniform(0.5, 1.5)
|
||||
# ---- Anti-scraping enhancements from bot-search-mcp ---- #
|
||||
|
||||
CAPTCHA_KEYWORDS = [
|
||||
"请输入验证码", "访问频率过高", "请稍后重试", "验证码",
|
||||
"captcha", "访问过于频繁", "请输入以下验证码",
|
||||
"403 Forbidden", "您的访问被拒绝",
|
||||
]
|
||||
|
||||
def _random_delay(self, min_s: float = 0.5, max_s: float = 1.5):
|
||||
"""Random delay to mimic human browsing."""
|
||||
delay = random.uniform(min_s, max_s)
|
||||
return asyncio.sleep(delay)
|
||||
|
||||
def _rotate_ua(self):
|
||||
@@ -82,7 +90,7 @@ class SogouWeChatEngine(SearchEngine):
|
||||
self._ua = random.choice(USER_AGENTS)
|
||||
|
||||
async def _get_cookie(self) -> dict[str, str]:
|
||||
"""Pre-fetch cookie from v.sogou.com to get SNUID."""
|
||||
"""Pre-fetch cookie from v.sogou.com to get SNUID/SUID/IPLOC/SUV."""
|
||||
try:
|
||||
response = await self._client.get(
|
||||
self.COOKIE_URL,
|
||||
@@ -92,19 +100,44 @@ class SogouWeChatEngine(SearchEngine):
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
},
|
||||
)
|
||||
cookies = {}
|
||||
for cookie in response.cookies.jar:
|
||||
if isinstance(cookie, tuple):
|
||||
# SimpleCookie format: (key, value)
|
||||
cookies[cookie[0]] = cookie[1]
|
||||
else:
|
||||
cookies[cookie.name] = cookie.value
|
||||
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com")
|
||||
cookies: dict[str, str] = {}
|
||||
|
||||
# Extract from httpx Cookies
|
||||
for key, value in response.cookies.items():
|
||||
if key.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
|
||||
cookies[key] = value
|
||||
|
||||
# Also parse from Set-Cookie header (more reliable)
|
||||
set_cookie = response.headers.get("set-cookie", "")
|
||||
for part in set_cookie.split(";"):
|
||||
if "=" in part:
|
||||
k, v = part.split("=", 1)
|
||||
k = k.strip()
|
||||
if k.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
|
||||
cookies[k] = v.split(",")[-1].strip()
|
||||
|
||||
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com: {list(cookies.keys())}")
|
||||
return cookies
|
||||
except Exception as e:
|
||||
logger.warning(f"Cookie pre-fetch failed: {e}")
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def _detect_captcha(html: str) -> bool:
|
||||
"""检测是否触发了验证码或反爬。"""
|
||||
html_lower = html.lower()
|
||||
for kw in SogouWeChatEngine.CAPTCHA_KEYWORDS:
|
||||
if kw in html_lower or kw.lower() in html_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _exponential_backoff(
|
||||
self, attempt: int, base_delay: float = 2.0
|
||||
):
|
||||
"""指数退避等待(带随机 jitter)。"""
|
||||
wait = (2 ** attempt) + random.uniform(0, 1)
|
||||
await asyncio.sleep(wait)
|
||||
|
||||
def build_url(self, query: str, search_type: int = 2, page: int = 1, **kwargs) -> str:
|
||||
"""
|
||||
Build Sogou WeChat search URL.
|
||||
@@ -153,15 +186,13 @@ class SogouWeChatEngine(SearchEngine):
|
||||
async def _search_page(
|
||||
self, url: str, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Search a single page with retry logic."""
|
||||
"""Search a single page with enhanced retry + anti-scraping logic."""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
# Rotate UA on each retry
|
||||
if attempt > 0:
|
||||
self._rotate_ua()
|
||||
self._client.headers.update({"User-Agent": self._ua})
|
||||
if self._cookies:
|
||||
pass # Keep existing cookies
|
||||
|
||||
response = await self._client.get(
|
||||
url,
|
||||
@@ -169,34 +200,43 @@ class SogouWeChatEngine(SearchEngine):
|
||||
follow_redirects=False,
|
||||
)
|
||||
|
||||
# Check for anti-scraping page
|
||||
# 302 = anti-scraping redirect
|
||||
if response.status_code == 302:
|
||||
# Got redirected - likely hit anti-scraping
|
||||
logger.warning(f"Sogou anti-scraping triggered (302), attempt {attempt + 1}")
|
||||
# Try to get new cookie
|
||||
logger.warning(
|
||||
f"Sogou anti-scraping triggered (302), "
|
||||
f"attempt {attempt + 1}"
|
||||
)
|
||||
self._cookies = await self._get_cookie()
|
||||
await self._random_delay()
|
||||
await self._exponential_backoff(attempt)
|
||||
continue
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.warning(f"Sogou returned {response.status_code}, attempt {attempt + 1}")
|
||||
await self._random_delay()
|
||||
logger.warning(
|
||||
f"Sogou returned {response.status_code}, "
|
||||
f"attempt {attempt + 1}"
|
||||
)
|
||||
await self._exponential_backoff(attempt)
|
||||
continue
|
||||
|
||||
html = response.text
|
||||
|
||||
# Check for captcha/block page
|
||||
if "请输入验证码" in html or "访问频率过高" in html:
|
||||
logger.warning(f"Sogou captcha triggered, attempt {attempt + 1}")
|
||||
await asyncio.sleep(random.uniform(2, 4))
|
||||
# Detect captcha / block page
|
||||
if self._detect_captcha(html):
|
||||
logger.warning(
|
||||
f"Sogou captcha/block detected, "
|
||||
f"attempt {attempt + 1}"
|
||||
)
|
||||
await self._exponential_backoff(attempt)
|
||||
self._cookies = await self._get_cookie()
|
||||
continue
|
||||
|
||||
return self._parse_results(html, query, max_results)
|
||||
|
||||
except (httpx.HTTPError, httpx.TimeoutException) as e:
|
||||
logger.warning(f"Sogou search attempt {attempt + 1} failed: {e}")
|
||||
await self._random_delay()
|
||||
logger.warning(
|
||||
f"Sogou search attempt {attempt + 1} failed: {e}"
|
||||
)
|
||||
await self._exponential_backoff(attempt)
|
||||
continue
|
||||
|
||||
return []
|
||||
|
||||
Reference in new issue
Block a user