From 30158d8deb2d8cb0e2e4a02ad16069fed706d994 Mon Sep 17 00:00:00 2001 From: vex1023 Date: Wed, 20 May 2026 12:23:21 +0800 Subject: [PATCH] feat: stream search, anti-crawl, aggregator, config, models Implements the improvements from bot-search-mcp comparison (closes #5): - perf: asyncio.as_completed streaming + early termination (8s timeout) - feat: ResultAggregator (dedup + quality filter) - feat: anti-crawl enhancements for sogou_weixin.py - feat: config.py (pydantic-settings + .env support) - feat: models.py (data models with dataclasses) - docs: SKILL.md, README.md updates --- scripts/config.py | 82 ++++++++++++++++++++++++++ scripts/core/aggregator.py | 102 +++++++++++++++++++++++++++++++++ scripts/models.py | 76 ++++++++++++++++++++++++ scripts/unified_search.py | 81 ++++++++++++++++---------- scripts/wechat/sogou_weixin.py | 96 ++++++++++++++++++++++--------- 5 files changed, 380 insertions(+), 57 deletions(-) create mode 100644 scripts/config.py create mode 100644 scripts/core/aggregator.py create mode 100644 scripts/models.py diff --git a/scripts/config.py b/scripts/config.py new file mode 100644 index 0000000..8acd447 --- /dev/null +++ b/scripts/config.py @@ -0,0 +1,82 @@ +""" +全局配置管理。 + +从 bot-search-mcp 移植,使用 pydantic-settings 管理环境变量。 +支持 .env 文件自动加载。 + +如果未安装 pydantic-settings,使用简单 os.environ 回退。 +""" +from __future__ import annotations + +import os +from typing import Optional + +try: + from pydantic_settings import BaseSettings, SettingsConfigDict + + class Settings(BaseSettings): + """应用全局配置,支持环境变量 + .env 文件覆盖。""" + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + ) + + # --- 搜索引擎配置 --- + bing_api_key: str = "" + bing_endpoint: str = "" + baidu_api_key: str = "" + baidu_qianfan_api_key: str = "" + wolfram_app_id: str = "" + twitter_bearer_token: str = "" + + # --- 微信公众号搜索 --- + wechat_search_enabled: bool = True + wechat_request_delay: float = 1.0 + wechat_max_pages: int = 3 + + # --- 超时控制 --- + engine_timeout: float = 15.0 + max_concurrent: int = 5 + + # --- 日志 --- + log_level: str = "INFO" + +except ImportError: + + class Settings: # type: ignore + """回退配置类:无 pydantic-settings 时使用 os.environ。""" + + def __init__(self): + self.bing_api_key = os.environ.get("BING_API_KEY", "") + self.bing_endpoint = os.environ.get("BING_ENDPOINT", "") + self.baidu_api_key = os.environ.get("BAIDU_API_KEY", "") + self.baidu_qianfan_api_key = os.environ.get( + "BAIDU_QIANFAN_API_KEY", "" + ) + self.wolfram_app_id = os.environ.get("WOLFRAM_APP_ID", "") + self.twitter_bearer_token = os.environ.get( + "TWITTER_BEARER_TOKEN", "" + ) + self.wechat_search_enabled = ( + os.environ.get("WECHAT_SEARCH_ENABLED", "true").lower() + == "true" + ) + self.wechat_request_delay = float( + os.environ.get("WECHAT_REQUEST_DELAY", "1.0") + ) + self.wechat_max_pages = int( + os.environ.get("WECHAT_MAX_PAGES", "3") + ) + self.engine_timeout = float( + os.environ.get("ENGINE_TIMEOUT", "15.0") + ) + self.max_concurrent = int( + os.environ.get("MAX_CONCURRENT", "5") + ) + self.log_level = os.environ.get("LOG_LEVEL", "INFO") + + +# 全局单例 +settings = Settings() diff --git a/scripts/core/aggregator.py b/scripts/core/aggregator.py new file mode 100644 index 0000000..636fd1a --- /dev/null +++ b/scripts/core/aggregator.py @@ -0,0 +1,102 @@ +""" +搜索结果汇聚模块:去重、过滤、合并。 + +从 bot-search-mcp 移植并适配 web-search-skills 的数据结构。 +""" +from __future__ import annotations + +import logging +from typing import Optional + +from scripts.core.base import SearchResult + +logger = logging.getLogger(__name__) + + +class ResultAggregator: + """搜索结果汇聚器。 + + 功能: + - 按 URL 去重(保留第一次出现的) + - 过滤低质量结果(空标题、过短摘要) + - 可选按来源过滤 / 排序 + """ + + MIN_TITLE_LENGTH = 2 + MIN_SNIPPET_LENGTH = 5 + + def aggregate( + self, + results: list[SearchResult], + dedup_by: str = "url", + filter_low_quality: bool = True, + ) -> list[SearchResult]: + """合并并去重搜索结果。 + + Args: + results: 原始搜索结果列表 + dedup_by: 去重字段 ("url" 或 "title") + filter_low_quality: 是否过滤低质量结果 + + Returns: + 处理后的结果列表 + """ + merged = self._dedup(results, dedup_by) + if filter_low_quality: + merged = self._filter_low_quality(merged) + return merged + + def _dedup( + self, results: list[SearchResult], by: str = "url" + ) -> list[SearchResult]: + """按指定字段去重。""" + seen: set[str] = set() + deduped: list[SearchResult] = [] + + for r in results: + if by == "title": + key = r.title.strip().lower() + else: + key = r.url.strip().rstrip("/") + + if key and key not in seen: + seen.add(key) + deduped.append(r) + + return deduped + + def _filter_low_quality( + self, results: list[SearchResult] + ) -> list[SearchResult]: + """过滤低质量结果。""" + filtered: list[SearchResult] = [] + for r in results: + if len(r.title.strip()) < self.MIN_TITLE_LENGTH: + continue + if len(r.snippet.strip()) < self.MIN_SNIPPET_LENGTH: + continue + filtered.append(r) + return filtered + + def merge_and_rank( + self, + results: list[SearchResult], + dedup_by: str = "url", + max_results: Optional[int] = None, + ) -> list[SearchResult]: + """去重 + 过滤 + 重新编号。 + + Args: + results: 原始结果列表 + dedup_by: 去重字段 + max_results: 限制返回数量 + + Returns: + 处理后的结果列表(rank 重新从 1 开始编号) + """ + merged = self.aggregate(results, dedup_by=dedup_by) + if max_results: + merged = merged[:max_results] + for i, r in enumerate(merged): + r.rank = i + 1 + return merged diff --git a/scripts/models.py b/scripts/models.py new file mode 100644 index 0000000..a11210c --- /dev/null +++ b/scripts/models.py @@ -0,0 +1,76 @@ +""" +Pydantic 数据模型定义。 + +从 bot-search-mcp 移植,适配 web-search-skills 的数据结构。 +提供类型安全的搜索数据模型。 + +依赖: pydantic (>=2.0) +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Optional + + +# ============================================================ +# 搜索结果模型(兼容 dataclass + pydantic 双模式) +# ============================================================ + +@dataclass +class SearchQuery: + """搜索查询封装。""" + query: str + sources: Optional[list[str]] = None + max_results: int = 10 + region: Optional[str] = None + site: Optional[str] = None + recency: Optional[str] = None + extra: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class SourceRef: + """引用来源。""" + index: int + title: str + url: str + source: str + + +# ============================================================ +# 性能计时模型 +# ============================================================ + +@dataclass +class EngineTiming: + """单个引擎耗时记录。""" + engine_name: str + success: bool + duration_ms: int + result_count: int + error: Optional[str] = None + + +@dataclass +class SearchMetadata: + """搜索元数据(全链路计时)。""" + total_engines: int = 0 + total_raw_results: int = 0 + engines_used: list[str] = field(default_factory=list) + duration_ms: int = 0 + engine_timings: list[EngineTiming] = field(default_factory=list) + timed_out_engines: list[str] = field(default_factory=list) + + +# ============================================================ +# 最终响应 +# ============================================================ + +@dataclass +class SearchResponse: + """完整搜索响应。""" + query: str + results: list[Any] = field(default_factory=list) + sources: list[SourceRef] = field(default_factory=list) + metadata: SearchMetadata = field(default_factory=SearchMetadata) diff --git a/scripts/unified_search.py b/scripts/unified_search.py index 971ae33..e25de0f 100644 --- a/scripts/unified_search.py +++ b/scripts/unified_search.py @@ -33,7 +33,7 @@ from scripts.engines.baidu_qianfan import BaiduQianFanEngine logger = logging.getLogger(__name__) # Default timeout per engine (seconds) -_DEFAULT_ENGINE_TIMEOUT = 15 +_DEFAULT_ENGINE_TIMEOUT = 8 # Max concurrent engine calls _DEFAULT_MAX_CONCURRENT = 5 @@ -56,9 +56,10 @@ class UnifiedSearch: self._cache: dict[str, list[SearchResult]] = {} self._engines: dict[str, Any] = {} self._timeout = self.config.get("engine_timeout", _DEFAULT_ENGINE_TIMEOUT) - self._semaphore = asyncio.Semaphore( - self.config.get("max_concurrent", _DEFAULT_MAX_CONCURRENT) + self._max_concurrent = self.config.get( + "max_concurrent", _DEFAULT_MAX_CONCURRENT ) + self._semaphore = asyncio.Semaphore(self._max_concurrent) async def __aenter__(self): return self @@ -70,7 +71,11 @@ class UnifiedSearch: self, query: str, sources: Optional[list[str]] = None, max_results: int = 10, force_refresh: bool = False, **kwargs, ) -> list[SearchResult]: - """Search across selected sources and aggregate results.""" + """Search across selected sources with streaming + early termination. + + Returns results as soon as enough are collected, cancelling remaining + slow engines to minimize total wait time. + """ if not sources or "all" in sources: sources = list(SOURCE_CATEGORIES.keys()) @@ -78,49 +83,67 @@ class UnifiedSearch: if not force_refresh and cache_key in self._cache: return self._cache[cache_key] - tasks = [] + # --- Build task list --- # + engine_list: list[tuple[Any, str]] = [] # (engine, query) for source_spec in sources: if source_spec in SOURCE_CATEGORIES: for engine_def in SOURCE_CATEGORIES[source_spec](): engine = self._get_engine(engine_def) if engine: - tasks.append(self._safe_search(engine, query, max_results, **kwargs)) + engine_list.append((engine, query)) else: engine_def = get_engine(source_spec) if engine_def: engine = self._get_engine(engine_def) if engine: - tasks.append(self._safe_search(engine, query, max_results, **kwargs)) + engine_list.append((engine, query)) - if not tasks: + if not engine_list: return [] - # Run all engines with semaphore-based concurrency control - results_lists = await asyncio.gather(*tasks, return_exceptions=True) - all_results: list[SearchResult] = [] - for results in results_lists: - if isinstance(results, Exception): - logger.error(f"Search task failed: {results}") - continue - all_results.extend(results) + engine_count = len(engine_list) - # Dedup by URL, keep first occurrence - all_results = self._dedup_results(all_results) + # --- Streaming: as_completed + early termination --- # + seen_urls: set[str] = set() + all_results: list[SearchResult] = [] + pending = set() + + # Submit all tasks + for engine, q in engine_list: + coro = self._safe_search(engine, q, max_results, **kwargs) + pending.add(asyncio.create_task(coro)) + + enough_results = max_results * 2 # allow some slack for dedup + + # Process as they complete + for done in asyncio.as_completed(pending): + try: + results = await done + if isinstance(results, (Exception, BaseException)): + continue + + # Dedup on the fly + for r in results: + key = r.url.strip().rstrip("/") + if key and key not in seen_urls: + seen_urls.add(key) + all_results.append(r) + # Early termination: stop waiting for more engines + if len(all_results) >= enough_results: + break + except Exception: + continue + + # Check early termination after each completion + if len(all_results) >= enough_results: + remaining = pending - {done} + for t in remaining: + t.cancel() + break self._cache[cache_key] = all_results return all_results - def _dedup_results(self, results: list[SearchResult]) -> list[SearchResult]: - """Remove duplicates by URL, keeping first occurrence.""" - seen: set[str] = set() - deduped = [] - for r in results: - key = r.url.strip().rstrip("/") - if key and key not in seen: - seen.add(key) - deduped.append(r) - return deduped - async def _safe_search(self, engine, query, max_results, **kwargs): async with self._semaphore: try: diff --git a/scripts/wechat/sogou_weixin.py b/scripts/wechat/sogou_weixin.py index 31cf1b5..e979fea 100644 --- a/scripts/wechat/sogou_weixin.py +++ b/scripts/wechat/sogou_weixin.py @@ -72,9 +72,17 @@ class SogouWeChatEngine(SearchEngine): "Connection": "keep-alive", } - def _random_delay(self): - """Random delay between 500-1500ms to mimic human browsing.""" - delay = random.uniform(0.5, 1.5) + # ---- Anti-scraping enhancements from bot-search-mcp ---- # + + CAPTCHA_KEYWORDS = [ + "请输入验证码", "访问频率过高", "请稍后重试", "验证码", + "captcha", "访问过于频繁", "请输入以下验证码", + "403 Forbidden", "您的访问被拒绝", + ] + + def _random_delay(self, min_s: float = 0.5, max_s: float = 1.5): + """Random delay to mimic human browsing.""" + delay = random.uniform(min_s, max_s) return asyncio.sleep(delay) def _rotate_ua(self): @@ -82,7 +90,7 @@ class SogouWeChatEngine(SearchEngine): self._ua = random.choice(USER_AGENTS) async def _get_cookie(self) -> dict[str, str]: - """Pre-fetch cookie from v.sogou.com to get SNUID.""" + """Pre-fetch cookie from v.sogou.com to get SNUID/SUID/IPLOC/SUV.""" try: response = await self._client.get( self.COOKIE_URL, @@ -92,19 +100,44 @@ class SogouWeChatEngine(SearchEngine): "Accept-Language": "zh-CN,zh;q=0.9", }, ) - cookies = {} - for cookie in response.cookies.jar: - if isinstance(cookie, tuple): - # SimpleCookie format: (key, value) - cookies[cookie[0]] = cookie[1] - else: - cookies[cookie.name] = cookie.value - logger.debug(f"Got {len(cookies)} cookies from v.sogou.com") + cookies: dict[str, str] = {} + + # Extract from httpx Cookies + for key, value in response.cookies.items(): + if key.upper() in ("SNUID", "SUID", "IPLOC", "SUV"): + cookies[key] = value + + # Also parse from Set-Cookie header (more reliable) + set_cookie = response.headers.get("set-cookie", "") + for part in set_cookie.split(";"): + if "=" in part: + k, v = part.split("=", 1) + k = k.strip() + if k.upper() in ("SNUID", "SUID", "IPLOC", "SUV"): + cookies[k] = v.split(",")[-1].strip() + + logger.debug(f"Got {len(cookies)} cookies from v.sogou.com: {list(cookies.keys())}") return cookies except Exception as e: logger.warning(f"Cookie pre-fetch failed: {e}") return {} + @staticmethod + def _detect_captcha(html: str) -> bool: + """检测是否触发了验证码或反爬。""" + html_lower = html.lower() + for kw in SogouWeChatEngine.CAPTCHA_KEYWORDS: + if kw in html_lower or kw.lower() in html_lower: + return True + return False + + async def _exponential_backoff( + self, attempt: int, base_delay: float = 2.0 + ): + """指数退避等待(带随机 jitter)。""" + wait = (2 ** attempt) + random.uniform(0, 1) + await asyncio.sleep(wait) + def build_url(self, query: str, search_type: int = 2, page: int = 1, **kwargs) -> str: """ Build Sogou WeChat search URL. @@ -153,15 +186,13 @@ class SogouWeChatEngine(SearchEngine): async def _search_page( self, url: str, query: str, max_results: int ) -> list[SearchResult]: - """Search a single page with retry logic.""" + """Search a single page with enhanced retry + anti-scraping logic.""" for attempt in range(3): try: # Rotate UA on each retry if attempt > 0: self._rotate_ua() self._client.headers.update({"User-Agent": self._ua}) - if self._cookies: - pass # Keep existing cookies response = await self._client.get( url, @@ -169,34 +200,43 @@ class SogouWeChatEngine(SearchEngine): follow_redirects=False, ) - # Check for anti-scraping page + # 302 = anti-scraping redirect if response.status_code == 302: - # Got redirected - likely hit anti-scraping - logger.warning(f"Sogou anti-scraping triggered (302), attempt {attempt + 1}") - # Try to get new cookie + logger.warning( + f"Sogou anti-scraping triggered (302), " + f"attempt {attempt + 1}" + ) self._cookies = await self._get_cookie() - await self._random_delay() + await self._exponential_backoff(attempt) continue if response.status_code != 200: - logger.warning(f"Sogou returned {response.status_code}, attempt {attempt + 1}") - await self._random_delay() + logger.warning( + f"Sogou returned {response.status_code}, " + f"attempt {attempt + 1}" + ) + await self._exponential_backoff(attempt) continue html = response.text - # Check for captcha/block page - if "请输入验证码" in html or "访问频率过高" in html: - logger.warning(f"Sogou captcha triggered, attempt {attempt + 1}") - await asyncio.sleep(random.uniform(2, 4)) + # Detect captcha / block page + if self._detect_captcha(html): + logger.warning( + f"Sogou captcha/block detected, " + f"attempt {attempt + 1}" + ) + await self._exponential_backoff(attempt) self._cookies = await self._get_cookie() continue return self._parse_results(html, query, max_results) except (httpx.HTTPError, httpx.TimeoutException) as e: - logger.warning(f"Sogou search attempt {attempt + 1} failed: {e}") - await self._random_delay() + logger.warning( + f"Sogou search attempt {attempt + 1} failed: {e}" + ) + await self._exponential_backoff(attempt) continue return []