feat: stream search, anti-crawl, aggregator, config, models

Implements the improvements from bot-search-mcp comparison (closes #5):

- perf: asyncio.as_completed streaming + early termination (8s timeout)
- feat: ResultAggregator (dedup + quality filter)
- feat: anti-crawl enhancements for sogou_weixin.py
- feat: config.py (pydantic-settings + .env support)
- feat: models.py (data models with dataclasses)
- docs: SKILL.md, README.md updates
This commit is contained in:
vex1023 committed 2026-05-20 12:23:21 +08:00
1 parent 06e3c6bdb9
commit 30158d8deb
5 files changed
+380 -57

No files matched your search

+82
View File
@@ -0,0 +1,82 @@
"""
全局配置管理。
从 bot-search-mcp 移植,使用 pydantic-settings 管理环境变量。
支持 .env 文件自动加载。
如果未安装 pydantic-settings,使用简单 os.environ 回退。
"""
from __future__ import annotations
import os
from typing import Optional
try:
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
"""应用全局配置,支持环境变量 + .env 文件覆盖。"""
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
extra="ignore",
)
# --- 搜索引擎配置 ---
bing_api_key: str = ""
bing_endpoint: str = ""
baidu_api_key: str = ""
baidu_qianfan_api_key: str = ""
wolfram_app_id: str = ""
twitter_bearer_token: str = ""
# --- 微信公众号搜索 ---
wechat_search_enabled: bool = True
wechat_request_delay: float = 1.0
wechat_max_pages: int = 3
# --- 超时控制 ---
engine_timeout: float = 15.0
max_concurrent: int = 5
# --- 日志 ---
log_level: str = "INFO"
except ImportError:
class Settings: # type: ignore
"""回退配置类:无 pydantic-settings 时使用 os.environ。"""
def __init__(self):
self.bing_api_key = os.environ.get("BING_API_KEY", "")
self.bing_endpoint = os.environ.get("BING_ENDPOINT", "")
self.baidu_api_key = os.environ.get("BAIDU_API_KEY", "")
self.baidu_qianfan_api_key = os.environ.get(
"BAIDU_QIANFAN_API_KEY", ""
)
self.wolfram_app_id = os.environ.get("WOLFRAM_APP_ID", "")
self.twitter_bearer_token = os.environ.get(
"TWITTER_BEARER_TOKEN", ""
)
self.wechat_search_enabled = (
os.environ.get("WECHAT_SEARCH_ENABLED", "true").lower()
== "true"
)
self.wechat_request_delay = float(
os.environ.get("WECHAT_REQUEST_DELAY", "1.0")
)
self.wechat_max_pages = int(
os.environ.get("WECHAT_MAX_PAGES", "3")
)
self.engine_timeout = float(
os.environ.get("ENGINE_TIMEOUT", "15.0")
)
self.max_concurrent = int(
os.environ.get("MAX_CONCURRENT", "5")
)
self.log_level = os.environ.get("LOG_LEVEL", "INFO")
# 全局单例
settings = Settings()
+102
View File
@@ -0,0 +1,102 @@
"""
搜索结果汇聚模块:去重、过滤、合并。
从 bot-search-mcp 移植并适配 web-search-skills 的数据结构。
"""
from __future__ import annotations
import logging
from typing import Optional
from scripts.core.base import SearchResult
logger = logging.getLogger(__name__)
class ResultAggregator:
"""搜索结果汇聚器。
功能:
- 按 URL 去重(保留第一次出现的)
- 过滤低质量结果(空标题、过短摘要)
- 可选按来源过滤 / 排序
"""
MIN_TITLE_LENGTH = 2
MIN_SNIPPET_LENGTH = 5
def aggregate(
self,
results: list[SearchResult],
dedup_by: str = "url",
filter_low_quality: bool = True,
) -> list[SearchResult]:
"""合并并去重搜索结果。
Args:
results: 原始搜索结果列表
dedup_by: 去重字段 ("url" 或 "title")
filter_low_quality: 是否过滤低质量结果
Returns:
处理后的结果列表
"""
merged = self._dedup(results, dedup_by)
if filter_low_quality:
merged = self._filter_low_quality(merged)
return merged
def _dedup(
self, results: list[SearchResult], by: str = "url"
) -> list[SearchResult]:
"""按指定字段去重。"""
seen: set[str] = set()
deduped: list[SearchResult] = []
for r in results:
if by == "title":
key = r.title.strip().lower()
else:
key = r.url.strip().rstrip("/")
if key and key not in seen:
seen.add(key)
deduped.append(r)
return deduped
def _filter_low_quality(
self, results: list[SearchResult]
) -> list[SearchResult]:
"""过滤低质量结果。"""
filtered: list[SearchResult] = []
for r in results:
if len(r.title.strip()) < self.MIN_TITLE_LENGTH:
continue
if len(r.snippet.strip()) < self.MIN_SNIPPET_LENGTH:
continue
filtered.append(r)
return filtered
def merge_and_rank(
self,
results: list[SearchResult],
dedup_by: str = "url",
max_results: Optional[int] = None,
) -> list[SearchResult]:
"""去重 + 过滤 + 重新编号。
Args:
results: 原始结果列表
dedup_by: 去重字段
max_results: 限制返回数量
Returns:
处理后的结果列表(rank 重新从 1 开始编号)
"""
merged = self.aggregate(results, dedup_by=dedup_by)
if max_results:
merged = merged[:max_results]
for i, r in enumerate(merged):
r.rank = i + 1
return merged
+76
View File
@@ -0,0 +1,76 @@
"""
Pydantic 数据模型定义。
从 bot-search-mcp 移植,适配 web-search-skills 的数据结构。
提供类型安全的搜索数据模型。
依赖: pydantic (>=2.0)
"""
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Optional
# ============================================================
# 搜索结果模型(兼容 dataclass + pydantic 双模式)
# ============================================================
@dataclass
class SearchQuery:
"""搜索查询封装。"""
query: str
sources: Optional[list[str]] = None
max_results: int = 10
region: Optional[str] = None
site: Optional[str] = None
recency: Optional[str] = None
extra: dict[str, Any] = field(default_factory=dict)
@dataclass
class SourceRef:
"""引用来源。"""
index: int
title: str
url: str
source: str
# ============================================================
# 性能计时模型
# ============================================================
@dataclass
class EngineTiming:
"""单个引擎耗时记录。"""
engine_name: str
success: bool
duration_ms: int
result_count: int
error: Optional[str] = None
@dataclass
class SearchMetadata:
"""搜索元数据(全链路计时)。"""
total_engines: int = 0
total_raw_results: int = 0
engines_used: list[str] = field(default_factory=list)
duration_ms: int = 0
engine_timings: list[EngineTiming] = field(default_factory=list)
timed_out_engines: list[str] = field(default_factory=list)
# ============================================================
# 最终响应
# ============================================================
@dataclass
class SearchResponse:
"""完整搜索响应。"""
query: str
results: list[Any] = field(default_factory=list)
sources: list[SourceRef] = field(default_factory=list)
metadata: SearchMetadata = field(default_factory=SearchMetadata)
+52 -29
View File
@@ -33,7 +33,7 @@ from scripts.engines.baidu_qianfan import BaiduQianFanEngine
logger = logging.getLogger(__name__)
# Default timeout per engine (seconds)
_DEFAULT_ENGINE_TIMEOUT = 15
_DEFAULT_ENGINE_TIMEOUT = 8
# Max concurrent engine calls
_DEFAULT_MAX_CONCURRENT = 5
@@ -56,9 +56,10 @@ class UnifiedSearch:
self._cache: dict[str, list[SearchResult]] = {}
self._engines: dict[str, Any] = {}
self._timeout = self.config.get("engine_timeout", _DEFAULT_ENGINE_TIMEOUT)
self._semaphore = asyncio.Semaphore(
self.config.get("max_concurrent", _DEFAULT_MAX_CONCURRENT)
self._max_concurrent = self.config.get(
"max_concurrent", _DEFAULT_MAX_CONCURRENT
)
self._semaphore = asyncio.Semaphore(self._max_concurrent)
async def __aenter__(self):
return self
@@ -70,7 +71,11 @@ class UnifiedSearch:
self, query: str, sources: Optional[list[str]] = None,
max_results: int = 10, force_refresh: bool = False, **kwargs,
) -> list[SearchResult]:
"""Search across selected sources and aggregate results."""
"""Search across selected sources with streaming + early termination.
Returns results as soon as enough are collected, cancelling remaining
slow engines to minimize total wait time.
"""
if not sources or "all" in sources:
sources = list(SOURCE_CATEGORIES.keys())
@@ -78,49 +83,67 @@ class UnifiedSearch:
if not force_refresh and cache_key in self._cache:
return self._cache[cache_key]
tasks = []
# --- Build task list --- #
engine_list: list[tuple[Any, str]] = [] # (engine, query)
for source_spec in sources:
if source_spec in SOURCE_CATEGORIES:
for engine_def in SOURCE_CATEGORIES[source_spec]():
engine = self._get_engine(engine_def)
if engine:
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
engine_list.append((engine, query))
else:
engine_def = get_engine(source_spec)
if engine_def:
engine = self._get_engine(engine_def)
if engine:
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
engine_list.append((engine, query))
if not tasks:
if not engine_list:
return []
# Run all engines with semaphore-based concurrency control
results_lists = await asyncio.gather(*tasks, return_exceptions=True)
all_results: list[SearchResult] = []
for results in results_lists:
if isinstance(results, Exception):
logger.error(f"Search task failed: {results}")
continue
all_results.extend(results)
engine_count = len(engine_list)
# Dedup by URL, keep first occurrence
all_results = self._dedup_results(all_results)
# --- Streaming: as_completed + early termination --- #
seen_urls: set[str] = set()
all_results: list[SearchResult] = []
pending = set()
# Submit all tasks
for engine, q in engine_list:
coro = self._safe_search(engine, q, max_results, **kwargs)
pending.add(asyncio.create_task(coro))
enough_results = max_results * 2 # allow some slack for dedup
# Process as they complete
for done in asyncio.as_completed(pending):
try:
results = await done
if isinstance(results, (Exception, BaseException)):
continue
# Dedup on the fly
for r in results:
key = r.url.strip().rstrip("/")
if key and key not in seen_urls:
seen_urls.add(key)
all_results.append(r)
# Early termination: stop waiting for more engines
if len(all_results) >= enough_results:
break
except Exception:
continue
# Check early termination after each completion
if len(all_results) >= enough_results:
remaining = pending - {done}
for t in remaining:
t.cancel()
break
self._cache[cache_key] = all_results
return all_results
def _dedup_results(self, results: list[SearchResult]) -> list[SearchResult]:
"""Remove duplicates by URL, keeping first occurrence."""
seen: set[str] = set()
deduped = []
for r in results:
key = r.url.strip().rstrip("/")
if key and key not in seen:
seen.add(key)
deduped.append(r)
return deduped
async def _safe_search(self, engine, query, max_results, **kwargs):
async with self._semaphore:
try:
+68 -28
View File
@@ -72,9 +72,17 @@ class SogouWeChatEngine(SearchEngine):
"Connection": "keep-alive",
}
def _random_delay(self):
"""Random delay between 500-1500ms to mimic human browsing."""
delay = random.uniform(0.5, 1.5)
# ---- Anti-scraping enhancements from bot-search-mcp ---- #
CAPTCHA_KEYWORDS = [
"请输入验证码", "访问频率过高", "请稍后重试", "验证码",
"captcha", "访问过于频繁", "请输入以下验证码",
"403 Forbidden", "您的访问被拒绝",
]
def _random_delay(self, min_s: float = 0.5, max_s: float = 1.5):
"""Random delay to mimic human browsing."""
delay = random.uniform(min_s, max_s)
return asyncio.sleep(delay)
def _rotate_ua(self):
@@ -82,7 +90,7 @@ class SogouWeChatEngine(SearchEngine):
self._ua = random.choice(USER_AGENTS)
async def _get_cookie(self) -> dict[str, str]:
"""Pre-fetch cookie from v.sogou.com to get SNUID."""
"""Pre-fetch cookie from v.sogou.com to get SNUID/SUID/IPLOC/SUV."""
try:
response = await self._client.get(
self.COOKIE_URL,
@@ -92,19 +100,44 @@ class SogouWeChatEngine(SearchEngine):
"Accept-Language": "zh-CN,zh;q=0.9",
},
)
cookies = {}
for cookie in response.cookies.jar:
if isinstance(cookie, tuple):
# SimpleCookie format: (key, value)
cookies[cookie[0]] = cookie[1]
else:
cookies[cookie.name] = cookie.value
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com")
cookies: dict[str, str] = {}
# Extract from httpx Cookies
for key, value in response.cookies.items():
if key.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
cookies[key] = value
# Also parse from Set-Cookie header (more reliable)
set_cookie = response.headers.get("set-cookie", "")
for part in set_cookie.split(";"):
if "=" in part:
k, v = part.split("=", 1)
k = k.strip()
if k.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
cookies[k] = v.split(",")[-1].strip()
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com: {list(cookies.keys())}")
return cookies
except Exception as e:
logger.warning(f"Cookie pre-fetch failed: {e}")
return {}
@staticmethod
def _detect_captcha(html: str) -> bool:
"""检测是否触发了验证码或反爬。"""
html_lower = html.lower()
for kw in SogouWeChatEngine.CAPTCHA_KEYWORDS:
if kw in html_lower or kw.lower() in html_lower:
return True
return False
async def _exponential_backoff(
self, attempt: int, base_delay: float = 2.0
):
"""指数退避等待(带随机 jitter)。"""
wait = (2 ** attempt) + random.uniform(0, 1)
await asyncio.sleep(wait)
def build_url(self, query: str, search_type: int = 2, page: int = 1, **kwargs) -> str:
"""
Build Sogou WeChat search URL.
@@ -153,15 +186,13 @@ class SogouWeChatEngine(SearchEngine):
async def _search_page(
self, url: str, query: str, max_results: int
) -> list[SearchResult]:
"""Search a single page with retry logic."""
"""Search a single page with enhanced retry + anti-scraping logic."""
for attempt in range(3):
try:
# Rotate UA on each retry
if attempt > 0:
self._rotate_ua()
self._client.headers.update({"User-Agent": self._ua})
if self._cookies:
pass # Keep existing cookies
response = await self._client.get(
url,
@@ -169,34 +200,43 @@ class SogouWeChatEngine(SearchEngine):
follow_redirects=False,
)
# Check for anti-scraping page
# 302 = anti-scraping redirect
if response.status_code == 302:
# Got redirected - likely hit anti-scraping
logger.warning(f"Sogou anti-scraping triggered (302), attempt {attempt + 1}")
# Try to get new cookie
logger.warning(
f"Sogou anti-scraping triggered (302), "
f"attempt {attempt + 1}"
)
self._cookies = await self._get_cookie()
await self._random_delay()
await self._exponential_backoff(attempt)
continue
if response.status_code != 200:
logger.warning(f"Sogou returned {response.status_code}, attempt {attempt + 1}")
await self._random_delay()
logger.warning(
f"Sogou returned {response.status_code}, "
f"attempt {attempt + 1}"
)
await self._exponential_backoff(attempt)
continue
html = response.text
# Check for captcha/block page
if "请输入验证码" in html or "访问频率过高" in html:
logger.warning(f"Sogou captcha triggered, attempt {attempt + 1}")
await asyncio.sleep(random.uniform(2, 4))
# Detect captcha / block page
if self._detect_captcha(html):
logger.warning(
f"Sogou captcha/block detected, "
f"attempt {attempt + 1}"
)
await self._exponential_backoff(attempt)
self._cookies = await self._get_cookie()
continue
return self._parse_results(html, query, max_results)
except (httpx.HTTPError, httpx.TimeoutException) as e:
logger.warning(f"Sogou search attempt {attempt + 1} failed: {e}")
await self._random_delay()
logger.warning(
f"Sogou search attempt {attempt + 1} failed: {e}"
)
await self._exponential_backoff(attempt)
continue
return []