feat: stream search, anti-crawl, aggregator, config, models
Implements the improvements from bot-search-mcp comparison (closes #5): - perf: asyncio.as_completed streaming + early termination (8s timeout) - feat: ResultAggregator (dedup + quality filter) - feat: anti-crawl enhancements for sogou_weixin.py - feat: config.py (pydantic-settings + .env support) - feat: models.py (data models with dataclasses) - docs: SKILL.md, README.md updates
This commit is contained in:
1 parent
06e3c6bdb9
commit
30158d8deb
5 files changed
+380
-57
No files matched your search
@@ -0,0 +1,82 @@
|
||||
"""
|
||||
全局配置管理。
|
||||
|
||||
从 bot-search-mcp 移植,使用 pydantic-settings 管理环境变量。
|
||||
支持 .env 文件自动加载。
|
||||
|
||||
如果未安装 pydantic-settings,使用简单 os.environ 回退。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
try:
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""应用全局配置,支持环境变量 + .env 文件覆盖。"""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
# --- 搜索引擎配置 ---
|
||||
bing_api_key: str = ""
|
||||
bing_endpoint: str = ""
|
||||
baidu_api_key: str = ""
|
||||
baidu_qianfan_api_key: str = ""
|
||||
wolfram_app_id: str = ""
|
||||
twitter_bearer_token: str = ""
|
||||
|
||||
# --- 微信公众号搜索 ---
|
||||
wechat_search_enabled: bool = True
|
||||
wechat_request_delay: float = 1.0
|
||||
wechat_max_pages: int = 3
|
||||
|
||||
# --- 超时控制 ---
|
||||
engine_timeout: float = 15.0
|
||||
max_concurrent: int = 5
|
||||
|
||||
# --- 日志 ---
|
||||
log_level: str = "INFO"
|
||||
|
||||
except ImportError:
|
||||
|
||||
class Settings: # type: ignore
|
||||
"""回退配置类:无 pydantic-settings 时使用 os.environ。"""
|
||||
|
||||
def __init__(self):
|
||||
self.bing_api_key = os.environ.get("BING_API_KEY", "")
|
||||
self.bing_endpoint = os.environ.get("BING_ENDPOINT", "")
|
||||
self.baidu_api_key = os.environ.get("BAIDU_API_KEY", "")
|
||||
self.baidu_qianfan_api_key = os.environ.get(
|
||||
"BAIDU_QIANFAN_API_KEY", ""
|
||||
)
|
||||
self.wolfram_app_id = os.environ.get("WOLFRAM_APP_ID", "")
|
||||
self.twitter_bearer_token = os.environ.get(
|
||||
"TWITTER_BEARER_TOKEN", ""
|
||||
)
|
||||
self.wechat_search_enabled = (
|
||||
os.environ.get("WECHAT_SEARCH_ENABLED", "true").lower()
|
||||
== "true"
|
||||
)
|
||||
self.wechat_request_delay = float(
|
||||
os.environ.get("WECHAT_REQUEST_DELAY", "1.0")
|
||||
)
|
||||
self.wechat_max_pages = int(
|
||||
os.environ.get("WECHAT_MAX_PAGES", "3")
|
||||
)
|
||||
self.engine_timeout = float(
|
||||
os.environ.get("ENGINE_TIMEOUT", "15.0")
|
||||
)
|
||||
self.max_concurrent = int(
|
||||
os.environ.get("MAX_CONCURRENT", "5")
|
||||
)
|
||||
self.log_level = os.environ.get("LOG_LEVEL", "INFO")
|
||||
|
||||
|
||||
# 全局单例
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,102 @@
|
||||
"""
|
||||
搜索结果汇聚模块:去重、过滤、合并。
|
||||
|
||||
从 bot-search-mcp 移植并适配 web-search-skills 的数据结构。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from scripts.core.base import SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ResultAggregator:
|
||||
"""搜索结果汇聚器。
|
||||
|
||||
功能:
|
||||
- 按 URL 去重(保留第一次出现的)
|
||||
- 过滤低质量结果(空标题、过短摘要)
|
||||
- 可选按来源过滤 / 排序
|
||||
"""
|
||||
|
||||
MIN_TITLE_LENGTH = 2
|
||||
MIN_SNIPPET_LENGTH = 5
|
||||
|
||||
def aggregate(
|
||||
self,
|
||||
results: list[SearchResult],
|
||||
dedup_by: str = "url",
|
||||
filter_low_quality: bool = True,
|
||||
) -> list[SearchResult]:
|
||||
"""合并并去重搜索结果。
|
||||
|
||||
Args:
|
||||
results: 原始搜索结果列表
|
||||
dedup_by: 去重字段 ("url" 或 "title")
|
||||
filter_low_quality: 是否过滤低质量结果
|
||||
|
||||
Returns:
|
||||
处理后的结果列表
|
||||
"""
|
||||
merged = self._dedup(results, dedup_by)
|
||||
if filter_low_quality:
|
||||
merged = self._filter_low_quality(merged)
|
||||
return merged
|
||||
|
||||
def _dedup(
|
||||
self, results: list[SearchResult], by: str = "url"
|
||||
) -> list[SearchResult]:
|
||||
"""按指定字段去重。"""
|
||||
seen: set[str] = set()
|
||||
deduped: list[SearchResult] = []
|
||||
|
||||
for r in results:
|
||||
if by == "title":
|
||||
key = r.title.strip().lower()
|
||||
else:
|
||||
key = r.url.strip().rstrip("/")
|
||||
|
||||
if key and key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(r)
|
||||
|
||||
return deduped
|
||||
|
||||
def _filter_low_quality(
|
||||
self, results: list[SearchResult]
|
||||
) -> list[SearchResult]:
|
||||
"""过滤低质量结果。"""
|
||||
filtered: list[SearchResult] = []
|
||||
for r in results:
|
||||
if len(r.title.strip()) < self.MIN_TITLE_LENGTH:
|
||||
continue
|
||||
if len(r.snippet.strip()) < self.MIN_SNIPPET_LENGTH:
|
||||
continue
|
||||
filtered.append(r)
|
||||
return filtered
|
||||
|
||||
def merge_and_rank(
|
||||
self,
|
||||
results: list[SearchResult],
|
||||
dedup_by: str = "url",
|
||||
max_results: Optional[int] = None,
|
||||
) -> list[SearchResult]:
|
||||
"""去重 + 过滤 + 重新编号。
|
||||
|
||||
Args:
|
||||
results: 原始结果列表
|
||||
dedup_by: 去重字段
|
||||
max_results: 限制返回数量
|
||||
|
||||
Returns:
|
||||
处理后的结果列表(rank 重新从 1 开始编号)
|
||||
"""
|
||||
merged = self.aggregate(results, dedup_by=dedup_by)
|
||||
if max_results:
|
||||
merged = merged[:max_results]
|
||||
for i, r in enumerate(merged):
|
||||
r.rank = i + 1
|
||||
return merged
|
||||
@@ -0,0 +1,76 @@
|
||||
"""
|
||||
Pydantic 数据模型定义。
|
||||
|
||||
从 bot-search-mcp 移植,适配 web-search-skills 的数据结构。
|
||||
提供类型安全的搜索数据模型。
|
||||
|
||||
依赖: pydantic (>=2.0)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 搜索结果模型(兼容 dataclass + pydantic 双模式)
|
||||
# ============================================================
|
||||
|
||||
@dataclass
|
||||
class SearchQuery:
|
||||
"""搜索查询封装。"""
|
||||
query: str
|
||||
sources: Optional[list[str]] = None
|
||||
max_results: int = 10
|
||||
region: Optional[str] = None
|
||||
site: Optional[str] = None
|
||||
recency: Optional[str] = None
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceRef:
|
||||
"""引用来源。"""
|
||||
index: int
|
||||
title: str
|
||||
url: str
|
||||
source: str
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 性能计时模型
|
||||
# ============================================================
|
||||
|
||||
@dataclass
|
||||
class EngineTiming:
|
||||
"""单个引擎耗时记录。"""
|
||||
engine_name: str
|
||||
success: bool
|
||||
duration_ms: int
|
||||
result_count: int
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchMetadata:
|
||||
"""搜索元数据(全链路计时)。"""
|
||||
total_engines: int = 0
|
||||
total_raw_results: int = 0
|
||||
engines_used: list[str] = field(default_factory=list)
|
||||
duration_ms: int = 0
|
||||
engine_timings: list[EngineTiming] = field(default_factory=list)
|
||||
timed_out_engines: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 最终响应
|
||||
# ============================================================
|
||||
|
||||
@dataclass
|
||||
class SearchResponse:
|
||||
"""完整搜索响应。"""
|
||||
query: str
|
||||
results: list[Any] = field(default_factory=list)
|
||||
sources: list[SourceRef] = field(default_factory=list)
|
||||
metadata: SearchMetadata = field(default_factory=SearchMetadata)
|
||||
+52
-29
@@ -33,7 +33,7 @@ from scripts.engines.baidu_qianfan import BaiduQianFanEngine
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default timeout per engine (seconds)
|
||||
_DEFAULT_ENGINE_TIMEOUT = 15
|
||||
_DEFAULT_ENGINE_TIMEOUT = 8
|
||||
# Max concurrent engine calls
|
||||
_DEFAULT_MAX_CONCURRENT = 5
|
||||
|
||||
@@ -56,9 +56,10 @@ class UnifiedSearch:
|
||||
self._cache: dict[str, list[SearchResult]] = {}
|
||||
self._engines: dict[str, Any] = {}
|
||||
self._timeout = self.config.get("engine_timeout", _DEFAULT_ENGINE_TIMEOUT)
|
||||
self._semaphore = asyncio.Semaphore(
|
||||
self.config.get("max_concurrent", _DEFAULT_MAX_CONCURRENT)
|
||||
self._max_concurrent = self.config.get(
|
||||
"max_concurrent", _DEFAULT_MAX_CONCURRENT
|
||||
)
|
||||
self._semaphore = asyncio.Semaphore(self._max_concurrent)
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
@@ -70,7 +71,11 @@ class UnifiedSearch:
|
||||
self, query: str, sources: Optional[list[str]] = None,
|
||||
max_results: int = 10, force_refresh: bool = False, **kwargs,
|
||||
) -> list[SearchResult]:
|
||||
"""Search across selected sources and aggregate results."""
|
||||
"""Search across selected sources with streaming + early termination.
|
||||
|
||||
Returns results as soon as enough are collected, cancelling remaining
|
||||
slow engines to minimize total wait time.
|
||||
"""
|
||||
if not sources or "all" in sources:
|
||||
sources = list(SOURCE_CATEGORIES.keys())
|
||||
|
||||
@@ -78,49 +83,67 @@ class UnifiedSearch:
|
||||
if not force_refresh and cache_key in self._cache:
|
||||
return self._cache[cache_key]
|
||||
|
||||
tasks = []
|
||||
# --- Build task list --- #
|
||||
engine_list: list[tuple[Any, str]] = [] # (engine, query)
|
||||
for source_spec in sources:
|
||||
if source_spec in SOURCE_CATEGORIES:
|
||||
for engine_def in SOURCE_CATEGORIES[source_spec]():
|
||||
engine = self._get_engine(engine_def)
|
||||
if engine:
|
||||
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
|
||||
engine_list.append((engine, query))
|
||||
else:
|
||||
engine_def = get_engine(source_spec)
|
||||
if engine_def:
|
||||
engine = self._get_engine(engine_def)
|
||||
if engine:
|
||||
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
|
||||
engine_list.append((engine, query))
|
||||
|
||||
if not tasks:
|
||||
if not engine_list:
|
||||
return []
|
||||
|
||||
# Run all engines with semaphore-based concurrency control
|
||||
results_lists = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
all_results: list[SearchResult] = []
|
||||
for results in results_lists:
|
||||
if isinstance(results, Exception):
|
||||
logger.error(f"Search task failed: {results}")
|
||||
continue
|
||||
all_results.extend(results)
|
||||
engine_count = len(engine_list)
|
||||
|
||||
# Dedup by URL, keep first occurrence
|
||||
all_results = self._dedup_results(all_results)
|
||||
# --- Streaming: as_completed + early termination --- #
|
||||
seen_urls: set[str] = set()
|
||||
all_results: list[SearchResult] = []
|
||||
pending = set()
|
||||
|
||||
# Submit all tasks
|
||||
for engine, q in engine_list:
|
||||
coro = self._safe_search(engine, q, max_results, **kwargs)
|
||||
pending.add(asyncio.create_task(coro))
|
||||
|
||||
enough_results = max_results * 2 # allow some slack for dedup
|
||||
|
||||
# Process as they complete
|
||||
for done in asyncio.as_completed(pending):
|
||||
try:
|
||||
results = await done
|
||||
if isinstance(results, (Exception, BaseException)):
|
||||
continue
|
||||
|
||||
# Dedup on the fly
|
||||
for r in results:
|
||||
key = r.url.strip().rstrip("/")
|
||||
if key and key not in seen_urls:
|
||||
seen_urls.add(key)
|
||||
all_results.append(r)
|
||||
# Early termination: stop waiting for more engines
|
||||
if len(all_results) >= enough_results:
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Check early termination after each completion
|
||||
if len(all_results) >= enough_results:
|
||||
remaining = pending - {done}
|
||||
for t in remaining:
|
||||
t.cancel()
|
||||
break
|
||||
|
||||
self._cache[cache_key] = all_results
|
||||
return all_results
|
||||
|
||||
def _dedup_results(self, results: list[SearchResult]) -> list[SearchResult]:
|
||||
"""Remove duplicates by URL, keeping first occurrence."""
|
||||
seen: set[str] = set()
|
||||
deduped = []
|
||||
for r in results:
|
||||
key = r.url.strip().rstrip("/")
|
||||
if key and key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(r)
|
||||
return deduped
|
||||
|
||||
async def _safe_search(self, engine, query, max_results, **kwargs):
|
||||
async with self._semaphore:
|
||||
try:
|
||||
|
||||
@@ -72,9 +72,17 @@ class SogouWeChatEngine(SearchEngine):
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
|
||||
def _random_delay(self):
|
||||
"""Random delay between 500-1500ms to mimic human browsing."""
|
||||
delay = random.uniform(0.5, 1.5)
|
||||
# ---- Anti-scraping enhancements from bot-search-mcp ---- #
|
||||
|
||||
CAPTCHA_KEYWORDS = [
|
||||
"请输入验证码", "访问频率过高", "请稍后重试", "验证码",
|
||||
"captcha", "访问过于频繁", "请输入以下验证码",
|
||||
"403 Forbidden", "您的访问被拒绝",
|
||||
]
|
||||
|
||||
def _random_delay(self, min_s: float = 0.5, max_s: float = 1.5):
|
||||
"""Random delay to mimic human browsing."""
|
||||
delay = random.uniform(min_s, max_s)
|
||||
return asyncio.sleep(delay)
|
||||
|
||||
def _rotate_ua(self):
|
||||
@@ -82,7 +90,7 @@ class SogouWeChatEngine(SearchEngine):
|
||||
self._ua = random.choice(USER_AGENTS)
|
||||
|
||||
async def _get_cookie(self) -> dict[str, str]:
|
||||
"""Pre-fetch cookie from v.sogou.com to get SNUID."""
|
||||
"""Pre-fetch cookie from v.sogou.com to get SNUID/SUID/IPLOC/SUV."""
|
||||
try:
|
||||
response = await self._client.get(
|
||||
self.COOKIE_URL,
|
||||
@@ -92,19 +100,44 @@ class SogouWeChatEngine(SearchEngine):
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
},
|
||||
)
|
||||
cookies = {}
|
||||
for cookie in response.cookies.jar:
|
||||
if isinstance(cookie, tuple):
|
||||
# SimpleCookie format: (key, value)
|
||||
cookies[cookie[0]] = cookie[1]
|
||||
else:
|
||||
cookies[cookie.name] = cookie.value
|
||||
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com")
|
||||
cookies: dict[str, str] = {}
|
||||
|
||||
# Extract from httpx Cookies
|
||||
for key, value in response.cookies.items():
|
||||
if key.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
|
||||
cookies[key] = value
|
||||
|
||||
# Also parse from Set-Cookie header (more reliable)
|
||||
set_cookie = response.headers.get("set-cookie", "")
|
||||
for part in set_cookie.split(";"):
|
||||
if "=" in part:
|
||||
k, v = part.split("=", 1)
|
||||
k = k.strip()
|
||||
if k.upper() in ("SNUID", "SUID", "IPLOC", "SUV"):
|
||||
cookies[k] = v.split(",")[-1].strip()
|
||||
|
||||
logger.debug(f"Got {len(cookies)} cookies from v.sogou.com: {list(cookies.keys())}")
|
||||
return cookies
|
||||
except Exception as e:
|
||||
logger.warning(f"Cookie pre-fetch failed: {e}")
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def _detect_captcha(html: str) -> bool:
|
||||
"""检测是否触发了验证码或反爬。"""
|
||||
html_lower = html.lower()
|
||||
for kw in SogouWeChatEngine.CAPTCHA_KEYWORDS:
|
||||
if kw in html_lower or kw.lower() in html_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _exponential_backoff(
|
||||
self, attempt: int, base_delay: float = 2.0
|
||||
):
|
||||
"""指数退避等待(带随机 jitter)。"""
|
||||
wait = (2 ** attempt) + random.uniform(0, 1)
|
||||
await asyncio.sleep(wait)
|
||||
|
||||
def build_url(self, query: str, search_type: int = 2, page: int = 1, **kwargs) -> str:
|
||||
"""
|
||||
Build Sogou WeChat search URL.
|
||||
@@ -153,15 +186,13 @@ class SogouWeChatEngine(SearchEngine):
|
||||
async def _search_page(
|
||||
self, url: str, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Search a single page with retry logic."""
|
||||
"""Search a single page with enhanced retry + anti-scraping logic."""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
# Rotate UA on each retry
|
||||
if attempt > 0:
|
||||
self._rotate_ua()
|
||||
self._client.headers.update({"User-Agent": self._ua})
|
||||
if self._cookies:
|
||||
pass # Keep existing cookies
|
||||
|
||||
response = await self._client.get(
|
||||
url,
|
||||
@@ -169,34 +200,43 @@ class SogouWeChatEngine(SearchEngine):
|
||||
follow_redirects=False,
|
||||
)
|
||||
|
||||
# Check for anti-scraping page
|
||||
# 302 = anti-scraping redirect
|
||||
if response.status_code == 302:
|
||||
# Got redirected - likely hit anti-scraping
|
||||
logger.warning(f"Sogou anti-scraping triggered (302), attempt {attempt + 1}")
|
||||
# Try to get new cookie
|
||||
logger.warning(
|
||||
f"Sogou anti-scraping triggered (302), "
|
||||
f"attempt {attempt + 1}"
|
||||
)
|
||||
self._cookies = await self._get_cookie()
|
||||
await self._random_delay()
|
||||
await self._exponential_backoff(attempt)
|
||||
continue
|
||||
|
||||
if response.status_code != 200:
|
||||
logger.warning(f"Sogou returned {response.status_code}, attempt {attempt + 1}")
|
||||
await self._random_delay()
|
||||
logger.warning(
|
||||
f"Sogou returned {response.status_code}, "
|
||||
f"attempt {attempt + 1}"
|
||||
)
|
||||
await self._exponential_backoff(attempt)
|
||||
continue
|
||||
|
||||
html = response.text
|
||||
|
||||
# Check for captcha/block page
|
||||
if "请输入验证码" in html or "访问频率过高" in html:
|
||||
logger.warning(f"Sogou captcha triggered, attempt {attempt + 1}")
|
||||
await asyncio.sleep(random.uniform(2, 4))
|
||||
# Detect captcha / block page
|
||||
if self._detect_captcha(html):
|
||||
logger.warning(
|
||||
f"Sogou captcha/block detected, "
|
||||
f"attempt {attempt + 1}"
|
||||
)
|
||||
await self._exponential_backoff(attempt)
|
||||
self._cookies = await self._get_cookie()
|
||||
continue
|
||||
|
||||
return self._parse_results(html, query, max_results)
|
||||
|
||||
except (httpx.HTTPError, httpx.TimeoutException) as e:
|
||||
logger.warning(f"Sogou search attempt {attempt + 1} failed: {e}")
|
||||
await self._random_delay()
|
||||
logger.warning(
|
||||
f"Sogou search attempt {attempt + 1} failed: {e}"
|
||||
)
|
||||
await self._exponential_backoff(attempt)
|
||||
continue
|
||||
|
||||
return []
|
||||
|
||||
Reference in new issue
Block a user