Implements the improvements from bot-search-mcp comparison (closes #5): - perf: asyncio.as_completed streaming + early termination (8s timeout) - feat: ResultAggregator (dedup + quality filter) - feat: anti-crawl enhancements for sogou_weixin.py - feat: config.py (pydantic-settings + .env support) - feat: models.py (data models with dataclasses) - docs: SKILL.md, README.md updates
103 lines
2.8 KiB
Python
103 lines
2.8 KiB
Python
"""
|
|
搜索结果汇聚模块:去重、过滤、合并。
|
|
|
|
从 bot-search-mcp 移植并适配 web-search-skills 的数据结构。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Optional
|
|
|
|
from scripts.core.base import SearchResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ResultAggregator:
|
|
"""搜索结果汇聚器。
|
|
|
|
功能:
|
|
- 按 URL 去重(保留第一次出现的)
|
|
- 过滤低质量结果(空标题、过短摘要)
|
|
- 可选按来源过滤 / 排序
|
|
"""
|
|
|
|
MIN_TITLE_LENGTH = 2
|
|
MIN_SNIPPET_LENGTH = 5
|
|
|
|
def aggregate(
|
|
self,
|
|
results: list[SearchResult],
|
|
dedup_by: str = "url",
|
|
filter_low_quality: bool = True,
|
|
) -> list[SearchResult]:
|
|
"""合并并去重搜索结果。
|
|
|
|
Args:
|
|
results: 原始搜索结果列表
|
|
dedup_by: 去重字段 ("url" 或 "title")
|
|
filter_low_quality: 是否过滤低质量结果
|
|
|
|
Returns:
|
|
处理后的结果列表
|
|
"""
|
|
merged = self._dedup(results, dedup_by)
|
|
if filter_low_quality:
|
|
merged = self._filter_low_quality(merged)
|
|
return merged
|
|
|
|
def _dedup(
|
|
self, results: list[SearchResult], by: str = "url"
|
|
) -> list[SearchResult]:
|
|
"""按指定字段去重。"""
|
|
seen: set[str] = set()
|
|
deduped: list[SearchResult] = []
|
|
|
|
for r in results:
|
|
if by == "title":
|
|
key = r.title.strip().lower()
|
|
else:
|
|
key = r.url.strip().rstrip("/")
|
|
|
|
if key and key not in seen:
|
|
seen.add(key)
|
|
deduped.append(r)
|
|
|
|
return deduped
|
|
|
|
def _filter_low_quality(
|
|
self, results: list[SearchResult]
|
|
) -> list[SearchResult]:
|
|
"""过滤低质量结果。"""
|
|
filtered: list[SearchResult] = []
|
|
for r in results:
|
|
if len(r.title.strip()) < self.MIN_TITLE_LENGTH:
|
|
continue
|
|
if len(r.snippet.strip()) < self.MIN_SNIPPET_LENGTH:
|
|
continue
|
|
filtered.append(r)
|
|
return filtered
|
|
|
|
def merge_and_rank(
|
|
self,
|
|
results: list[SearchResult],
|
|
dedup_by: str = "url",
|
|
max_results: Optional[int] = None,
|
|
) -> list[SearchResult]:
|
|
"""去重 + 过滤 + 重新编号。
|
|
|
|
Args:
|
|
results: 原始结果列表
|
|
dedup_by: 去重字段
|
|
max_results: 限制返回数量
|
|
|
|
Returns:
|
|
处理后的结果列表(rank 重新从 1 开始编号)
|
|
"""
|
|
merged = self.aggregate(results, dedup_by=dedup_by)
|
|
if max_results:
|
|
merged = merged[:max_results]
|
|
for i, r in enumerate(merged):
|
|
r.rank = i + 1
|
|
return merged
|