Files
vex1023 30158d8deb feat: stream search, anti-crawl, aggregator, config, models
Implements the improvements from bot-search-mcp comparison (closes #5):

- perf: asyncio.as_completed streaming + early termination (8s timeout)
- feat: ResultAggregator (dedup + quality filter)
- feat: anti-crawl enhancements for sogou_weixin.py
- feat: config.py (pydantic-settings + .env support)
- feat: models.py (data models with dataclasses)
- docs: SKILL.md, README.md updates
2026-05-20 12:23:21 +08:00

103 lines
2.8 KiB
Python

"""
搜索结果汇聚模块:去重、过滤、合并。
从 bot-search-mcp 移植并适配 web-search-skills 的数据结构。
"""
from __future__ import annotations
import logging
from typing import Optional
from scripts.core.base import SearchResult
logger = logging.getLogger(__name__)
class ResultAggregator:
"""搜索结果汇聚器。
功能:
- 按 URL 去重(保留第一次出现的)
- 过滤低质量结果(空标题、过短摘要)
- 可选按来源过滤 / 排序
"""
MIN_TITLE_LENGTH = 2
MIN_SNIPPET_LENGTH = 5
def aggregate(
self,
results: list[SearchResult],
dedup_by: str = "url",
filter_low_quality: bool = True,
) -> list[SearchResult]:
"""合并并去重搜索结果。
Args:
results: 原始搜索结果列表
dedup_by: 去重字段 ("url" 或 "title")
filter_low_quality: 是否过滤低质量结果
Returns:
处理后的结果列表
"""
merged = self._dedup(results, dedup_by)
if filter_low_quality:
merged = self._filter_low_quality(merged)
return merged
def _dedup(
self, results: list[SearchResult], by: str = "url"
) -> list[SearchResult]:
"""按指定字段去重。"""
seen: set[str] = set()
deduped: list[SearchResult] = []
for r in results:
if by == "title":
key = r.title.strip().lower()
else:
key = r.url.strip().rstrip("/")
if key and key not in seen:
seen.add(key)
deduped.append(r)
return deduped
def _filter_low_quality(
self, results: list[SearchResult]
) -> list[SearchResult]:
"""过滤低质量结果。"""
filtered: list[SearchResult] = []
for r in results:
if len(r.title.strip()) < self.MIN_TITLE_LENGTH:
continue
if len(r.snippet.strip()) < self.MIN_SNIPPET_LENGTH:
continue
filtered.append(r)
return filtered
def merge_and_rank(
self,
results: list[SearchResult],
dedup_by: str = "url",
max_results: Optional[int] = None,
) -> list[SearchResult]:
"""去重 + 过滤 + 重新编号。
Args:
results: 原始结果列表
dedup_by: 去重字段
max_results: 限制返回数量
Returns:
处理后的结果列表(rank 重新从 1 开始编号)
"""
merged = self.aggregate(results, dedup_by=dedup_by)
if max_results:
merged = merged[:max_results]
for i, r in enumerate(merged):
r.rank = i + 1
return merged