""" 搜索结果汇聚模块:去重、过滤、合并。 从 bot-search-mcp 移植并适配 web-search-skills 的数据结构。 """ from __future__ import annotations import logging from typing import Optional from scripts.core.base import SearchResult logger = logging.getLogger(__name__) class ResultAggregator: """搜索结果汇聚器。 功能: - 按 URL 去重(保留第一次出现的) - 过滤低质量结果(空标题、过短摘要) - 可选按来源过滤 / 排序 """ MIN_TITLE_LENGTH = 2 MIN_SNIPPET_LENGTH = 5 def aggregate( self, results: list[SearchResult], dedup_by: str = "url", filter_low_quality: bool = True, ) -> list[SearchResult]: """合并并去重搜索结果。 Args: results: 原始搜索结果列表 dedup_by: 去重字段 ("url" 或 "title") filter_low_quality: 是否过滤低质量结果 Returns: 处理后的结果列表 """ merged = self._dedup(results, dedup_by) if filter_low_quality: merged = self._filter_low_quality(merged) return merged def _dedup( self, results: list[SearchResult], by: str = "url" ) -> list[SearchResult]: """按指定字段去重。""" seen: set[str] = set() deduped: list[SearchResult] = [] for r in results: if by == "title": key = r.title.strip().lower() else: key = r.url.strip().rstrip("/") if key and key not in seen: seen.add(key) deduped.append(r) return deduped def _filter_low_quality( self, results: list[SearchResult] ) -> list[SearchResult]: """过滤低质量结果。""" filtered: list[SearchResult] = [] for r in results: if len(r.title.strip()) < self.MIN_TITLE_LENGTH: continue if len(r.snippet.strip()) < self.MIN_SNIPPET_LENGTH: continue filtered.append(r) return filtered def merge_and_rank( self, results: list[SearchResult], dedup_by: str = "url", max_results: Optional[int] = None, ) -> list[SearchResult]: """去重 + 过滤 + 重新编号。 Args: results: 原始结果列表 dedup_by: 去重字段 max_results: 限制返回数量 Returns: 处理后的结果列表(rank 重新从 1 开始编号) """ merged = self.aggregate(results, dedup_by=dedup_by) if max_results: merged = merged[:max_results] for i, r in enumerate(merged): r.rank = i + 1 return merged