feat: stream search, anti-crawl, aggregator, config, models
Implements the improvements from bot-search-mcp comparison (closes #5): - perf: asyncio.as_completed streaming + early termination (8s timeout) - feat: ResultAggregator (dedup + quality filter) - feat: anti-crawl enhancements for sogou_weixin.py - feat: config.py (pydantic-settings + .env support) - feat: models.py (data models with dataclasses) - docs: SKILL.md, README.md updates
This commit is contained in:
1 parent
06e3c6bdb9
commit
30158d8deb
5 files changed
+380
-57
No files matched your search
@@ -0,0 +1,102 @@
|
||||
"""
|
||||
搜索结果汇聚模块:去重、过滤、合并。
|
||||
|
||||
从 bot-search-mcp 移植并适配 web-search-skills 的数据结构。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from scripts.core.base import SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ResultAggregator:
|
||||
"""搜索结果汇聚器。
|
||||
|
||||
功能:
|
||||
- 按 URL 去重(保留第一次出现的)
|
||||
- 过滤低质量结果(空标题、过短摘要)
|
||||
- 可选按来源过滤 / 排序
|
||||
"""
|
||||
|
||||
MIN_TITLE_LENGTH = 2
|
||||
MIN_SNIPPET_LENGTH = 5
|
||||
|
||||
def aggregate(
|
||||
self,
|
||||
results: list[SearchResult],
|
||||
dedup_by: str = "url",
|
||||
filter_low_quality: bool = True,
|
||||
) -> list[SearchResult]:
|
||||
"""合并并去重搜索结果。
|
||||
|
||||
Args:
|
||||
results: 原始搜索结果列表
|
||||
dedup_by: 去重字段 ("url" 或 "title")
|
||||
filter_low_quality: 是否过滤低质量结果
|
||||
|
||||
Returns:
|
||||
处理后的结果列表
|
||||
"""
|
||||
merged = self._dedup(results, dedup_by)
|
||||
if filter_low_quality:
|
||||
merged = self._filter_low_quality(merged)
|
||||
return merged
|
||||
|
||||
def _dedup(
|
||||
self, results: list[SearchResult], by: str = "url"
|
||||
) -> list[SearchResult]:
|
||||
"""按指定字段去重。"""
|
||||
seen: set[str] = set()
|
||||
deduped: list[SearchResult] = []
|
||||
|
||||
for r in results:
|
||||
if by == "title":
|
||||
key = r.title.strip().lower()
|
||||
else:
|
||||
key = r.url.strip().rstrip("/")
|
||||
|
||||
if key and key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(r)
|
||||
|
||||
return deduped
|
||||
|
||||
def _filter_low_quality(
|
||||
self, results: list[SearchResult]
|
||||
) -> list[SearchResult]:
|
||||
"""过滤低质量结果。"""
|
||||
filtered: list[SearchResult] = []
|
||||
for r in results:
|
||||
if len(r.title.strip()) < self.MIN_TITLE_LENGTH:
|
||||
continue
|
||||
if len(r.snippet.strip()) < self.MIN_SNIPPET_LENGTH:
|
||||
continue
|
||||
filtered.append(r)
|
||||
return filtered
|
||||
|
||||
def merge_and_rank(
|
||||
self,
|
||||
results: list[SearchResult],
|
||||
dedup_by: str = "url",
|
||||
max_results: Optional[int] = None,
|
||||
) -> list[SearchResult]:
|
||||
"""去重 + 过滤 + 重新编号。
|
||||
|
||||
Args:
|
||||
results: 原始结果列表
|
||||
dedup_by: 去重字段
|
||||
max_results: 限制返回数量
|
||||
|
||||
Returns:
|
||||
处理后的结果列表(rank 重新从 1 开始编号)
|
||||
"""
|
||||
merged = self.aggregate(results, dedup_by=dedup_by)
|
||||
if max_results:
|
||||
merged = merged[:max_results]
|
||||
for i, r in enumerate(merged):
|
||||
r.rank = i + 1
|
||||
return merged
|
||||
Reference in new issue
Block a user