feat: stream search, anti-crawl, aggregator, config, models

Implements the improvements from bot-search-mcp comparison (closes #5):

- perf: asyncio.as_completed streaming + early termination (8s timeout)
- feat: ResultAggregator (dedup + quality filter)
- feat: anti-crawl enhancements for sogou_weixin.py
- feat: config.py (pydantic-settings + .env support)
- feat: models.py (data models with dataclasses)
- docs: SKILL.md, README.md updates
This commit is contained in:
vex1023 committed 2026-05-20 12:23:21 +08:00
1 parent 06e3c6bdb9
commit 30158d8deb
5 files changed
+380 -57

No files matched your search

+52 -29
View File
@@ -33,7 +33,7 @@ from scripts.engines.baidu_qianfan import BaiduQianFanEngine
logger = logging.getLogger(__name__)
# Default timeout per engine (seconds)
_DEFAULT_ENGINE_TIMEOUT = 15
_DEFAULT_ENGINE_TIMEOUT = 8
# Max concurrent engine calls
_DEFAULT_MAX_CONCURRENT = 5
@@ -56,9 +56,10 @@ class UnifiedSearch:
self._cache: dict[str, list[SearchResult]] = {}
self._engines: dict[str, Any] = {}
self._timeout = self.config.get("engine_timeout", _DEFAULT_ENGINE_TIMEOUT)
self._semaphore = asyncio.Semaphore(
self.config.get("max_concurrent", _DEFAULT_MAX_CONCURRENT)
self._max_concurrent = self.config.get(
"max_concurrent", _DEFAULT_MAX_CONCURRENT
)
self._semaphore = asyncio.Semaphore(self._max_concurrent)
async def __aenter__(self):
return self
@@ -70,7 +71,11 @@ class UnifiedSearch:
self, query: str, sources: Optional[list[str]] = None,
max_results: int = 10, force_refresh: bool = False, **kwargs,
) -> list[SearchResult]:
"""Search across selected sources and aggregate results."""
"""Search across selected sources with streaming + early termination.
Returns results as soon as enough are collected, cancelling remaining
slow engines to minimize total wait time.
"""
if not sources or "all" in sources:
sources = list(SOURCE_CATEGORIES.keys())
@@ -78,49 +83,67 @@ class UnifiedSearch:
if not force_refresh and cache_key in self._cache:
return self._cache[cache_key]
tasks = []
# --- Build task list --- #
engine_list: list[tuple[Any, str]] = [] # (engine, query)
for source_spec in sources:
if source_spec in SOURCE_CATEGORIES:
for engine_def in SOURCE_CATEGORIES[source_spec]():
engine = self._get_engine(engine_def)
if engine:
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
engine_list.append((engine, query))
else:
engine_def = get_engine(source_spec)
if engine_def:
engine = self._get_engine(engine_def)
if engine:
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
engine_list.append((engine, query))
if not tasks:
if not engine_list:
return []
# Run all engines with semaphore-based concurrency control
results_lists = await asyncio.gather(*tasks, return_exceptions=True)
all_results: list[SearchResult] = []
for results in results_lists:
if isinstance(results, Exception):
logger.error(f"Search task failed: {results}")
continue
all_results.extend(results)
engine_count = len(engine_list)
# Dedup by URL, keep first occurrence
all_results = self._dedup_results(all_results)
# --- Streaming: as_completed + early termination --- #
seen_urls: set[str] = set()
all_results: list[SearchResult] = []
pending = set()
# Submit all tasks
for engine, q in engine_list:
coro = self._safe_search(engine, q, max_results, **kwargs)
pending.add(asyncio.create_task(coro))
enough_results = max_results * 2 # allow some slack for dedup
# Process as they complete
for done in asyncio.as_completed(pending):
try:
results = await done
if isinstance(results, (Exception, BaseException)):
continue
# Dedup on the fly
for r in results:
key = r.url.strip().rstrip("/")
if key and key not in seen_urls:
seen_urls.add(key)
all_results.append(r)
# Early termination: stop waiting for more engines
if len(all_results) >= enough_results:
break
except Exception:
continue
# Check early termination after each completion
if len(all_results) >= enough_results:
remaining = pending - {done}
for t in remaining:
t.cancel()
break
self._cache[cache_key] = all_results
return all_results
def _dedup_results(self, results: list[SearchResult]) -> list[SearchResult]:
"""Remove duplicates by URL, keeping first occurrence."""
seen: set[str] = set()
deduped = []
for r in results:
key = r.url.strip().rstrip("/")
if key and key not in seen:
seen.add(key)
deduped.append(r)
return deduped
async def _safe_search(self, engine, query, max_results, **kwargs):
async with self._semaphore:
try: