feat: stream search, anti-crawl, aggregator, config, models
Implements the improvements from bot-search-mcp comparison (closes #5): - perf: asyncio.as_completed streaming + early termination (8s timeout) - feat: ResultAggregator (dedup + quality filter) - feat: anti-crawl enhancements for sogou_weixin.py - feat: config.py (pydantic-settings + .env support) - feat: models.py (data models with dataclasses) - docs: SKILL.md, README.md updates
This commit is contained in:
1 parent
06e3c6bdb9
commit
30158d8deb
5 files changed
+380
-57
No files matched your search
+52
-29
@@ -33,7 +33,7 @@ from scripts.engines.baidu_qianfan import BaiduQianFanEngine
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default timeout per engine (seconds)
|
||||
_DEFAULT_ENGINE_TIMEOUT = 15
|
||||
_DEFAULT_ENGINE_TIMEOUT = 8
|
||||
# Max concurrent engine calls
|
||||
_DEFAULT_MAX_CONCURRENT = 5
|
||||
|
||||
@@ -56,9 +56,10 @@ class UnifiedSearch:
|
||||
self._cache: dict[str, list[SearchResult]] = {}
|
||||
self._engines: dict[str, Any] = {}
|
||||
self._timeout = self.config.get("engine_timeout", _DEFAULT_ENGINE_TIMEOUT)
|
||||
self._semaphore = asyncio.Semaphore(
|
||||
self.config.get("max_concurrent", _DEFAULT_MAX_CONCURRENT)
|
||||
self._max_concurrent = self.config.get(
|
||||
"max_concurrent", _DEFAULT_MAX_CONCURRENT
|
||||
)
|
||||
self._semaphore = asyncio.Semaphore(self._max_concurrent)
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
@@ -70,7 +71,11 @@ class UnifiedSearch:
|
||||
self, query: str, sources: Optional[list[str]] = None,
|
||||
max_results: int = 10, force_refresh: bool = False, **kwargs,
|
||||
) -> list[SearchResult]:
|
||||
"""Search across selected sources and aggregate results."""
|
||||
"""Search across selected sources with streaming + early termination.
|
||||
|
||||
Returns results as soon as enough are collected, cancelling remaining
|
||||
slow engines to minimize total wait time.
|
||||
"""
|
||||
if not sources or "all" in sources:
|
||||
sources = list(SOURCE_CATEGORIES.keys())
|
||||
|
||||
@@ -78,49 +83,67 @@ class UnifiedSearch:
|
||||
if not force_refresh and cache_key in self._cache:
|
||||
return self._cache[cache_key]
|
||||
|
||||
tasks = []
|
||||
# --- Build task list --- #
|
||||
engine_list: list[tuple[Any, str]] = [] # (engine, query)
|
||||
for source_spec in sources:
|
||||
if source_spec in SOURCE_CATEGORIES:
|
||||
for engine_def in SOURCE_CATEGORIES[source_spec]():
|
||||
engine = self._get_engine(engine_def)
|
||||
if engine:
|
||||
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
|
||||
engine_list.append((engine, query))
|
||||
else:
|
||||
engine_def = get_engine(source_spec)
|
||||
if engine_def:
|
||||
engine = self._get_engine(engine_def)
|
||||
if engine:
|
||||
tasks.append(self._safe_search(engine, query, max_results, **kwargs))
|
||||
engine_list.append((engine, query))
|
||||
|
||||
if not tasks:
|
||||
if not engine_list:
|
||||
return []
|
||||
|
||||
# Run all engines with semaphore-based concurrency control
|
||||
results_lists = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
all_results: list[SearchResult] = []
|
||||
for results in results_lists:
|
||||
if isinstance(results, Exception):
|
||||
logger.error(f"Search task failed: {results}")
|
||||
continue
|
||||
all_results.extend(results)
|
||||
engine_count = len(engine_list)
|
||||
|
||||
# Dedup by URL, keep first occurrence
|
||||
all_results = self._dedup_results(all_results)
|
||||
# --- Streaming: as_completed + early termination --- #
|
||||
seen_urls: set[str] = set()
|
||||
all_results: list[SearchResult] = []
|
||||
pending = set()
|
||||
|
||||
# Submit all tasks
|
||||
for engine, q in engine_list:
|
||||
coro = self._safe_search(engine, q, max_results, **kwargs)
|
||||
pending.add(asyncio.create_task(coro))
|
||||
|
||||
enough_results = max_results * 2 # allow some slack for dedup
|
||||
|
||||
# Process as they complete
|
||||
for done in asyncio.as_completed(pending):
|
||||
try:
|
||||
results = await done
|
||||
if isinstance(results, (Exception, BaseException)):
|
||||
continue
|
||||
|
||||
# Dedup on the fly
|
||||
for r in results:
|
||||
key = r.url.strip().rstrip("/")
|
||||
if key and key not in seen_urls:
|
||||
seen_urls.add(key)
|
||||
all_results.append(r)
|
||||
# Early termination: stop waiting for more engines
|
||||
if len(all_results) >= enough_results:
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Check early termination after each completion
|
||||
if len(all_results) >= enough_results:
|
||||
remaining = pending - {done}
|
||||
for t in remaining:
|
||||
t.cancel()
|
||||
break
|
||||
|
||||
self._cache[cache_key] = all_results
|
||||
return all_results
|
||||
|
||||
def _dedup_results(self, results: list[SearchResult]) -> list[SearchResult]:
|
||||
"""Remove duplicates by URL, keeping first occurrence."""
|
||||
seen: set[str] = set()
|
||||
deduped = []
|
||||
for r in results:
|
||||
key = r.url.strip().rstrip("/")
|
||||
if key and key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(r)
|
||||
return deduped
|
||||
|
||||
async def _safe_search(self, engine, query, max_results, **kwargs):
|
||||
async with self._semaphore:
|
||||
try:
|
||||
|
||||
Reference in new issue
Block a user