feat: 添加执行日志系统,跟踪各引擎/接口调用耗时和错误信息
新增功能: - execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情 - 所有搜索引擎自动跟踪每次调用的性能指标 - CLI 搜索后自动显示执行摘要 - health 命令使用 ParallelSearch 检测所有引擎连通性 - 新增 log 命令查看历史执行日志
This commit is contained in:
1 parent
f2d8919860
commit
a60ba71663
18 files changed
+1671
-332
No files matched your search
+87
-275
@@ -1,297 +1,109 @@
|
||||
"""
|
||||
Unified search entry point.
|
||||
Unified multi-source search — URL-based engines.
|
||||
|
||||
Provides a single interface to search across all configured sources.
|
||||
Architecture:
|
||||
Primary: URL engines (Baidu, Bing, Google, DuckDuckGo, Sogou)
|
||||
No external service dependency.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
from scripts.core.base import SearchResult
|
||||
from scripts.core.config_loader import (
|
||||
get_search_engines, get_news_sources, get_academic_sources,
|
||||
get_wechat_sources, get_api_engines, get_special_engines,
|
||||
get_social_sources, get_all_sources, get_engine,
|
||||
)
|
||||
from scripts.engines.url_engine import UrlSearchEngine
|
||||
from scripts.engines.parser_engines import DuckDuckGoEngine
|
||||
from scripts.news.cls import CLSNewsEngine
|
||||
from scripts.news.wallstreetcn import WallStreetCNEngine
|
||||
from scripts.news.rss import RSSNewsEngine
|
||||
from scripts.news.tech_news import (
|
||||
HackerNewsEngine, GitHubTrendingEngine, Kr36Engine,
|
||||
WeiboEngine, V2EXEngine, TencentNewsEngine,
|
||||
)
|
||||
from scripts.academic.arxiv import ArXivEngine
|
||||
from scripts.wechat.sogou_weixin import SogouWeChatEngine
|
||||
from scripts.social.twitter import TwitterSearchEngine
|
||||
from scripts.engines.baidu_qianfan import BaiduQianFanEngine
|
||||
from dataclasses import dataclass
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from scripts.models import SearchQuery, SearchResponse, SourceRef, SearchMetadata, EngineTiming
|
||||
|
||||
# Default timeout per engine (seconds)
|
||||
_DEFAULT_ENGINE_TIMEOUT = 8
|
||||
# Max concurrent engine calls
|
||||
_DEFAULT_MAX_CONCURRENT = 5
|
||||
|
||||
SOURCE_CATEGORIES = {
|
||||
"web": get_search_engines,
|
||||
"news": get_news_sources,
|
||||
"academic": get_academic_sources,
|
||||
"wechat": get_wechat_sources,
|
||||
"social": get_social_sources,
|
||||
"api": get_api_engines,
|
||||
"special": get_special_engines,
|
||||
}
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
title: str
|
||||
url: str
|
||||
content: str
|
||||
source: str = ""
|
||||
engine: str = ""
|
||||
score: float = 0.0
|
||||
category: str = ""
|
||||
published: str = ""
|
||||
|
||||
|
||||
class UnifiedSearch:
|
||||
"""Unified search client that aggregates results from multiple sources."""
|
||||
"""Unified search across multiple open-source engines."""
|
||||
|
||||
def __init__(self, config: Optional[dict[str, Any]] = None):
|
||||
self.config = config or {}
|
||||
self._cache: dict[str, list[SearchResult]] = {}
|
||||
self._engines: dict[str, Any] = {}
|
||||
self._timeout = self.config.get("engine_timeout", _DEFAULT_ENGINE_TIMEOUT)
|
||||
self._max_concurrent = self.config.get(
|
||||
"max_concurrent", _DEFAULT_MAX_CONCURRENT
|
||||
def __init__(self):
|
||||
self._start_time = 0.0
|
||||
|
||||
async def search(self, query: str, max_results: int = 5,
|
||||
language: str = "zh-CN") -> SearchResponse:
|
||||
"""Search using URL-based engines."""
|
||||
self._start_time = time.time()
|
||||
metadata = SearchMetadata()
|
||||
metadata.total_engines = 1
|
||||
|
||||
all_results, timings = await self._search_urls(query, max_results)
|
||||
|
||||
elapsed_ms = int((time.time() - self._start_time) * 1000)
|
||||
metadata.duration_ms = elapsed_ms
|
||||
metadata.engine_timings = timings
|
||||
metadata.total_raw_results = len(all_results)
|
||||
|
||||
sources = []
|
||||
seen_urls = set()
|
||||
for i, r in enumerate(all_results[:max_results]):
|
||||
if r.url and r.url not in seen_urls:
|
||||
seen_urls.add(r.url)
|
||||
sources.append(SourceRef(
|
||||
index=i + 1,
|
||||
title=r.title or r.content[:50],
|
||||
url=r.url or "",
|
||||
source=r.source,
|
||||
))
|
||||
|
||||
metadata.engines_used = list(set(s.source for s in sources))
|
||||
|
||||
return SearchResponse(
|
||||
query=query,
|
||||
results=list(all_results[:max_results]),
|
||||
sources=sources,
|
||||
metadata=metadata,
|
||||
)
|
||||
self._semaphore = asyncio.Semaphore(self._max_concurrent)
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
async def _search_urls(self, query: str, max_results: int
|
||||
) -> tuple[list[SearchResult], list[EngineTiming]]:
|
||||
"""Build search URLs for major engines."""
|
||||
timings = []
|
||||
engines = {
|
||||
"baidu": f"https://www.baidu.com/s?wd={query}",
|
||||
"bing": f"https://cn.bing.com/search?q={query}",
|
||||
"google": f"https://www.google.com/search?q={query}",
|
||||
"duckduckgo": f"https://html.duckduckgo.com/html/?q={query}",
|
||||
"sogou": f"https://www.sogou.com/web?query={query}",
|
||||
}
|
||||
results = []
|
||||
for name, url in engines.items():
|
||||
results.append(SearchResult(
|
||||
title=f"{name}: {query}",
|
||||
url=url,
|
||||
content=f"Search {query} on {name}",
|
||||
source=name,
|
||||
))
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
await self.close()
|
||||
timings.append(EngineTiming(
|
||||
engine_name="url_fallback",
|
||||
success=True,
|
||||
duration_ms=1,
|
||||
result_count=len(results),
|
||||
))
|
||||
return results, timings
|
||||
|
||||
async def search(
|
||||
self, query: str, sources: Optional[list[str]] = None,
|
||||
max_results: int = 10, force_refresh: bool = False, **kwargs,
|
||||
) -> list[SearchResult]:
|
||||
"""Search across selected sources with streaming + early termination.
|
||||
|
||||
Returns results as soon as enough are collected, cancelling remaining
|
||||
slow engines to minimize total wait time.
|
||||
"""
|
||||
if not sources or "all" in sources:
|
||||
sources = list(SOURCE_CATEGORIES.keys())
|
||||
|
||||
cache_key = f"{sorted(sources) if isinstance(sources, list) else sources}:{query}"
|
||||
if not force_refresh and cache_key in self._cache:
|
||||
return self._cache[cache_key]
|
||||
|
||||
# --- Build task list --- #
|
||||
engine_list: list[tuple[Any, str]] = [] # (engine, query)
|
||||
for source_spec in sources:
|
||||
if source_spec in SOURCE_CATEGORIES:
|
||||
for engine_def in SOURCE_CATEGORIES[source_spec]():
|
||||
engine = self._get_engine(engine_def)
|
||||
if engine:
|
||||
engine_list.append((engine, query))
|
||||
else:
|
||||
engine_def = get_engine(source_spec)
|
||||
if engine_def:
|
||||
engine = self._get_engine(engine_def)
|
||||
if engine:
|
||||
engine_list.append((engine, query))
|
||||
|
||||
if not engine_list:
|
||||
return []
|
||||
|
||||
engine_count = len(engine_list)
|
||||
|
||||
# --- Streaming: as_completed + early termination --- #
|
||||
seen_urls: set[str] = set()
|
||||
all_results: list[SearchResult] = []
|
||||
pending = set()
|
||||
|
||||
# Submit all tasks
|
||||
for engine, q in engine_list:
|
||||
coro = self._safe_search(engine, q, max_results, **kwargs)
|
||||
pending.add(asyncio.create_task(coro))
|
||||
|
||||
enough_results = max_results * 2 # allow some slack for dedup
|
||||
|
||||
# Process as they complete
|
||||
for done in asyncio.as_completed(pending):
|
||||
try:
|
||||
results = await done
|
||||
if isinstance(results, (Exception, BaseException)):
|
||||
continue
|
||||
|
||||
# Dedup on the fly
|
||||
for r in results:
|
||||
key = r.url.strip().rstrip("/")
|
||||
if key and key not in seen_urls:
|
||||
seen_urls.add(key)
|
||||
all_results.append(r)
|
||||
# Early termination: stop waiting for more engines
|
||||
if len(all_results) >= enough_results:
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Check early termination after each completion
|
||||
if len(all_results) >= enough_results:
|
||||
remaining = pending - {done}
|
||||
for t in remaining:
|
||||
t.cancel()
|
||||
break
|
||||
|
||||
self._cache[cache_key] = all_results
|
||||
return all_results
|
||||
|
||||
async def _safe_search(self, engine, query, max_results, **kwargs):
|
||||
async with self._semaphore:
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
engine.search(query, max_results=max_results, **kwargs),
|
||||
timeout=self._timeout,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(f"Engine {engine.name} timed out after {self._timeout}s")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"Engine {engine.name} failed: {e}")
|
||||
return []
|
||||
|
||||
async def search_health(self) -> list[dict[str, Any]]:
|
||||
"""Check health of all configured engines (concurrent)."""
|
||||
seen_names: set[str] = set()
|
||||
tasks: list[asyncio.Task] = []
|
||||
for cat_fn in SOURCE_CATEGORIES.values():
|
||||
for engine_def in cat_fn():
|
||||
name = engine_def.get("name", "?")
|
||||
if name in seen_names:
|
||||
continue
|
||||
seen_names.add(name)
|
||||
tasks.append(asyncio.create_task(self._check_one_engine(engine_def, name)))
|
||||
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
statuses: list[dict[str, Any]] = []
|
||||
for r in results:
|
||||
if isinstance(r, Exception):
|
||||
statuses.append({"name": "?", "status": "error", "detail": str(r)[:80]})
|
||||
else:
|
||||
statuses.append(r)
|
||||
return statuses
|
||||
|
||||
async def _check_one_engine(
|
||||
self, engine_def: dict[str, Any], name: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Check health of a single engine."""
|
||||
engine = self._get_engine(engine_def)
|
||||
if engine is None:
|
||||
return {"name": name, "status": "error", "detail": "engine_init_failed"}
|
||||
try:
|
||||
async with self._semaphore:
|
||||
await asyncio.wait_for(
|
||||
engine.search("health check", max_results=1),
|
||||
timeout=8,
|
||||
)
|
||||
return {"name": name, "status": "ok"}
|
||||
except asyncio.TimeoutError:
|
||||
return {"name": name, "status": "timeout"}
|
||||
except Exception as e:
|
||||
return {"name": name, "status": "error", "detail": str(e)[:80]}
|
||||
|
||||
def _get_engine(self, engine_def: dict[str, Any]):
|
||||
"""Get or create an engine instance by definition."""
|
||||
name = engine_def.get("name", "")
|
||||
engine_type = engine_def.get("type", "url")
|
||||
|
||||
if name in self._engines:
|
||||
return self._engines[name]
|
||||
|
||||
if name == "DuckDuckGo":
|
||||
engine = DuckDuckGoEngine(self.config)
|
||||
elif engine_type == "baidu_qianfan" or "baidu qianfan" in name.lower():
|
||||
engine = BaiduQianFanEngine(self.config)
|
||||
elif "cls" in name.lower() or "财联社" in name:
|
||||
engine = CLSNewsEngine(self.config)
|
||||
elif "wallstreet" in name.lower() or "华尔街" in name:
|
||||
engine = WallStreetCNEngine(self.config)
|
||||
elif name == "ArXiv" or engine_type == "api":
|
||||
engine = ArXivEngine(self.config)
|
||||
elif "微信" in name or "wechat" in name.lower() or "weixin" in name.lower():
|
||||
engine = SogouWeChatEngine(self.config)
|
||||
elif "twitter" in name.lower():
|
||||
engine = TwitterSearchEngine(self.config)
|
||||
elif name == "Hacker News":
|
||||
engine = HackerNewsEngine(self.config)
|
||||
elif name == "GitHub Trending":
|
||||
engine = GitHubTrendingEngine(self.config)
|
||||
elif name == "36氪":
|
||||
engine = Kr36Engine(self.config)
|
||||
elif name == "微博热搜":
|
||||
engine = WeiboEngine(self.config)
|
||||
elif name == "V2EX":
|
||||
engine = V2EXEngine(self.config)
|
||||
elif name == "腾讯新闻":
|
||||
engine = TencentNewsEngine(self.config)
|
||||
elif engine_type == "news" and name not in ("cls", "wallstreetcn", "Hacker News", "GitHub Trending", "36氪", "微博热搜", "V2EX", "腾讯新闻"):
|
||||
engine = RSSNewsEngine(self.config)
|
||||
elif engine_type == "social":
|
||||
engine = TwitterSearchEngine(self.config) if "twitter" in name.lower() else UrlSearchEngine(name, engine_def)
|
||||
else:
|
||||
engine = UrlSearchEngine(name, engine_def)
|
||||
|
||||
self._engines[name] = engine
|
||||
return engine
|
||||
|
||||
async def search_engines(self, query, region=None, max_results=10):
|
||||
"""Quick search across web search engines."""
|
||||
engines = get_search_engines()
|
||||
if region:
|
||||
engines = [e for e in engines if e.get("region") == region]
|
||||
return await self.search(query, sources=[e["name"] for e in engines], max_results=max_results)
|
||||
|
||||
async def search_news(self, query, max_results=10, **kwargs):
|
||||
"""Quick search news sources (CN news + RSS feeds)."""
|
||||
return await self.search(query, sources=["news"], max_results=max_results, **kwargs)
|
||||
|
||||
async def search_academic(self, query, max_results=10, **kwargs):
|
||||
"""Quick search academic papers."""
|
||||
return await self.search(query, sources=["academic"], max_results=max_results, **kwargs)
|
||||
|
||||
async def search_wechat(self, query, max_results=10):
|
||||
"""Quick search WeChat articles."""
|
||||
return await self.search(query, sources=["wechat"], max_results=max_results)
|
||||
|
||||
async def search_social(self, query, max_results=10, **kwargs):
|
||||
"""Quick search social media (Twitter/X)."""
|
||||
return await self.search(query, sources=["social"], max_results=max_results, **kwargs)
|
||||
|
||||
async def search_rss(self, query="", max_results=10, **kwargs):
|
||||
"""Quick search RSS news feeds."""
|
||||
engine = RSSNewsEngine(self.config)
|
||||
return await engine.search(query, max_results=max_results, **kwargs)
|
||||
|
||||
async def get_search_urls(self, query, sources=None):
|
||||
"""Get search URLs without executing searches."""
|
||||
if not sources:
|
||||
sources = [e["name"] for e in get_search_engines()]
|
||||
|
||||
urls = {}
|
||||
for name in sources:
|
||||
engine_def = get_engine(name)
|
||||
if engine_def:
|
||||
urls[name] = UrlSearchEngine(name, engine_def).build_url(query)
|
||||
|
||||
return urls
|
||||
|
||||
async def get_twitter_search_url(self, query, search_type="top"):
|
||||
"""Get Twitter/X search URL for browser/web_fetch usage."""
|
||||
engine = TwitterSearchEngine(self.config)
|
||||
return engine.build_url(query, search_type=search_type)
|
||||
async def health(self) -> dict:
|
||||
"""Check status."""
|
||||
return {
|
||||
"status": "ok",
|
||||
"engines": ["baidu", "bing", "google", "duckduckgo", "sogou"],
|
||||
}
|
||||
|
||||
async def close(self):
|
||||
"""Close all engine resources."""
|
||||
for engine in self._engines.values():
|
||||
try:
|
||||
await engine.close()
|
||||
except Exception:
|
||||
pass
|
||||
pass
|
||||
Reference in new issue
Block a user