feat: 添加执行日志系统,跟踪各引擎/接口调用耗时和错误信息
新增功能: - execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情 - 所有搜索引擎自动跟踪每次调用的性能指标 - CLI 搜索后自动显示执行摘要 - health 命令使用 ParallelSearch 检测所有引擎连通性 - 新增 log 命令查看历史执行日志
This commit is contained in:
1 parent
f2d8919860
commit
9d97a3a7e8
18 files changed
+1671
-332
No files matched your search
+182
-35
@@ -68,6 +68,9 @@ logging.basicConfig(
|
||||
)
|
||||
logger = logging.getLogger("cli")
|
||||
|
||||
# 执行日志模块
|
||||
from scripts.core.execution_log import execution_log
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lazy engine imports (import on demand to keep startup fast)
|
||||
@@ -78,6 +81,11 @@ def _get_unified():
|
||||
return UnifiedSearch
|
||||
|
||||
|
||||
def _get_parallel():
|
||||
from scripts.engines.parallel_search import ParallelSearch
|
||||
return ParallelSearch
|
||||
|
||||
|
||||
def _get_sogou():
|
||||
from scripts.wechat.sogou_weixin import SogouWeChatEngine
|
||||
return SogouWeChatEngine
|
||||
@@ -160,7 +168,41 @@ def _sanitize_for_console(text: str) -> str:
|
||||
|
||||
|
||||
def _format_result(r, fmt: str = "text") -> str:
|
||||
"""Format a single SearchResult."""
|
||||
"""Format a single search result (supports both EngineResult and old models)."""
|
||||
# Handle EngineResult (new engines)
|
||||
if hasattr(r, "source") and hasattr(r, "snippet"):
|
||||
title = getattr(r, "title", "")
|
||||
url = getattr(r, "url", "")
|
||||
snippet = getattr(r, "snippet", "")
|
||||
source = getattr(r, "source", "")
|
||||
category = getattr(r, "category", "")
|
||||
extra = getattr(r, "extra", {})
|
||||
published = getattr(r, "published", "")
|
||||
|
||||
if fmt == "json":
|
||||
return json.dumps({
|
||||
"title": title, "url": url, "snippet": snippet[:300],
|
||||
"source": source, "category": category,
|
||||
"published": published, "extra": extra,
|
||||
}, ensure_ascii=False, indent=2)
|
||||
|
||||
lines = [_sanitize_for_console(title)]
|
||||
lines.append(f" URL: {url}")
|
||||
if snippet:
|
||||
lines.append(f" {_sanitize_for_console(snippet[:200])}")
|
||||
extra_parts = []
|
||||
if source:
|
||||
extra_parts.append(f"来源: {source}")
|
||||
if published:
|
||||
extra_parts.append(f"时间: {published}")
|
||||
for k, v in extra.items():
|
||||
if k in ("stars", "language") and v:
|
||||
extra_parts.append(f"{k}: {v}")
|
||||
if extra_parts:
|
||||
lines.append(f" ({'; '.join(extra_parts)})")
|
||||
return "\n".join(lines)
|
||||
|
||||
# Handle old SearchResult format (rank, title, url, snippet, source, category, extra)
|
||||
if fmt == "json":
|
||||
return json.dumps({
|
||||
"rank": r.rank,
|
||||
@@ -224,49 +266,61 @@ def _print_results(results, fmt: str = "text"):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _search_all(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search(query, max_results=max_results, **kwargs)
|
||||
finally:
|
||||
await us.close()
|
||||
"""Parallel search across all general engines."""
|
||||
ps = _get_parallel()(timeout=10.0)
|
||||
region = kwargs.get("region", "cn")
|
||||
results, stats = await ps.search(
|
||||
query, engines=None, max_results=max_results, region=region,
|
||||
)
|
||||
logger.debug(f"Stats: {stats}")
|
||||
return results
|
||||
|
||||
|
||||
async def _search_web(query: str, max_results: int, region: str = None, **kwargs) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search_engines(query, region=region, max_results=max_results)
|
||||
finally:
|
||||
await us.close()
|
||||
"""Parallel web search (general engines)."""
|
||||
ps = _get_parallel()(timeout=10.0)
|
||||
results, stats = await ps.search(
|
||||
query, engines=None, max_results=max_results, region=region or "cn",
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
async def _search_news(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search_news(query, max_results=max_results, **kwargs)
|
||||
finally:
|
||||
await us.close()
|
||||
"""Vertical news search."""
|
||||
ps = _get_parallel()(timeout=10.0)
|
||||
results, stats = await ps.search_vertical(
|
||||
query, "news", max_results=max_results, **kwargs,
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
async def _search_wechat(query: str, max_results: int) -> list[Any]:
|
||||
engine = _get_sogou()()
|
||||
try:
|
||||
return await engine.search(query, max_results=max_results)
|
||||
with execution_log.track("weixin", "search", query=query) as ctx:
|
||||
results = await engine.search(query, max_results=max_results)
|
||||
ctx.result_count = len(results)
|
||||
return results
|
||||
finally:
|
||||
await engine.close()
|
||||
|
||||
|
||||
async def _search_academic(query: str, max_results: int) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search_academic(query, max_results=max_results)
|
||||
finally:
|
||||
await us.close()
|
||||
"""Vertical academic search (ArXiv + CrossRef)."""
|
||||
ps = _get_parallel()(timeout=15.0)
|
||||
results, stats = await ps.search_vertical(
|
||||
query, "academic", max_results=max_results,
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search_social(query, max_results=max_results, **kwargs)
|
||||
with execution_log.track("social", "search", query=query) as ctx:
|
||||
results = await us.search_social(query, max_results=max_results, **kwargs)
|
||||
ctx.result_count = len(results) if results else 0
|
||||
return results
|
||||
finally:
|
||||
await us.close()
|
||||
|
||||
@@ -275,7 +329,10 @@ async def _search_baidu(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||
from scripts.engines.baidu_qianfan import BaiduQianFanEngine
|
||||
engine = BaiduQianFanEngine({"api_key": os.getenv("BAIDU_API_KEY", "")})
|
||||
try:
|
||||
return await engine.search(query, max_results=max_results, **kwargs)
|
||||
with execution_log.track("baidu_qianfan", "search", query=query) as ctx:
|
||||
results = await engine.search(query, max_results=max_results, **kwargs)
|
||||
ctx.result_count = len(results)
|
||||
return results
|
||||
finally:
|
||||
await engine.close()
|
||||
|
||||
@@ -283,11 +340,24 @@ async def _search_baidu(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||
async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
return await us.search_rss(query, max_results=max_results, **kwargs)
|
||||
with execution_log.track("rss", "search", query=query) as ctx:
|
||||
results = await us.search_rss(query, max_results=max_results, **kwargs)
|
||||
ctx.result_count = len(results) if results else 0
|
||||
return results
|
||||
finally:
|
||||
await us.close()
|
||||
|
||||
|
||||
async def _search_github(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||
"""GitHub code/repo search."""
|
||||
ps = _get_parallel()(timeout=10.0)
|
||||
search_type = kwargs.get("search_type", "repositories")
|
||||
results, stats = await ps.search_vertical(
|
||||
query, "github", max_results=max_results, search_type=search_type,
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]:
|
||||
us = _get_unified()()
|
||||
try:
|
||||
@@ -300,6 +370,69 @@ async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]:
|
||||
# CLI actions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def cmd_log(args):
|
||||
"""查看执行日志历史。"""
|
||||
import json as json_mod
|
||||
|
||||
log_file = "logs/search_log.jsonl"
|
||||
try:
|
||||
with open(log_file, "r", encoding="utf-8") as f:
|
||||
lines = [json_mod.loads(line) for line in f if line.strip()]
|
||||
except FileNotFoundError:
|
||||
print(f"暂无日志文件 ({log_file})")
|
||||
return
|
||||
|
||||
if not lines:
|
||||
print("暂无执行记录")
|
||||
return
|
||||
|
||||
# 按 session 分组
|
||||
sessions = {}
|
||||
for entry in lines:
|
||||
session = entry.get("session_start", "unknown")
|
||||
if session not in sessions:
|
||||
sessions[session] = []
|
||||
sessions[session].append(entry)
|
||||
|
||||
print(f"═══════════════════════════════════════════")
|
||||
print(f" 执行日志历史 (共 {len(lines)} 条记录, {len(sessions)} 个会话)")
|
||||
print(f"═══════════════════════════════════════════")
|
||||
|
||||
# 只显示最近5个会话
|
||||
recent_sessions = list(sessions.items())[-5:]
|
||||
for session_start, entries in recent_sessions:
|
||||
total = len(entries)
|
||||
success = sum(1 for e in entries if e.get("success", True))
|
||||
failed = total - success
|
||||
durations = [e.get("duration_ms", 0) for e in entries]
|
||||
avg_dur = sum(durations) // total if total else 0
|
||||
|
||||
print(f"\n 会话: {session_start}")
|
||||
print(f" 调用: {total} 次 (成功 {success}, 失败 {failed})")
|
||||
print(f" 平均耗时: {avg_dur}ms")
|
||||
|
||||
# 按引擎分组
|
||||
engines = {}
|
||||
for e in entries:
|
||||
eng = e.get("engine", "unknown")
|
||||
if eng not in engines:
|
||||
engines[eng] = {"calls": 0, "errors": []}
|
||||
engines[eng]["calls"] += 1
|
||||
if not e.get("success", True):
|
||||
engines[eng]["errors"].append(e.get("error", "unknown"))
|
||||
|
||||
for eng, info in sorted(engines.items()):
|
||||
err_count = len(info["errors"])
|
||||
status = "✓" if err_count == 0 else f"✗ ({err_count} errors)"
|
||||
print(f" {eng:15s} {status} calls={info['calls']}")
|
||||
|
||||
# 显示错误详情
|
||||
for err in info["errors"][:3]: # 最多显示3个
|
||||
print(f" └─ {err[:80]}")
|
||||
|
||||
print(f"\n═══════════════════════════════════════════")
|
||||
|
||||
|
||||
def cmd_sources(args):
|
||||
"""List all available search sources."""
|
||||
cl = _get_config_loader()
|
||||
@@ -337,22 +470,25 @@ def cmd_urls(args):
|
||||
|
||||
async def cmd_health(args):
|
||||
"""Test all engine connectivity."""
|
||||
us = _get_unified()()
|
||||
ps = _get_parallel()(timeout=10.0)
|
||||
try:
|
||||
print("正在检测各引擎连通性...\n")
|
||||
statuses = await us.search_health()
|
||||
ok_count = sum(1 for s in statuses if s["status"] == "ok")
|
||||
for s in statuses:
|
||||
statuses = await ps.health()
|
||||
ok_count = sum(1 for s in statuses.values() if s["status"] == "ok")
|
||||
for name, s in statuses.items():
|
||||
if s["status"] == "ok":
|
||||
print(f" [OK] {s['name']}")
|
||||
print(f" [OK] {name}")
|
||||
elif s["status"] == "timeout":
|
||||
print(f" [TIMEOUT] {s['name']}")
|
||||
print(f" [TIMEOUT] {name}")
|
||||
else:
|
||||
detail = s.get("detail", "")
|
||||
print(f" [FAIL] {s['name']} {detail}")
|
||||
detail = s.get("error", s.get("detail", ""))
|
||||
print(f" [FAIL] {name} {detail}")
|
||||
print(f"\n总计: {len(statuses)} 个引擎, {ok_count} 个可用")
|
||||
print()
|
||||
execution_log.print_summary()
|
||||
execution_log.save("logs/search_log.jsonl")
|
||||
finally:
|
||||
await us.close()
|
||||
pass
|
||||
|
||||
|
||||
async def _run_search(args):
|
||||
@@ -369,6 +505,7 @@ async def _run_search(args):
|
||||
"academic": _search_academic,
|
||||
"social": _search_social,
|
||||
"rss": _search_rss,
|
||||
"github": _search_github,
|
||||
}
|
||||
|
||||
search_fn = method_map.get(args.command, _search_all)
|
||||
@@ -408,6 +545,7 @@ async def _run_search(args):
|
||||
"academic": "学术搜索",
|
||||
"social": "社交搜索",
|
||||
"rss": "RSS 新闻搜索",
|
||||
"github": "GitHub 搜索",
|
||||
}
|
||||
header = cmd_names.get(args.command, "搜索")
|
||||
print(f"{header}: {args.query}")
|
||||
@@ -415,6 +553,11 @@ async def _run_search(args):
|
||||
|
||||
_print_results(results, fmt="json" if args.json else "text")
|
||||
|
||||
# 打印执行日志摘要并保存到文件
|
||||
print()
|
||||
execution_log.print_summary()
|
||||
execution_log.save("logs/search_log.jsonl")
|
||||
|
||||
# Save to file if requested (with basic path-sanitization)
|
||||
if args.output:
|
||||
output_data = json.dumps(
|
||||
@@ -468,7 +611,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
nargs="?",
|
||||
default="search",
|
||||
choices=["search", "web", "baidu", "news", "wechat", "academic",
|
||||
"social", "rss", "health", "sources", "urls"],
|
||||
"social", "rss", "github", "health", "sources", "urls", "log"],
|
||||
help="Search type (default: search)",
|
||||
)
|
||||
parser.add_argument(
|
||||
@@ -542,6 +685,10 @@ def main():
|
||||
cmd_sources(args)
|
||||
return
|
||||
|
||||
if args.command == "log":
|
||||
cmd_log(args)
|
||||
return
|
||||
|
||||
if args.command == "health":
|
||||
asyncio.run(cmd_health(args))
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user