feat: 添加执行日志系统,跟踪各引擎/接口调用耗时和错误信息
新增功能: - execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情 - 所有搜索引擎自动跟踪每次调用的性能指标 - CLI 搜索后自动显示执行摘要 - health 命令使用 ParallelSearch 检测所有引擎连通性 - 新增 log 命令查看历史执行日志
This commit is contained in:
1 parent
f2d8919860
commit
a60ba71663
18 files changed
+1671
-332
No files matched your search
@@ -0,0 +1,232 @@
|
||||
"""
|
||||
执行日志模块 — 记录各引擎/接口的调用耗时、报错信息。
|
||||
|
||||
用于跟踪各个接口的执行效果,便于后续优化。
|
||||
|
||||
Usage:
|
||||
from scripts.core.execution_log import execution_log
|
||||
|
||||
# 自动记录耗时(推荐用法)
|
||||
with execution_log.track("bing", "search") as ctx:
|
||||
results = await engine.search(query)
|
||||
ctx.result_count = len(results)
|
||||
|
||||
# 手动记录
|
||||
execution_log.record(EngineCallLog(
|
||||
engine="bing",
|
||||
operation="search",
|
||||
duration_ms=320,
|
||||
success=True,
|
||||
result_count=5,
|
||||
))
|
||||
|
||||
# 打印摘要
|
||||
execution_log.print_summary()
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineCallLog:
|
||||
"""单次引擎调用记录。"""
|
||||
engine: str # 引擎名(bing, sogou, baidu, ...)
|
||||
operation: str = "search" # 操作类型(search, fetch, health, ...)
|
||||
timestamp: str = "" # 调用时间 ISO 格式
|
||||
start_time: float = 0.0 # 起始时间戳
|
||||
duration_ms: int = 0 # 耗时(毫秒)
|
||||
success: bool = True # 是否成功
|
||||
result_count: int = 0 # 返回结果数
|
||||
error: Optional[str] = None # 错误信息
|
||||
error_type: Optional[str] = None # 错误类型(异常类名)
|
||||
status_code: Optional[int] = None # HTTP 状态码(如有)
|
||||
detail: dict[str, Any] = field(default_factory=dict) # 额外信息
|
||||
|
||||
|
||||
class _ExecutionLogCollector:
|
||||
"""全局执行日志收集器。"""
|
||||
|
||||
def __init__(self):
|
||||
self._logs: list[EngineCallLog] = []
|
||||
self._session_start = time.time()
|
||||
|
||||
def record(self, log: EngineCallLog):
|
||||
"""记录一次引擎调用。"""
|
||||
self._logs.append(log)
|
||||
|
||||
# 同时写 logger
|
||||
status = "✓" if log.success else "✗"
|
||||
msg = (
|
||||
f"{status} [{log.engine}] {log.operation} "
|
||||
f"{log.duration_ms}ms"
|
||||
)
|
||||
if log.result_count > 0:
|
||||
msg += f" → {log.result_count} results"
|
||||
if log.error:
|
||||
msg += f" | error: {log.error}"
|
||||
if log.status_code and log.status_code >= 400:
|
||||
msg += f" | HTTP {log.status_code}"
|
||||
|
||||
if log.success:
|
||||
logger.info(msg)
|
||||
else:
|
||||
logger.warning(msg)
|
||||
|
||||
@contextmanager
|
||||
def track(self, engine: str, operation: str = "search",
|
||||
**extra):
|
||||
"""上下文管理器,自动记录耗时和异常。
|
||||
|
||||
Example:
|
||||
with execution_log.track("bing", "search", region="cn") as ctx:
|
||||
results = await engine.search(query)
|
||||
ctx.result_count = len(results)
|
||||
"""
|
||||
entry = EngineCallLog(
|
||||
engine=engine,
|
||||
operation=operation,
|
||||
timestamp=datetime.now().isoformat(),
|
||||
start_time=time.time(),
|
||||
detail=dict(extra),
|
||||
)
|
||||
try:
|
||||
yield entry
|
||||
except Exception as e:
|
||||
entry.success = False
|
||||
entry.error = str(e)
|
||||
entry.error_type = type(e).__name__
|
||||
raise
|
||||
finally:
|
||||
entry.duration_ms = int((time.time() - entry.start_time) * 1000)
|
||||
self.record(entry)
|
||||
|
||||
def get_logs(self, engine: str | None = None) -> list[EngineCallLog]:
|
||||
"""获取日志,可按引擎名过滤。"""
|
||||
if engine:
|
||||
return [l for l in self._logs if l.engine == engine]
|
||||
return list(self._logs)
|
||||
|
||||
def get_stats(self, engine: str | None = None) -> dict[str, Any]:
|
||||
"""获取统计摘要。"""
|
||||
logs = self.get_logs(engine)
|
||||
if not logs:
|
||||
return {"total_calls": 0}
|
||||
|
||||
total = len(logs)
|
||||
success = sum(1 for l in logs if l.success)
|
||||
failed = total - success
|
||||
durations = [l.duration_ms for l in logs]
|
||||
result_counts = [l.result_count for l in logs]
|
||||
|
||||
errors = []
|
||||
for l in logs:
|
||||
if l.error:
|
||||
errors.append({
|
||||
"engine": l.engine,
|
||||
"operation": l.operation,
|
||||
"error_type": l.error_type,
|
||||
"error": l.error[:200],
|
||||
"timestamp": l.timestamp,
|
||||
})
|
||||
|
||||
return {
|
||||
"total_calls": total,
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"success_rate": f"{success/total*100:.1f}%",
|
||||
"avg_duration_ms": sum(durations) // total,
|
||||
"max_duration_ms": max(durations),
|
||||
"min_duration_ms": min(durations),
|
||||
"total_results": sum(result_counts),
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
def print_summary(self):
|
||||
"""打印本次会话的执行摘要。"""
|
||||
if not self._logs:
|
||||
print("暂无执行记录")
|
||||
return
|
||||
|
||||
total_time = int((time.time() - self._session_start) * 1000)
|
||||
stats = self.get_stats()
|
||||
|
||||
lines = [
|
||||
"═══════════════════════════════════════════",
|
||||
" 执行日志摘要",
|
||||
"═══════════════════════════════════════════",
|
||||
f" 总调用次数: {stats['total_calls']}",
|
||||
f" 成功/失败: {stats['success']}/{stats['failed']} ({stats['success_rate']})",
|
||||
f" 平均耗时: {stats['avg_duration_ms']}ms",
|
||||
f" 最长耗时: {stats['max_duration_ms']}ms",
|
||||
f" 最短耗时: {stats['min_duration_ms']}ms",
|
||||
f" 总结果数: {stats['total_results']}",
|
||||
f" 总耗时: {total_time}ms",
|
||||
"───────────────────────────────────────────",
|
||||
]
|
||||
|
||||
# 按引擎分组
|
||||
engine_names = sorted(set(l.engine for l in self._logs))
|
||||
lines.append(" 引擎明细:")
|
||||
for name in engine_names:
|
||||
estat = self.get_stats(name)
|
||||
status = "✓" if estat["failed"] == 0 else "✗"
|
||||
lines.append(
|
||||
f" {status} {name:15s} "
|
||||
f"calls={estat['total_calls']} "
|
||||
f"avg={estat['avg_duration_ms']}ms "
|
||||
f"max={estat['max_duration_ms']}ms "
|
||||
f"results={estat['total_results']}"
|
||||
)
|
||||
|
||||
# 错误详情
|
||||
if stats["errors"]:
|
||||
lines.append("───────────────────────────────────────────")
|
||||
lines.append(" 错误详情:")
|
||||
for err in stats["errors"]:
|
||||
lines.append(
|
||||
f" ✗ [{err['engine']}] {err['error_type']}: {err['error'][:100]}"
|
||||
)
|
||||
|
||||
lines.append("═══════════════════════════════════════════")
|
||||
print("\n".join(lines))
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""导出为 JSON。"""
|
||||
data = {
|
||||
"session_start": datetime.fromtimestamp(self._session_start).isoformat(),
|
||||
"total_calls": len(self._logs),
|
||||
"stats": self.get_stats(),
|
||||
"logs": [asdict(l) for l in self._logs],
|
||||
}
|
||||
return json.dumps(data, ensure_ascii=False, indent=2)
|
||||
|
||||
def save(self, path: str = "logs/search_log.jsonl"):
|
||||
"""追加写入 JSONL 日志文件。"""
|
||||
import os
|
||||
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
||||
with open(path, "a", encoding="utf-8") as f:
|
||||
for log in self._logs:
|
||||
record = asdict(log)
|
||||
record["session_start"] = datetime.fromtimestamp(
|
||||
self._session_start
|
||||
).isoformat()
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
def clear(self):
|
||||
"""清空日志。"""
|
||||
self._logs.clear()
|
||||
self._session_start = time.time()
|
||||
|
||||
|
||||
# 全局单例
|
||||
execution_log = _ExecutionLogCollector()
|
||||
Reference in new issue
Block a user