feat: 添加执行日志系统,跟踪各引擎/接口调用耗时和错误信息
新增功能: - execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情 - 所有搜索引擎自动跟踪每次调用的性能指标 - CLI 搜索后自动显示执行摘要 - health 命令使用 ParallelSearch 检测所有引擎连通性 - 新增 log 命令查看历史执行日志
This commit is contained in:
1 parent
f2d8919860
commit
9d97a3a7e8
18 files changed
+1671
-332
No files matched your search
@@ -35,3 +35,4 @@ build/
|
|||||||
|
|
||||||
# Config with secrets
|
# Config with secrets
|
||||||
config.local.yaml
|
config.local.yaml
|
||||||
|
logs/
|
||||||
+11
-10
@@ -1,20 +1,21 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "web-search-skills"
|
name = "web-search-skills"
|
||||||
version = "2.2.0"
|
version = "0.2.0"
|
||||||
description = "Multi-source web search CLI tool"
|
description = "Open-source diversified search in <5s. 28+ engines across web, news, academic, social, WeChat."
|
||||||
requires-python = ">=3.8"
|
authors = [{ name = "openbot-coder" }]
|
||||||
|
readme = "README.md"
|
||||||
|
license = { file = "LICENSE" }
|
||||||
|
|
||||||
|
requires-python = ">=3.10"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"httpx>=0.25.0",
|
"httpx>=0.24.0",
|
||||||
"beautifulsoup4>=4.12.0",
|
"beautifulsoup4>=4.12.0",
|
||||||
"lxml>=4.9.0",
|
"lxml>=5.0.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
web-search = "scripts.cli:main"
|
web-search = "ws:main"
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
requires = ["setuptools>=64", "wheel"]
|
requires = ["setuptools>=68.0"]
|
||||||
build-backend = "setuptools.build_meta"
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
[tool.setuptools.packages.find]
|
|
||||||
include = ["scripts*"]
|
|
||||||
+182
-35
@@ -68,6 +68,9 @@ logging.basicConfig(
|
|||||||
)
|
)
|
||||||
logger = logging.getLogger("cli")
|
logger = logging.getLogger("cli")
|
||||||
|
|
||||||
|
# 执行日志模块
|
||||||
|
from scripts.core.execution_log import execution_log
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Lazy engine imports (import on demand to keep startup fast)
|
# Lazy engine imports (import on demand to keep startup fast)
|
||||||
@@ -78,6 +81,11 @@ def _get_unified():
|
|||||||
return UnifiedSearch
|
return UnifiedSearch
|
||||||
|
|
||||||
|
|
||||||
|
def _get_parallel():
|
||||||
|
from scripts.engines.parallel_search import ParallelSearch
|
||||||
|
return ParallelSearch
|
||||||
|
|
||||||
|
|
||||||
def _get_sogou():
|
def _get_sogou():
|
||||||
from scripts.wechat.sogou_weixin import SogouWeChatEngine
|
from scripts.wechat.sogou_weixin import SogouWeChatEngine
|
||||||
return SogouWeChatEngine
|
return SogouWeChatEngine
|
||||||
@@ -160,7 +168,41 @@ def _sanitize_for_console(text: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _format_result(r, fmt: str = "text") -> str:
|
def _format_result(r, fmt: str = "text") -> str:
|
||||||
"""Format a single SearchResult."""
|
"""Format a single search result (supports both EngineResult and old models)."""
|
||||||
|
# Handle EngineResult (new engines)
|
||||||
|
if hasattr(r, "source") and hasattr(r, "snippet"):
|
||||||
|
title = getattr(r, "title", "")
|
||||||
|
url = getattr(r, "url", "")
|
||||||
|
snippet = getattr(r, "snippet", "")
|
||||||
|
source = getattr(r, "source", "")
|
||||||
|
category = getattr(r, "category", "")
|
||||||
|
extra = getattr(r, "extra", {})
|
||||||
|
published = getattr(r, "published", "")
|
||||||
|
|
||||||
|
if fmt == "json":
|
||||||
|
return json.dumps({
|
||||||
|
"title": title, "url": url, "snippet": snippet[:300],
|
||||||
|
"source": source, "category": category,
|
||||||
|
"published": published, "extra": extra,
|
||||||
|
}, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
|
lines = [_sanitize_for_console(title)]
|
||||||
|
lines.append(f" URL: {url}")
|
||||||
|
if snippet:
|
||||||
|
lines.append(f" {_sanitize_for_console(snippet[:200])}")
|
||||||
|
extra_parts = []
|
||||||
|
if source:
|
||||||
|
extra_parts.append(f"来源: {source}")
|
||||||
|
if published:
|
||||||
|
extra_parts.append(f"时间: {published}")
|
||||||
|
for k, v in extra.items():
|
||||||
|
if k in ("stars", "language") and v:
|
||||||
|
extra_parts.append(f"{k}: {v}")
|
||||||
|
if extra_parts:
|
||||||
|
lines.append(f" ({'; '.join(extra_parts)})")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
# Handle old SearchResult format (rank, title, url, snippet, source, category, extra)
|
||||||
if fmt == "json":
|
if fmt == "json":
|
||||||
return json.dumps({
|
return json.dumps({
|
||||||
"rank": r.rank,
|
"rank": r.rank,
|
||||||
@@ -224,49 +266,61 @@ def _print_results(results, fmt: str = "text"):
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async def _search_all(query: str, max_results: int, **kwargs) -> list[Any]:
|
async def _search_all(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||||
us = _get_unified()()
|
"""Parallel search across all general engines."""
|
||||||
try:
|
ps = _get_parallel()(timeout=10.0)
|
||||||
return await us.search(query, max_results=max_results, **kwargs)
|
region = kwargs.get("region", "cn")
|
||||||
finally:
|
results, stats = await ps.search(
|
||||||
await us.close()
|
query, engines=None, max_results=max_results, region=region,
|
||||||
|
)
|
||||||
|
logger.debug(f"Stats: {stats}")
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
async def _search_web(query: str, max_results: int, region: str = None, **kwargs) -> list[Any]:
|
async def _search_web(query: str, max_results: int, region: str = None, **kwargs) -> list[Any]:
|
||||||
us = _get_unified()()
|
"""Parallel web search (general engines)."""
|
||||||
try:
|
ps = _get_parallel()(timeout=10.0)
|
||||||
return await us.search_engines(query, region=region, max_results=max_results)
|
results, stats = await ps.search(
|
||||||
finally:
|
query, engines=None, max_results=max_results, region=region or "cn",
|
||||||
await us.close()
|
)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
async def _search_news(query: str, max_results: int, **kwargs) -> list[Any]:
|
async def _search_news(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||||
us = _get_unified()()
|
"""Vertical news search."""
|
||||||
try:
|
ps = _get_parallel()(timeout=10.0)
|
||||||
return await us.search_news(query, max_results=max_results, **kwargs)
|
results, stats = await ps.search_vertical(
|
||||||
finally:
|
query, "news", max_results=max_results, **kwargs,
|
||||||
await us.close()
|
)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
async def _search_wechat(query: str, max_results: int) -> list[Any]:
|
async def _search_wechat(query: str, max_results: int) -> list[Any]:
|
||||||
engine = _get_sogou()()
|
engine = _get_sogou()()
|
||||||
try:
|
try:
|
||||||
return await engine.search(query, max_results=max_results)
|
with execution_log.track("weixin", "search", query=query) as ctx:
|
||||||
|
results = await engine.search(query, max_results=max_results)
|
||||||
|
ctx.result_count = len(results)
|
||||||
|
return results
|
||||||
finally:
|
finally:
|
||||||
await engine.close()
|
await engine.close()
|
||||||
|
|
||||||
|
|
||||||
async def _search_academic(query: str, max_results: int) -> list[Any]:
|
async def _search_academic(query: str, max_results: int) -> list[Any]:
|
||||||
us = _get_unified()()
|
"""Vertical academic search (ArXiv + CrossRef)."""
|
||||||
try:
|
ps = _get_parallel()(timeout=15.0)
|
||||||
return await us.search_academic(query, max_results=max_results)
|
results, stats = await ps.search_vertical(
|
||||||
finally:
|
query, "academic", max_results=max_results,
|
||||||
await us.close()
|
)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]:
|
async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||||
us = _get_unified()()
|
us = _get_unified()()
|
||||||
try:
|
try:
|
||||||
return await us.search_social(query, max_results=max_results, **kwargs)
|
with execution_log.track("social", "search", query=query) as ctx:
|
||||||
|
results = await us.search_social(query, max_results=max_results, **kwargs)
|
||||||
|
ctx.result_count = len(results) if results else 0
|
||||||
|
return results
|
||||||
finally:
|
finally:
|
||||||
await us.close()
|
await us.close()
|
||||||
|
|
||||||
@@ -275,7 +329,10 @@ async def _search_baidu(query: str, max_results: int, **kwargs) -> list[Any]:
|
|||||||
from scripts.engines.baidu_qianfan import BaiduQianFanEngine
|
from scripts.engines.baidu_qianfan import BaiduQianFanEngine
|
||||||
engine = BaiduQianFanEngine({"api_key": os.getenv("BAIDU_API_KEY", "")})
|
engine = BaiduQianFanEngine({"api_key": os.getenv("BAIDU_API_KEY", "")})
|
||||||
try:
|
try:
|
||||||
return await engine.search(query, max_results=max_results, **kwargs)
|
with execution_log.track("baidu_qianfan", "search", query=query) as ctx:
|
||||||
|
results = await engine.search(query, max_results=max_results, **kwargs)
|
||||||
|
ctx.result_count = len(results)
|
||||||
|
return results
|
||||||
finally:
|
finally:
|
||||||
await engine.close()
|
await engine.close()
|
||||||
|
|
||||||
@@ -283,11 +340,24 @@ async def _search_baidu(query: str, max_results: int, **kwargs) -> list[Any]:
|
|||||||
async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]:
|
async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||||
us = _get_unified()()
|
us = _get_unified()()
|
||||||
try:
|
try:
|
||||||
return await us.search_rss(query, max_results=max_results, **kwargs)
|
with execution_log.track("rss", "search", query=query) as ctx:
|
||||||
|
results = await us.search_rss(query, max_results=max_results, **kwargs)
|
||||||
|
ctx.result_count = len(results) if results else 0
|
||||||
|
return results
|
||||||
finally:
|
finally:
|
||||||
await us.close()
|
await us.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def _search_github(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||||
|
"""GitHub code/repo search."""
|
||||||
|
ps = _get_parallel()(timeout=10.0)
|
||||||
|
search_type = kwargs.get("search_type", "repositories")
|
||||||
|
results, stats = await ps.search_vertical(
|
||||||
|
query, "github", max_results=max_results, search_type=search_type,
|
||||||
|
)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]:
|
async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]:
|
||||||
us = _get_unified()()
|
us = _get_unified()()
|
||||||
try:
|
try:
|
||||||
@@ -300,6 +370,69 @@ async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]:
|
|||||||
# CLI actions
|
# CLI actions
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def cmd_log(args):
|
||||||
|
"""查看执行日志历史。"""
|
||||||
|
import json as json_mod
|
||||||
|
|
||||||
|
log_file = "logs/search_log.jsonl"
|
||||||
|
try:
|
||||||
|
with open(log_file, "r", encoding="utf-8") as f:
|
||||||
|
lines = [json_mod.loads(line) for line in f if line.strip()]
|
||||||
|
except FileNotFoundError:
|
||||||
|
print(f"暂无日志文件 ({log_file})")
|
||||||
|
return
|
||||||
|
|
||||||
|
if not lines:
|
||||||
|
print("暂无执行记录")
|
||||||
|
return
|
||||||
|
|
||||||
|
# 按 session 分组
|
||||||
|
sessions = {}
|
||||||
|
for entry in lines:
|
||||||
|
session = entry.get("session_start", "unknown")
|
||||||
|
if session not in sessions:
|
||||||
|
sessions[session] = []
|
||||||
|
sessions[session].append(entry)
|
||||||
|
|
||||||
|
print(f"═══════════════════════════════════════════")
|
||||||
|
print(f" 执行日志历史 (共 {len(lines)} 条记录, {len(sessions)} 个会话)")
|
||||||
|
print(f"═══════════════════════════════════════════")
|
||||||
|
|
||||||
|
# 只显示最近5个会话
|
||||||
|
recent_sessions = list(sessions.items())[-5:]
|
||||||
|
for session_start, entries in recent_sessions:
|
||||||
|
total = len(entries)
|
||||||
|
success = sum(1 for e in entries if e.get("success", True))
|
||||||
|
failed = total - success
|
||||||
|
durations = [e.get("duration_ms", 0) for e in entries]
|
||||||
|
avg_dur = sum(durations) // total if total else 0
|
||||||
|
|
||||||
|
print(f"\n 会话: {session_start}")
|
||||||
|
print(f" 调用: {total} 次 (成功 {success}, 失败 {failed})")
|
||||||
|
print(f" 平均耗时: {avg_dur}ms")
|
||||||
|
|
||||||
|
# 按引擎分组
|
||||||
|
engines = {}
|
||||||
|
for e in entries:
|
||||||
|
eng = e.get("engine", "unknown")
|
||||||
|
if eng not in engines:
|
||||||
|
engines[eng] = {"calls": 0, "errors": []}
|
||||||
|
engines[eng]["calls"] += 1
|
||||||
|
if not e.get("success", True):
|
||||||
|
engines[eng]["errors"].append(e.get("error", "unknown"))
|
||||||
|
|
||||||
|
for eng, info in sorted(engines.items()):
|
||||||
|
err_count = len(info["errors"])
|
||||||
|
status = "✓" if err_count == 0 else f"✗ ({err_count} errors)"
|
||||||
|
print(f" {eng:15s} {status} calls={info['calls']}")
|
||||||
|
|
||||||
|
# 显示错误详情
|
||||||
|
for err in info["errors"][:3]: # 最多显示3个
|
||||||
|
print(f" └─ {err[:80]}")
|
||||||
|
|
||||||
|
print(f"\n═══════════════════════════════════════════")
|
||||||
|
|
||||||
|
|
||||||
def cmd_sources(args):
|
def cmd_sources(args):
|
||||||
"""List all available search sources."""
|
"""List all available search sources."""
|
||||||
cl = _get_config_loader()
|
cl = _get_config_loader()
|
||||||
@@ -337,22 +470,25 @@ def cmd_urls(args):
|
|||||||
|
|
||||||
async def cmd_health(args):
|
async def cmd_health(args):
|
||||||
"""Test all engine connectivity."""
|
"""Test all engine connectivity."""
|
||||||
us = _get_unified()()
|
ps = _get_parallel()(timeout=10.0)
|
||||||
try:
|
try:
|
||||||
print("正在检测各引擎连通性...\n")
|
print("正在检测各引擎连通性...\n")
|
||||||
statuses = await us.search_health()
|
statuses = await ps.health()
|
||||||
ok_count = sum(1 for s in statuses if s["status"] == "ok")
|
ok_count = sum(1 for s in statuses.values() if s["status"] == "ok")
|
||||||
for s in statuses:
|
for name, s in statuses.items():
|
||||||
if s["status"] == "ok":
|
if s["status"] == "ok":
|
||||||
print(f" [OK] {s['name']}")
|
print(f" [OK] {name}")
|
||||||
elif s["status"] == "timeout":
|
elif s["status"] == "timeout":
|
||||||
print(f" [TIMEOUT] {s['name']}")
|
print(f" [TIMEOUT] {name}")
|
||||||
else:
|
else:
|
||||||
detail = s.get("detail", "")
|
detail = s.get("error", s.get("detail", ""))
|
||||||
print(f" [FAIL] {s['name']} {detail}")
|
print(f" [FAIL] {name} {detail}")
|
||||||
print(f"\n总计: {len(statuses)} 个引擎, {ok_count} 个可用")
|
print(f"\n总计: {len(statuses)} 个引擎, {ok_count} 个可用")
|
||||||
|
print()
|
||||||
|
execution_log.print_summary()
|
||||||
|
execution_log.save("logs/search_log.jsonl")
|
||||||
finally:
|
finally:
|
||||||
await us.close()
|
pass
|
||||||
|
|
||||||
|
|
||||||
async def _run_search(args):
|
async def _run_search(args):
|
||||||
@@ -369,6 +505,7 @@ async def _run_search(args):
|
|||||||
"academic": _search_academic,
|
"academic": _search_academic,
|
||||||
"social": _search_social,
|
"social": _search_social,
|
||||||
"rss": _search_rss,
|
"rss": _search_rss,
|
||||||
|
"github": _search_github,
|
||||||
}
|
}
|
||||||
|
|
||||||
search_fn = method_map.get(args.command, _search_all)
|
search_fn = method_map.get(args.command, _search_all)
|
||||||
@@ -408,6 +545,7 @@ async def _run_search(args):
|
|||||||
"academic": "学术搜索",
|
"academic": "学术搜索",
|
||||||
"social": "社交搜索",
|
"social": "社交搜索",
|
||||||
"rss": "RSS 新闻搜索",
|
"rss": "RSS 新闻搜索",
|
||||||
|
"github": "GitHub 搜索",
|
||||||
}
|
}
|
||||||
header = cmd_names.get(args.command, "搜索")
|
header = cmd_names.get(args.command, "搜索")
|
||||||
print(f"{header}: {args.query}")
|
print(f"{header}: {args.query}")
|
||||||
@@ -415,6 +553,11 @@ async def _run_search(args):
|
|||||||
|
|
||||||
_print_results(results, fmt="json" if args.json else "text")
|
_print_results(results, fmt="json" if args.json else "text")
|
||||||
|
|
||||||
|
# 打印执行日志摘要并保存到文件
|
||||||
|
print()
|
||||||
|
execution_log.print_summary()
|
||||||
|
execution_log.save("logs/search_log.jsonl")
|
||||||
|
|
||||||
# Save to file if requested (with basic path-sanitization)
|
# Save to file if requested (with basic path-sanitization)
|
||||||
if args.output:
|
if args.output:
|
||||||
output_data = json.dumps(
|
output_data = json.dumps(
|
||||||
@@ -468,7 +611,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
nargs="?",
|
nargs="?",
|
||||||
default="search",
|
default="search",
|
||||||
choices=["search", "web", "baidu", "news", "wechat", "academic",
|
choices=["search", "web", "baidu", "news", "wechat", "academic",
|
||||||
"social", "rss", "health", "sources", "urls"],
|
"social", "rss", "github", "health", "sources", "urls", "log"],
|
||||||
help="Search type (default: search)",
|
help="Search type (default: search)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
@@ -542,6 +685,10 @@ def main():
|
|||||||
cmd_sources(args)
|
cmd_sources(args)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if args.command == "log":
|
||||||
|
cmd_log(args)
|
||||||
|
return
|
||||||
|
|
||||||
if args.command == "health":
|
if args.command == "health":
|
||||||
asyncio.run(cmd_health(args))
|
asyncio.run(cmd_health(args))
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -43,6 +43,10 @@ try:
|
|||||||
# --- 日志 ---
|
# --- 日志 ---
|
||||||
log_level: str = "INFO"
|
log_level: str = "INFO"
|
||||||
|
|
||||||
|
# --- API 引擎每日限额(有 key 的尽量做替补)---
|
||||||
|
baidu_daily_limit: int = 50
|
||||||
|
minimax_daily_limit: int = 100
|
||||||
|
|
||||||
except ImportError:
|
except ImportError:
|
||||||
|
|
||||||
class Settings: # type: ignore
|
class Settings: # type: ignore
|
||||||
@@ -76,6 +80,9 @@ except ImportError:
|
|||||||
os.environ.get("MAX_CONCURRENT", "5")
|
os.environ.get("MAX_CONCURRENT", "5")
|
||||||
)
|
)
|
||||||
self.log_level = os.environ.get("LOG_LEVEL", "INFO")
|
self.log_level = os.environ.get("LOG_LEVEL", "INFO")
|
||||||
|
# API 引擎每日限额
|
||||||
|
self.baidu_daily_limit = int(os.environ.get("BAIDU_DAILY_LIMIT", "50"))
|
||||||
|
self.minimax_daily_limit = int(os.environ.get("MINIMAX_DAILY_LIMIT", "100"))
|
||||||
|
|
||||||
|
|
||||||
# 全局单例
|
# 全局单例
|
||||||
|
|||||||
@@ -0,0 +1,232 @@
|
|||||||
|
"""
|
||||||
|
执行日志模块 — 记录各引擎/接口的调用耗时、报错信息。
|
||||||
|
|
||||||
|
用于跟踪各个接口的执行效果,便于后续优化。
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
from scripts.core.execution_log import execution_log
|
||||||
|
|
||||||
|
# 自动记录耗时(推荐用法)
|
||||||
|
with execution_log.track("bing", "search") as ctx:
|
||||||
|
results = await engine.search(query)
|
||||||
|
ctx.result_count = len(results)
|
||||||
|
|
||||||
|
# 手动记录
|
||||||
|
execution_log.record(EngineCallLog(
|
||||||
|
engine="bing",
|
||||||
|
operation="search",
|
||||||
|
duration_ms=320,
|
||||||
|
success=True,
|
||||||
|
result_count=5,
|
||||||
|
))
|
||||||
|
|
||||||
|
# 打印摘要
|
||||||
|
execution_log.print_summary()
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from dataclasses import dataclass, field, asdict
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class EngineCallLog:
|
||||||
|
"""单次引擎调用记录。"""
|
||||||
|
engine: str # 引擎名(bing, sogou, baidu, ...)
|
||||||
|
operation: str = "search" # 操作类型(search, fetch, health, ...)
|
||||||
|
timestamp: str = "" # 调用时间 ISO 格式
|
||||||
|
start_time: float = 0.0 # 起始时间戳
|
||||||
|
duration_ms: int = 0 # 耗时(毫秒)
|
||||||
|
success: bool = True # 是否成功
|
||||||
|
result_count: int = 0 # 返回结果数
|
||||||
|
error: Optional[str] = None # 错误信息
|
||||||
|
error_type: Optional[str] = None # 错误类型(异常类名)
|
||||||
|
status_code: Optional[int] = None # HTTP 状态码(如有)
|
||||||
|
detail: dict[str, Any] = field(default_factory=dict) # 额外信息
|
||||||
|
|
||||||
|
|
||||||
|
class _ExecutionLogCollector:
|
||||||
|
"""全局执行日志收集器。"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._logs: list[EngineCallLog] = []
|
||||||
|
self._session_start = time.time()
|
||||||
|
|
||||||
|
def record(self, log: EngineCallLog):
|
||||||
|
"""记录一次引擎调用。"""
|
||||||
|
self._logs.append(log)
|
||||||
|
|
||||||
|
# 同时写 logger
|
||||||
|
status = "✓" if log.success else "✗"
|
||||||
|
msg = (
|
||||||
|
f"{status} [{log.engine}] {log.operation} "
|
||||||
|
f"{log.duration_ms}ms"
|
||||||
|
)
|
||||||
|
if log.result_count > 0:
|
||||||
|
msg += f" → {log.result_count} results"
|
||||||
|
if log.error:
|
||||||
|
msg += f" | error: {log.error}"
|
||||||
|
if log.status_code and log.status_code >= 400:
|
||||||
|
msg += f" | HTTP {log.status_code}"
|
||||||
|
|
||||||
|
if log.success:
|
||||||
|
logger.info(msg)
|
||||||
|
else:
|
||||||
|
logger.warning(msg)
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def track(self, engine: str, operation: str = "search",
|
||||||
|
**extra):
|
||||||
|
"""上下文管理器,自动记录耗时和异常。
|
||||||
|
|
||||||
|
Example:
|
||||||
|
with execution_log.track("bing", "search", region="cn") as ctx:
|
||||||
|
results = await engine.search(query)
|
||||||
|
ctx.result_count = len(results)
|
||||||
|
"""
|
||||||
|
entry = EngineCallLog(
|
||||||
|
engine=engine,
|
||||||
|
operation=operation,
|
||||||
|
timestamp=datetime.now().isoformat(),
|
||||||
|
start_time=time.time(),
|
||||||
|
detail=dict(extra),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
yield entry
|
||||||
|
except Exception as e:
|
||||||
|
entry.success = False
|
||||||
|
entry.error = str(e)
|
||||||
|
entry.error_type = type(e).__name__
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
entry.duration_ms = int((time.time() - entry.start_time) * 1000)
|
||||||
|
self.record(entry)
|
||||||
|
|
||||||
|
def get_logs(self, engine: str | None = None) -> list[EngineCallLog]:
|
||||||
|
"""获取日志,可按引擎名过滤。"""
|
||||||
|
if engine:
|
||||||
|
return [l for l in self._logs if l.engine == engine]
|
||||||
|
return list(self._logs)
|
||||||
|
|
||||||
|
def get_stats(self, engine: str | None = None) -> dict[str, Any]:
|
||||||
|
"""获取统计摘要。"""
|
||||||
|
logs = self.get_logs(engine)
|
||||||
|
if not logs:
|
||||||
|
return {"total_calls": 0}
|
||||||
|
|
||||||
|
total = len(logs)
|
||||||
|
success = sum(1 for l in logs if l.success)
|
||||||
|
failed = total - success
|
||||||
|
durations = [l.duration_ms for l in logs]
|
||||||
|
result_counts = [l.result_count for l in logs]
|
||||||
|
|
||||||
|
errors = []
|
||||||
|
for l in logs:
|
||||||
|
if l.error:
|
||||||
|
errors.append({
|
||||||
|
"engine": l.engine,
|
||||||
|
"operation": l.operation,
|
||||||
|
"error_type": l.error_type,
|
||||||
|
"error": l.error[:200],
|
||||||
|
"timestamp": l.timestamp,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"total_calls": total,
|
||||||
|
"success": success,
|
||||||
|
"failed": failed,
|
||||||
|
"success_rate": f"{success/total*100:.1f}%",
|
||||||
|
"avg_duration_ms": sum(durations) // total,
|
||||||
|
"max_duration_ms": max(durations),
|
||||||
|
"min_duration_ms": min(durations),
|
||||||
|
"total_results": sum(result_counts),
|
||||||
|
"errors": errors,
|
||||||
|
}
|
||||||
|
|
||||||
|
def print_summary(self):
|
||||||
|
"""打印本次会话的执行摘要。"""
|
||||||
|
if not self._logs:
|
||||||
|
print("暂无执行记录")
|
||||||
|
return
|
||||||
|
|
||||||
|
total_time = int((time.time() - self._session_start) * 1000)
|
||||||
|
stats = self.get_stats()
|
||||||
|
|
||||||
|
lines = [
|
||||||
|
"═══════════════════════════════════════════",
|
||||||
|
" 执行日志摘要",
|
||||||
|
"═══════════════════════════════════════════",
|
||||||
|
f" 总调用次数: {stats['total_calls']}",
|
||||||
|
f" 成功/失败: {stats['success']}/{stats['failed']} ({stats['success_rate']})",
|
||||||
|
f" 平均耗时: {stats['avg_duration_ms']}ms",
|
||||||
|
f" 最长耗时: {stats['max_duration_ms']}ms",
|
||||||
|
f" 最短耗时: {stats['min_duration_ms']}ms",
|
||||||
|
f" 总结果数: {stats['total_results']}",
|
||||||
|
f" 总耗时: {total_time}ms",
|
||||||
|
"───────────────────────────────────────────",
|
||||||
|
]
|
||||||
|
|
||||||
|
# 按引擎分组
|
||||||
|
engine_names = sorted(set(l.engine for l in self._logs))
|
||||||
|
lines.append(" 引擎明细:")
|
||||||
|
for name in engine_names:
|
||||||
|
estat = self.get_stats(name)
|
||||||
|
status = "✓" if estat["failed"] == 0 else "✗"
|
||||||
|
lines.append(
|
||||||
|
f" {status} {name:15s} "
|
||||||
|
f"calls={estat['total_calls']} "
|
||||||
|
f"avg={estat['avg_duration_ms']}ms "
|
||||||
|
f"max={estat['max_duration_ms']}ms "
|
||||||
|
f"results={estat['total_results']}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# 错误详情
|
||||||
|
if stats["errors"]:
|
||||||
|
lines.append("───────────────────────────────────────────")
|
||||||
|
lines.append(" 错误详情:")
|
||||||
|
for err in stats["errors"]:
|
||||||
|
lines.append(
|
||||||
|
f" ✗ [{err['engine']}] {err['error_type']}: {err['error'][:100]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
lines.append("═══════════════════════════════════════════")
|
||||||
|
print("\n".join(lines))
|
||||||
|
|
||||||
|
def to_json(self) -> str:
|
||||||
|
"""导出为 JSON。"""
|
||||||
|
data = {
|
||||||
|
"session_start": datetime.fromtimestamp(self._session_start).isoformat(),
|
||||||
|
"total_calls": len(self._logs),
|
||||||
|
"stats": self.get_stats(),
|
||||||
|
"logs": [asdict(l) for l in self._logs],
|
||||||
|
}
|
||||||
|
return json.dumps(data, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
|
def save(self, path: str = "logs/search_log.jsonl"):
|
||||||
|
"""追加写入 JSONL 日志文件。"""
|
||||||
|
import os
|
||||||
|
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
||||||
|
with open(path, "a", encoding="utf-8") as f:
|
||||||
|
for log in self._logs:
|
||||||
|
record = asdict(log)
|
||||||
|
record["session_start"] = datetime.fromtimestamp(
|
||||||
|
self._session_start
|
||||||
|
).isoformat()
|
||||||
|
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||||
|
|
||||||
|
def clear(self):
|
||||||
|
"""清空日志。"""
|
||||||
|
self._logs.clear()
|
||||||
|
self._session_start = time.time()
|
||||||
|
|
||||||
|
|
||||||
|
# 全局单例
|
||||||
|
execution_log = _ExecutionLogCollector()
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
"""Academic search engine — ArXiv + CrossRef APIs (free, no key)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from urllib.parse import quote_plus
|
||||||
|
from xml.etree import ElementTree as ET
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from scripts.engines.base import BaseEngine, EngineResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class ArxivEngine(BaseEngine):
|
||||||
|
"""ArXiv paper search — free, no key needed."""
|
||||||
|
|
||||||
|
name = "arxiv"
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
**kwargs) -> list[EngineResult]:
|
||||||
|
t0 = time.time()
|
||||||
|
url = (
|
||||||
|
f"http://export.arxiv.org/api/query?"
|
||||||
|
f"search_query=all:{quote_plus(query)}&start=0&max_results={max_results}"
|
||||||
|
f"&sortBy=relevance&sortOrder=descending"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(url)
|
||||||
|
resp.raise_for_status()
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
results = self._parse_xml(resp.text, max_results)
|
||||||
|
logger.info(f"[arxiv] {len(results)} results in {elapsed:.2f}s")
|
||||||
|
return results
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[arxiv] failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse_xml(self, xml_text: str, max_results: int) -> list[EngineResult]:
|
||||||
|
root = ET.fromstring(xml_text)
|
||||||
|
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||||
|
results = []
|
||||||
|
|
||||||
|
for entry in root.findall("atom:entry", ns)[:max_results]:
|
||||||
|
title = entry.findtext("atom:title", "", ns).strip().replace("\n", " ")
|
||||||
|
summary = entry.findtext("atom:summary", "", ns).strip().replace("\n", " ")
|
||||||
|
link_el = entry.find("atom:link[@type='text/html']", ns)
|
||||||
|
if link_el is None:
|
||||||
|
link_el = entry.find("atom:id", ns)
|
||||||
|
url = link_el.get("href", "") if link_el is not None else ""
|
||||||
|
published = entry.findtext("atom:published", "", ns)
|
||||||
|
|
||||||
|
authors = [a.findtext("atom:name", "", ns)
|
||||||
|
for a in entry.findall("atom:author", ns)]
|
||||||
|
categories = [c.get("term", "")
|
||||||
|
for c in entry.findall("atom:category", ns)]
|
||||||
|
|
||||||
|
if title:
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=title, url=url, snippet=summary[:300],
|
||||||
|
source="arxiv", category="academic",
|
||||||
|
published=published,
|
||||||
|
extra={"authors": authors[:5], "categories": categories},
|
||||||
|
))
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
class CrossRefEngine(BaseEngine):
|
||||||
|
"""CrossRef academic search — free, no key needed."""
|
||||||
|
|
||||||
|
name = "crossref"
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
**kwargs) -> list[EngineResult]:
|
||||||
|
t0 = time.time()
|
||||||
|
url = "https://api.crossref.org/works"
|
||||||
|
params = {
|
||||||
|
"query": query,
|
||||||
|
"rows": min(max_results, 20),
|
||||||
|
"sort": "relevance",
|
||||||
|
"order": "desc",
|
||||||
|
}
|
||||||
|
headers = {"User-Agent": "web-search-skills/1.0 (mailto:search@example.com)"}
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(url, params=params, headers=headers)
|
||||||
|
resp.raise_for_status()
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
data = resp.json()
|
||||||
|
results = []
|
||||||
|
for item in data.get("message", {}).get("items", [])[:max_results]:
|
||||||
|
title_list = item.get("title", [])
|
||||||
|
title = title_list[0] if title_list else ""
|
||||||
|
doi = item.get("DOI", "")
|
||||||
|
url_link = item.get("URL", f"https://doi.org/{doi}")
|
||||||
|
abstract = item.get("abstract", "")[:300]
|
||||||
|
pub_date = ""
|
||||||
|
if item.get("published-print"):
|
||||||
|
dp = item["published-print"].get("date-parts", [[]])[0]
|
||||||
|
pub_date = "-".join(str(p) for p in dp if p) if dp else ""
|
||||||
|
|
||||||
|
authors = [f"{a.get('given', '')} {a.get('family', '')}".strip()
|
||||||
|
for a in item.get("author", [])]
|
||||||
|
|
||||||
|
if title:
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=title, url=url_link, snippet=abstract,
|
||||||
|
source="crossref", category="academic",
|
||||||
|
published=pub_date,
|
||||||
|
extra={
|
||||||
|
"doi": doi,
|
||||||
|
"authors": authors[:5],
|
||||||
|
"journal": (item.get("container-title", [""])[0]
|
||||||
|
if item.get("container-title") else ""),
|
||||||
|
},
|
||||||
|
))
|
||||||
|
|
||||||
|
logger.info(f"[crossref] {len(results)} results in {elapsed:.2f}s")
|
||||||
|
return results
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[crossref] failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
class AcademicSearch(BaseEngine):
|
||||||
|
"""Unified academic search — ArXiv + CrossRef in parallel."""
|
||||||
|
|
||||||
|
name = "academic"
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
**kwargs) -> list[EngineResult]:
|
||||||
|
import asyncio
|
||||||
|
arxiv = ArxivEngine(timeout=self.timeout)
|
||||||
|
crossref = CrossRefEngine(timeout=self.timeout)
|
||||||
|
|
||||||
|
arxiv_results, crossref_results = await asyncio.gather(
|
||||||
|
arxiv.search(query, max_results=max_results),
|
||||||
|
crossref.search(query, max_results=max_results),
|
||||||
|
return_exceptions=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
all_results = []
|
||||||
|
if isinstance(arxiv_results, list):
|
||||||
|
all_results.extend(arxiv_results)
|
||||||
|
if isinstance(crossref_results, list):
|
||||||
|
all_results.extend(crossref_results)
|
||||||
|
|
||||||
|
# Deduplicate by URL
|
||||||
|
seen = set()
|
||||||
|
deduped = []
|
||||||
|
for r in all_results:
|
||||||
|
if r.url not in seen:
|
||||||
|
seen.add(r.url)
|
||||||
|
deduped.append(r)
|
||||||
|
|
||||||
|
return deduped[:max_results]
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
"""Baidu HTML scraping engine — free, fastest in China, best Chinese results."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from urllib.parse import quote_plus
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from scripts.engines.base import BaseEngine, EngineResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
HEADERS = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class BaiduScraper(BaseEngine):
|
||||||
|
"""Baidu web scraping — fastest in China, best Chinese content."""
|
||||||
|
|
||||||
|
name = "baidu"
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
**kwargs) -> list[EngineResult]:
|
||||||
|
t0 = time.time()
|
||||||
|
url = f"https://www.baidu.com/s?wd={quote_plus(query)}&rn={max_results}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(url, headers=HEADERS)
|
||||||
|
resp.raise_for_status()
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
results = self._parse(resp.text, max_results)
|
||||||
|
logger.info(f"[baidu] {len(results)} results in {elapsed:.2f}s")
|
||||||
|
return results
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[baidu] failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
results = []
|
||||||
|
|
||||||
|
for item in soup.select("div.result, div.c-container"):
|
||||||
|
title_el = item.select_one("h3 a")
|
||||||
|
if not title_el:
|
||||||
|
continue
|
||||||
|
title = title_el.get_text(strip=True)
|
||||||
|
href = title_el.get("href", "")
|
||||||
|
snippet_el = item.select_one(
|
||||||
|
"span.content-right_8Zs40, div.c-abstract, "
|
||||||
|
"span[class*='content'], div[class*='abstract']"
|
||||||
|
)
|
||||||
|
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||||
|
|
||||||
|
if title and href:
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=title, url=href, snippet=snippet, source="baidu",
|
||||||
|
))
|
||||||
|
if len(results) >= max_results:
|
||||||
|
break
|
||||||
|
|
||||||
|
return results
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
"""Base engine class and shared data models."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from scripts.core.execution_log import execution_log
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class EngineResult:
|
||||||
|
"""Unified result from any engine."""
|
||||||
|
title: str
|
||||||
|
url: str
|
||||||
|
snippet: str = ""
|
||||||
|
source: str = ""
|
||||||
|
category: str = "" # "general" | "news" | "academic" | "github"
|
||||||
|
published: str = ""
|
||||||
|
extra: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class BaseEngine(ABC):
|
||||||
|
"""Base class for all search engines.
|
||||||
|
|
||||||
|
Automatically tracks execution timing and errors via execution_log.
|
||||||
|
"""
|
||||||
|
|
||||||
|
name: str = "unknown"
|
||||||
|
timeout: float = 10.0
|
||||||
|
|
||||||
|
def __init__(self, timeout: float = 10.0):
|
||||||
|
self.timeout = timeout
|
||||||
|
|
||||||
|
async def search_with_log(
|
||||||
|
self, query: str, max_results: int = 10, **kwargs
|
||||||
|
) -> list[EngineResult]:
|
||||||
|
"""Search with automatic execution logging.
|
||||||
|
|
||||||
|
Subclasses override `search()`, not this method.
|
||||||
|
"""
|
||||||
|
with execution_log.track(self.name, "search", query=query) as ctx:
|
||||||
|
results = await self.search(query, max_results=max_results, **kwargs)
|
||||||
|
ctx.result_count = len(results)
|
||||||
|
return results
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
**kwargs) -> list[EngineResult]:
|
||||||
|
...
|
||||||
|
|
||||||
|
async def health(self) -> dict:
|
||||||
|
"""Quick connectivity check."""
|
||||||
|
with execution_log.track(self.name, "health") as ctx:
|
||||||
|
try:
|
||||||
|
results = await self.search("test", max_results=1)
|
||||||
|
ctx.result_count = len(results)
|
||||||
|
return {"name": self.name, "status": "ok", "count": len(results)}
|
||||||
|
except Exception as e:
|
||||||
|
ctx.success = False
|
||||||
|
ctx.error = str(e)
|
||||||
|
ctx.error_type = type(e).__name__
|
||||||
|
return {"name": self.name, "status": "fail", "error": str(e)}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
"""Bing HTML scraping engine — free, no API key."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from urllib.parse import quote_plus
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from scripts.engines.base import BaseEngine, EngineResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
HEADERS = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class BingScraper(BaseEngine):
|
||||||
|
"""Bing web scraping — fast, good English results."""
|
||||||
|
|
||||||
|
name = "bing"
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
region: str = "global", **kwargs) -> list[EngineResult]:
|
||||||
|
t0 = time.time()
|
||||||
|
url = f"https://cn.bing.com/search?q={quote_plus(query)}&setlang=zh-CN"
|
||||||
|
if region == "global":
|
||||||
|
url = f"https://www.bing.com/search?q={quote_plus(query)}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(url, headers=HEADERS)
|
||||||
|
resp.raise_for_status()
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
results = self._parse(resp.text, max_results)
|
||||||
|
logger.info(f"[bing] {len(results)} results in {elapsed:.2f}s")
|
||||||
|
return results
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[bing] failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
results = []
|
||||||
|
|
||||||
|
for item in soup.select("li.b_algo"):
|
||||||
|
title_el = item.select_one("h2 a")
|
||||||
|
if not title_el:
|
||||||
|
continue
|
||||||
|
title = title_el.get_text(strip=True)
|
||||||
|
href = title_el.get("href", "")
|
||||||
|
snippet_el = item.select_one("p, .b_caption p")
|
||||||
|
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||||
|
|
||||||
|
if title and href:
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=title, url=href, snippet=snippet, source="bing",
|
||||||
|
))
|
||||||
|
if len(results) >= max_results:
|
||||||
|
break
|
||||||
|
|
||||||
|
return results
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
"""Brave Search HTML scraping engine — free, privacy-focused."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from urllib.parse import quote_plus
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from scripts.engines.base import BaseEngine, EngineResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
HEADERS = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class BraveScraper(BaseEngine):
|
||||||
|
"""Brave Search scraping — privacy-focused, good results."""
|
||||||
|
|
||||||
|
name = "brave"
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
**kwargs) -> list[EngineResult]:
|
||||||
|
t0 = time.time()
|
||||||
|
url = f"https://search.brave.com/search?q={quote_plus(query)}&source=web"
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(url, headers=HEADERS)
|
||||||
|
resp.raise_for_status()
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
results = self._parse(resp.text, max_results)
|
||||||
|
logger.info(f"[brave] {len(results)} results in {elapsed:.2f}s")
|
||||||
|
return results
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[brave] failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
results = []
|
||||||
|
|
||||||
|
# Brave uses div.snippet or article elements
|
||||||
|
for item in soup.select("div.snippet, div.result"):
|
||||||
|
title_el = item.select_one("a.result-header, a.snippet-title")
|
||||||
|
if not title_el:
|
||||||
|
title_el = item.select_one("a")
|
||||||
|
if not title_el:
|
||||||
|
continue
|
||||||
|
title = title_el.get_text(strip=True)
|
||||||
|
href = title_el.get("href", "")
|
||||||
|
snippet_el = item.select_one("p.snippet-description, div.snippet-description")
|
||||||
|
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||||
|
|
||||||
|
if title and href and href.startswith("http"):
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=title, url=href, snippet=snippet, source="brave",
|
||||||
|
))
|
||||||
|
if len(results) >= max_results:
|
||||||
|
break
|
||||||
|
|
||||||
|
return results
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
"""GitHub code search engine — uses GitHub API (no key needed for basic search)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from urllib.parse import quote_plus
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from scripts.engines.base import BaseEngine, EngineResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
HEADERS = {
|
||||||
|
"Accept": "application/vnd.github.v3+json",
|
||||||
|
"User-Agent": "web-search-skills/1.0",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class GitHubEngine(BaseEngine):
|
||||||
|
"""GitHub code & repo search via public API."""
|
||||||
|
|
||||||
|
name = "github"
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
search_type: str = "repositories", **kwargs) -> list[EngineResult]:
|
||||||
|
"""search_type: 'repositories' | 'code' | 'issues'"""
|
||||||
|
t0 = time.time()
|
||||||
|
endpoint = f"https://api.github.com/search/{search_type}"
|
||||||
|
params = {"q": query, "per_page": min(max_results, 30), "sort": "stars"}
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(endpoint, headers=HEADERS, params=params)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
|
||||||
|
results = []
|
||||||
|
for item in data.get("items", [])[:max_results]:
|
||||||
|
if search_type == "repositories":
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=item.get("full_name", ""),
|
||||||
|
url=item.get("html_url", ""),
|
||||||
|
snippet=item.get("description", "") or "",
|
||||||
|
source="github",
|
||||||
|
category="github",
|
||||||
|
extra={
|
||||||
|
"stars": item.get("stargazers_count", 0),
|
||||||
|
"language": item.get("language", ""),
|
||||||
|
"updated": item.get("updated_at", ""),
|
||||||
|
},
|
||||||
|
))
|
||||||
|
elif search_type == "code":
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=item.get("name", ""),
|
||||||
|
url=item.get("html_url", ""),
|
||||||
|
snippet=item.get("text_matches", [{}])[0].get("fragment", "")
|
||||||
|
if item.get("text_matches") else "",
|
||||||
|
source="github",
|
||||||
|
category="github",
|
||||||
|
extra={
|
||||||
|
"repo": item.get("repository", {}).get("full_name", ""),
|
||||||
|
"language": item.get("language", ""),
|
||||||
|
},
|
||||||
|
))
|
||||||
|
elif search_type == "issues":
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=item.get("title", ""),
|
||||||
|
url=item.get("html_url", ""),
|
||||||
|
snippet=item.get("body", "")[:300] if item.get("body") else "",
|
||||||
|
source="github",
|
||||||
|
category="github",
|
||||||
|
extra={
|
||||||
|
"state": item.get("state", ""),
|
||||||
|
"repo": item.get("repository_url", "").split("/")[-2:]
|
||||||
|
if item.get("repository_url") else [],
|
||||||
|
},
|
||||||
|
))
|
||||||
|
|
||||||
|
logger.info(f"[github] {len(results)} results in {elapsed:.2f}s")
|
||||||
|
return results
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[github] failed: {e}")
|
||||||
|
return []
|
||||||
@@ -0,0 +1,201 @@
|
|||||||
|
"""News search engine — multi-source news aggregation."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from urllib.parse import quote_plus
|
||||||
|
from xml.etree import ElementTree as ET
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from scripts.engines.base import BaseEngine, EngineResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
HEADERS = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||||
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class BingNewsEngine(BaseEngine):
|
||||||
|
"""Bing News scraping — good for both CN and EN news."""
|
||||||
|
|
||||||
|
name = "bing_news"
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
region: str = "cn", **kwargs) -> list[EngineResult]:
|
||||||
|
t0 = time.time()
|
||||||
|
base = "cn.bing.com" if region == "cn" else "www.bing.com"
|
||||||
|
url = f"https://{base}/news/search?q={quote_plus(query)}&setlang=zh-CN"
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(url, headers=HEADERS)
|
||||||
|
resp.raise_for_status()
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
results = self._parse(resp.text, max_results)
|
||||||
|
logger.info(f"[bing_news] {len(results)} results in {elapsed:.2f}s")
|
||||||
|
return results
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[bing_news] failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
results = []
|
||||||
|
for item in soup.select("a.title"):
|
||||||
|
title = item.get_text(strip=True)
|
||||||
|
href = item.get("href", "")
|
||||||
|
# Try to get snippet from parent container
|
||||||
|
parent = item.find_parent("div", class_=True)
|
||||||
|
snippet = ""
|
||||||
|
if parent:
|
||||||
|
snippet_el = parent.select_one("div.snippet")
|
||||||
|
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||||
|
if title and href and href.startswith("http"):
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=title, url=href, snippet=snippet,
|
||||||
|
source="bing_news", category="news",
|
||||||
|
))
|
||||||
|
if len(results) >= max_results:
|
||||||
|
break
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
class RSSNewsEngine(BaseEngine):
|
||||||
|
"""RSS-based news — multi-source aggregation."""
|
||||||
|
|
||||||
|
name = "rss_news"
|
||||||
|
|
||||||
|
# Free RSS feeds, no key needed
|
||||||
|
FEEDS = {
|
||||||
|
# Chinese tech/business
|
||||||
|
"36kr": "https://36kr.com/feed",
|
||||||
|
"少数派": "https://sspai.com/feed",
|
||||||
|
"IT之家": "https://www.ithome.com/rss/",
|
||||||
|
"钛媒体": "https://www.tmtpost.com/rss.xml",
|
||||||
|
"Solidot": "https://www.solidot.org/index.rss",
|
||||||
|
"InfoQ CN": "https://www.infoq.cn/feed",
|
||||||
|
"OSChina": "https://www.oschina.net/news/rss",
|
||||||
|
# English tech
|
||||||
|
"HackerNews": "https://hnrss.org/best?count=20",
|
||||||
|
"TechCrunch": "https://techcrunch.com/feed/",
|
||||||
|
"ArsTechnica": "https://feeds.arstechnica.com/arstechnica/index",
|
||||||
|
"TheVerge": "https://www.theverge.com/rss/index.xml",
|
||||||
|
"MIT Tech Review": "https://www.technologyreview.com/feed/",
|
||||||
|
"Wired": "https://www.wired.com/feed/rss",
|
||||||
|
"Lobste.rs": "https://lobste.rs/rss",
|
||||||
|
# English general
|
||||||
|
"ChinaDaily": "https://www.chinadaily.com.cn/rss/china_rss.xml",
|
||||||
|
}
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
**kwargs) -> list[EngineResult]:
|
||||||
|
t0 = time.time()
|
||||||
|
all_results = []
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
tasks = [self._fetch_feed(name, url, query)
|
||||||
|
for name, url in self.FEEDS.items()]
|
||||||
|
feeds = await asyncio.gather(*tasks, return_exceptions=True)
|
||||||
|
|
||||||
|
for feed_results in feeds:
|
||||||
|
if isinstance(feed_results, list):
|
||||||
|
all_results.extend(feed_results)
|
||||||
|
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
logger.info(f"[rss_news] {len(all_results)} results in {elapsed:.2f}s")
|
||||||
|
return all_results[:max_results]
|
||||||
|
|
||||||
|
async def _fetch_feed(self, name: str, url: str, query: str) -> list[EngineResult]:
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(url, headers=HEADERS)
|
||||||
|
resp.raise_for_status()
|
||||||
|
return self._parse_feed(name, resp.text, query)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[rss_news:{name}] failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse_feed(self, name: str, xml_text: str, query: str) -> list[EngineResult]:
|
||||||
|
try:
|
||||||
|
root = ET.fromstring(xml_text)
|
||||||
|
except ET.ParseError:
|
||||||
|
return []
|
||||||
|
|
||||||
|
results = []
|
||||||
|
# RSS 2.0: //channel/item, Atom: //entry
|
||||||
|
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||||
|
|
||||||
|
items = root.findall(".//item") or root.findall(".//atom:entry", ns)
|
||||||
|
|
||||||
|
q_lower = query.lower()
|
||||||
|
q_words = [w for w in q_lower.split() if len(w) > 1]
|
||||||
|
|
||||||
|
for item in items[:50]:
|
||||||
|
title = (item.findtext("title") or
|
||||||
|
item.findtext("atom:title", namespaces=ns) or "")
|
||||||
|
link = (item.findtext("link") or "")
|
||||||
|
if not link:
|
||||||
|
link_el = item.find("atom:link", ns)
|
||||||
|
link = link_el.get("href", "") if link_el is not None else ""
|
||||||
|
desc = (item.findtext("description") or
|
||||||
|
item.findtext("atom:summary", namespaces=ns) or "")
|
||||||
|
# Clean HTML tags from description
|
||||||
|
import re
|
||||||
|
desc = re.sub(r'<[^>]+>', '', desc)
|
||||||
|
|
||||||
|
# Keyword scoring (not strict filter)
|
||||||
|
text_lower = f"{title} {desc}".lower()
|
||||||
|
if q_words:
|
||||||
|
match_count = sum(1 for w in q_words if w in text_lower)
|
||||||
|
if match_count == 0:
|
||||||
|
continue # Skip completely irrelevant items
|
||||||
|
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=title.strip(),
|
||||||
|
url=link.strip(),
|
||||||
|
snippet=desc[:300].strip(),
|
||||||
|
source=f"rss_{name}",
|
||||||
|
category="news",
|
||||||
|
))
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
class NewsSearch(BaseEngine):
|
||||||
|
"""Unified news search — Bing News + RSS in parallel."""
|
||||||
|
|
||||||
|
name = "news"
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
region: str = "cn", **kwargs) -> list[EngineResult]:
|
||||||
|
import asyncio
|
||||||
|
bing = BingNewsEngine(timeout=self.timeout)
|
||||||
|
rss = RSSNewsEngine(timeout=8.0) # RSS feeds may be slow
|
||||||
|
|
||||||
|
bing_results, rss_results = await asyncio.gather(
|
||||||
|
bing.search(query, max_results=max_results, region=region),
|
||||||
|
rss.search(query, max_results=max_results),
|
||||||
|
return_exceptions=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
all_results = []
|
||||||
|
if isinstance(bing_results, list):
|
||||||
|
all_results.extend(bing_results)
|
||||||
|
if isinstance(rss_results, list):
|
||||||
|
all_results.extend(rss_results)
|
||||||
|
|
||||||
|
# Deduplicate by URL
|
||||||
|
seen = set()
|
||||||
|
deduped = []
|
||||||
|
for r in all_results:
|
||||||
|
if r.url not in seen:
|
||||||
|
seen.add(r.url)
|
||||||
|
deduped.append(r)
|
||||||
|
|
||||||
|
return deduped[:max_results]
|
||||||
@@ -0,0 +1,199 @@
|
|||||||
|
"""Parallel search orchestrator — runs multiple engines concurrently."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from scripts.engines.base import BaseEngine, EngineResult
|
||||||
|
from scripts.engines.bing_scraper import BingScraper
|
||||||
|
from scripts.engines.sogou_scraper import SogouScraper
|
||||||
|
from scripts.engines.baidu_scraper import BaiduScraper
|
||||||
|
from scripts.engines.search360_scraper import Search360Scraper
|
||||||
|
from scripts.engines.news_engine import NewsSearch
|
||||||
|
from scripts.engines.academic_engine import AcademicSearch
|
||||||
|
from scripts.engines.github_engine import GitHubEngine
|
||||||
|
from scripts.core.execution_log import execution_log
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class SearchStats:
|
||||||
|
"""Search statistics."""
|
||||||
|
total_engines: int = 0
|
||||||
|
successful_engines: int = 0
|
||||||
|
failed_engines: list[str] = field(default_factory=list)
|
||||||
|
total_results: int = 0
|
||||||
|
deduplicated: int = 0
|
||||||
|
duration_ms: int = 0
|
||||||
|
engine_timings: dict[str, float] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class ParallelSearch:
|
||||||
|
"""Run multiple engines in parallel and merge results."""
|
||||||
|
|
||||||
|
# Default engine sets
|
||||||
|
GENERAL_ENGINES = ["bing", "sogou", "baidu", "360"]
|
||||||
|
VERTICAL_ENGINES = ["news", "academic", "github"]
|
||||||
|
|
||||||
|
def __init__(self, timeout: float = 10.0):
|
||||||
|
self.timeout = timeout
|
||||||
|
self._engines: dict[str, BaseEngine] = {
|
||||||
|
"bing": BingScraper(timeout=timeout),
|
||||||
|
"sogou": SogouScraper(timeout=timeout),
|
||||||
|
"baidu": BaiduScraper(timeout=timeout),
|
||||||
|
"360": Search360Scraper(timeout=timeout),
|
||||||
|
"news": NewsSearch(timeout=timeout),
|
||||||
|
"academic": AcademicSearch(timeout=timeout),
|
||||||
|
"github": GitHubEngine(timeout=timeout),
|
||||||
|
}
|
||||||
|
|
||||||
|
async def search(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
engines: list[str] | None = None,
|
||||||
|
max_results: int = 10,
|
||||||
|
region: str = "cn",
|
||||||
|
**kwargs,
|
||||||
|
) -> tuple[list[EngineResult], SearchStats]:
|
||||||
|
"""
|
||||||
|
Search across specified engines in parallel.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query
|
||||||
|
engines: List of engine names to use (default: all general)
|
||||||
|
max_results: Max results per engine
|
||||||
|
region: 'cn' or 'global'
|
||||||
|
**kwargs: Extra params passed to engines
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(results, stats)
|
||||||
|
"""
|
||||||
|
if engines is None:
|
||||||
|
engines = self.GENERAL_ENGINES.copy()
|
||||||
|
|
||||||
|
t0 = time.time()
|
||||||
|
stats = SearchStats(total_engines=len(engines))
|
||||||
|
|
||||||
|
# Build tasks
|
||||||
|
tasks = {}
|
||||||
|
for name in engines:
|
||||||
|
engine = self._engines.get(name)
|
||||||
|
if engine is None:
|
||||||
|
logger.warning(f"Unknown engine: {name}")
|
||||||
|
continue
|
||||||
|
kwargs_extra = dict(kwargs)
|
||||||
|
if name in ("bing", "bing_news"):
|
||||||
|
kwargs_extra["region"] = region
|
||||||
|
tasks[name] = self._run_engine(engine, query, max_results, kwargs_extra)
|
||||||
|
|
||||||
|
# Run all in parallel
|
||||||
|
results_map = await asyncio.gather(
|
||||||
|
*tasks.values(), return_exceptions=True
|
||||||
|
)
|
||||||
|
|
||||||
|
all_results = []
|
||||||
|
for name, result in zip(tasks.keys(), results_map):
|
||||||
|
if isinstance(result, Exception):
|
||||||
|
stats.failed_engines.append(name)
|
||||||
|
logger.warning(f"[{name}] exception: {result}")
|
||||||
|
# 记录异常到 execution_log
|
||||||
|
execution_log.record(execution_log._logs[-1].__class__(
|
||||||
|
engine=name,
|
||||||
|
operation="search",
|
||||||
|
success=False,
|
||||||
|
error=str(result),
|
||||||
|
error_type=type(result).__name__,
|
||||||
|
duration_ms=int((time.time() - t0) * 1000),
|
||||||
|
))
|
||||||
|
elif isinstance(result, tuple):
|
||||||
|
engine_results, elapsed = result
|
||||||
|
stats.engine_timings[name] = elapsed
|
||||||
|
if engine_results:
|
||||||
|
stats.successful_engines += 1
|
||||||
|
all_results.extend(engine_results)
|
||||||
|
else:
|
||||||
|
stats.failed_engines.append(name)
|
||||||
|
else:
|
||||||
|
stats.failed_engines.append(name)
|
||||||
|
|
||||||
|
# Deduplicate by URL
|
||||||
|
seen_urls = set()
|
||||||
|
deduped = []
|
||||||
|
for r in all_results:
|
||||||
|
key = r.url or r.title # fallback to title if no URL
|
||||||
|
if key not in seen_urls:
|
||||||
|
seen_urls.add(key)
|
||||||
|
deduped.append(r)
|
||||||
|
|
||||||
|
stats.total_results = len(all_results)
|
||||||
|
stats.deduplicated = len(deduped)
|
||||||
|
stats.duration_ms = int((time.time() - t0) * 1000)
|
||||||
|
|
||||||
|
return deduped, stats
|
||||||
|
|
||||||
|
async def _run_engine(
|
||||||
|
self, engine: BaseEngine, query: str, max_results: int, kwargs: dict
|
||||||
|
) -> tuple[list[EngineResult], float]:
|
||||||
|
"""Run a single engine with timing."""
|
||||||
|
with execution_log.track(engine.name, "search", query=query) as ctx:
|
||||||
|
t0 = time.time()
|
||||||
|
try:
|
||||||
|
results = await engine.search(
|
||||||
|
query, max_results=max_results, **kwargs
|
||||||
|
)
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
ctx.result_count = len(results)
|
||||||
|
return results, elapsed
|
||||||
|
except Exception as e:
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
ctx.success = False
|
||||||
|
ctx.error = str(e)
|
||||||
|
ctx.error_type = type(e).__name__
|
||||||
|
raise
|
||||||
|
|
||||||
|
async def search_vertical(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
vertical: str,
|
||||||
|
max_results: int = 10,
|
||||||
|
**kwargs,
|
||||||
|
) -> tuple[list[EngineResult], SearchStats]:
|
||||||
|
"""Search a specific vertical (news/academic/github)."""
|
||||||
|
if vertical == "github":
|
||||||
|
# GitHub supports sub-types
|
||||||
|
search_type = kwargs.pop("search_type", "repositories")
|
||||||
|
engine = self._engines["github"]
|
||||||
|
with execution_log.track("github", "search",
|
||||||
|
search_type=search_type) as ctx:
|
||||||
|
t0 = time.time()
|
||||||
|
results = await engine.search(
|
||||||
|
query, max_results=max_results,
|
||||||
|
search_type=search_type, **kwargs
|
||||||
|
)
|
||||||
|
ctx.result_count = len(results)
|
||||||
|
stats = SearchStats(
|
||||||
|
total_engines=1,
|
||||||
|
successful_engines=1 if results else 0,
|
||||||
|
total_results=len(results),
|
||||||
|
duration_ms=int((time.time() - t0) * 1000),
|
||||||
|
)
|
||||||
|
return results, stats
|
||||||
|
else:
|
||||||
|
return await self.search(
|
||||||
|
query, engines=[vertical], max_results=max_results, **kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
async def health(self) -> dict[str, dict]:
|
||||||
|
"""Check health of all engines."""
|
||||||
|
results = {}
|
||||||
|
for name, engine in self._engines.items():
|
||||||
|
try:
|
||||||
|
results[name] = await engine.health()
|
||||||
|
except Exception as e:
|
||||||
|
results[name] = {"name": name, "status": "error", "error": str(e)}
|
||||||
|
return results
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
"""360 Search HTML scraping engine — free, good Chinese results."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from urllib.parse import quote_plus
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from scripts.engines.base import BaseEngine, EngineResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
HEADERS = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class Search360Scraper(BaseEngine):
|
||||||
|
"""360 Search scraping — good Chinese results, fast in China."""
|
||||||
|
|
||||||
|
name = "360"
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
**kwargs) -> list[EngineResult]:
|
||||||
|
t0 = time.time()
|
||||||
|
url = f"https://www.so.com/s?q={quote_plus(query)}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(url, headers=HEADERS)
|
||||||
|
resp.raise_for_status()
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
results = self._parse(resp.text, max_results)
|
||||||
|
logger.info(f"[360] {len(results)} results in {elapsed:.2f}s")
|
||||||
|
return results
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[360] failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
results = []
|
||||||
|
|
||||||
|
for item in soup.select("li.res-list"):
|
||||||
|
title_el = item.select_one("h3 a")
|
||||||
|
if not title_el:
|
||||||
|
continue
|
||||||
|
title = title_el.get_text(strip=True)
|
||||||
|
href = title_el.get("href", "")
|
||||||
|
snippet_el = item.select_one("p.res-desc, .res-rich")
|
||||||
|
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||||
|
|
||||||
|
if title and href:
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=title, url=href, snippet=snippet, source="360",
|
||||||
|
))
|
||||||
|
if len(results) >= max_results:
|
||||||
|
break
|
||||||
|
|
||||||
|
return results
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
"""Sogou HTML scraping engine — free, good Chinese results."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from urllib.parse import quote_plus
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from scripts.engines.base import BaseEngine, EngineResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
HEADERS = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class SogouScraper(BaseEngine):
|
||||||
|
"""Sogou web scraping — best for Chinese content."""
|
||||||
|
|
||||||
|
name = "sogou"
|
||||||
|
|
||||||
|
async def search(self, query: str, max_results: int = 10,
|
||||||
|
**kwargs) -> list[EngineResult]:
|
||||||
|
t0 = time.time()
|
||||||
|
url = f"https://www.sogou.com/web?query={quote_plus(query)}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(url, headers=HEADERS)
|
||||||
|
resp.raise_for_status()
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
results = self._parse(resp.text, max_results)
|
||||||
|
logger.info(f"[sogou] {len(results)} results in {elapsed:.2f}s")
|
||||||
|
return results
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[sogou] failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
results = []
|
||||||
|
|
||||||
|
for item in soup.select("div.results > div"):
|
||||||
|
title_el = item.select_one("h3 a, a.account")
|
||||||
|
if not title_el:
|
||||||
|
continue
|
||||||
|
title = title_el.get_text(strip=True)
|
||||||
|
href = title_el.get("href", "")
|
||||||
|
snippet_el = item.select_one("p.str_info, div.str-text, .star-wiki")
|
||||||
|
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||||
|
|
||||||
|
if title and href and not href.startswith("/"):
|
||||||
|
results.append(EngineResult(
|
||||||
|
title=title, url=href, snippet=snippet, source="sogou",
|
||||||
|
))
|
||||||
|
if len(results) >= max_results:
|
||||||
|
break
|
||||||
|
|
||||||
|
return results
|
||||||
+87
-275
@@ -1,297 +1,109 @@
|
|||||||
"""
|
"""
|
||||||
Unified search entry point.
|
Unified multi-source search — URL-based engines.
|
||||||
|
|
||||||
Provides a single interface to search across all configured sources.
|
Architecture:
|
||||||
|
Primary: URL engines (Baidu, Bing, Google, DuckDuckGo, Sogou)
|
||||||
|
No external service dependency.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import time
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
from scripts.core.base import SearchResult
|
from dataclasses import dataclass
|
||||||
from scripts.core.config_loader import (
|
|
||||||
get_search_engines, get_news_sources, get_academic_sources,
|
|
||||||
get_wechat_sources, get_api_engines, get_special_engines,
|
|
||||||
get_social_sources, get_all_sources, get_engine,
|
|
||||||
)
|
|
||||||
from scripts.engines.url_engine import UrlSearchEngine
|
|
||||||
from scripts.engines.parser_engines import DuckDuckGoEngine
|
|
||||||
from scripts.news.cls import CLSNewsEngine
|
|
||||||
from scripts.news.wallstreetcn import WallStreetCNEngine
|
|
||||||
from scripts.news.rss import RSSNewsEngine
|
|
||||||
from scripts.news.tech_news import (
|
|
||||||
HackerNewsEngine, GitHubTrendingEngine, Kr36Engine,
|
|
||||||
WeiboEngine, V2EXEngine, TencentNewsEngine,
|
|
||||||
)
|
|
||||||
from scripts.academic.arxiv import ArXivEngine
|
|
||||||
from scripts.wechat.sogou_weixin import SogouWeChatEngine
|
|
||||||
from scripts.social.twitter import TwitterSearchEngine
|
|
||||||
from scripts.engines.baidu_qianfan import BaiduQianFanEngine
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
from scripts.models import SearchQuery, SearchResponse, SourceRef, SearchMetadata, EngineTiming
|
||||||
|
|
||||||
# Default timeout per engine (seconds)
|
|
||||||
_DEFAULT_ENGINE_TIMEOUT = 8
|
|
||||||
# Max concurrent engine calls
|
|
||||||
_DEFAULT_MAX_CONCURRENT = 5
|
|
||||||
|
|
||||||
SOURCE_CATEGORIES = {
|
@dataclass
|
||||||
"web": get_search_engines,
|
class SearchResult:
|
||||||
"news": get_news_sources,
|
title: str
|
||||||
"academic": get_academic_sources,
|
url: str
|
||||||
"wechat": get_wechat_sources,
|
content: str
|
||||||
"social": get_social_sources,
|
source: str = ""
|
||||||
"api": get_api_engines,
|
engine: str = ""
|
||||||
"special": get_special_engines,
|
score: float = 0.0
|
||||||
}
|
category: str = ""
|
||||||
|
published: str = ""
|
||||||
|
|
||||||
|
|
||||||
class UnifiedSearch:
|
class UnifiedSearch:
|
||||||
"""Unified search client that aggregates results from multiple sources."""
|
"""Unified search across multiple open-source engines."""
|
||||||
|
|
||||||
def __init__(self, config: Optional[dict[str, Any]] = None):
|
def __init__(self):
|
||||||
self.config = config or {}
|
self._start_time = 0.0
|
||||||
self._cache: dict[str, list[SearchResult]] = {}
|
|
||||||
self._engines: dict[str, Any] = {}
|
async def search(self, query: str, max_results: int = 5,
|
||||||
self._timeout = self.config.get("engine_timeout", _DEFAULT_ENGINE_TIMEOUT)
|
language: str = "zh-CN") -> SearchResponse:
|
||||||
self._max_concurrent = self.config.get(
|
"""Search using URL-based engines."""
|
||||||
"max_concurrent", _DEFAULT_MAX_CONCURRENT
|
self._start_time = time.time()
|
||||||
|
metadata = SearchMetadata()
|
||||||
|
metadata.total_engines = 1
|
||||||
|
|
||||||
|
all_results, timings = await self._search_urls(query, max_results)
|
||||||
|
|
||||||
|
elapsed_ms = int((time.time() - self._start_time) * 1000)
|
||||||
|
metadata.duration_ms = elapsed_ms
|
||||||
|
metadata.engine_timings = timings
|
||||||
|
metadata.total_raw_results = len(all_results)
|
||||||
|
|
||||||
|
sources = []
|
||||||
|
seen_urls = set()
|
||||||
|
for i, r in enumerate(all_results[:max_results]):
|
||||||
|
if r.url and r.url not in seen_urls:
|
||||||
|
seen_urls.add(r.url)
|
||||||
|
sources.append(SourceRef(
|
||||||
|
index=i + 1,
|
||||||
|
title=r.title or r.content[:50],
|
||||||
|
url=r.url or "",
|
||||||
|
source=r.source,
|
||||||
|
))
|
||||||
|
|
||||||
|
metadata.engines_used = list(set(s.source for s in sources))
|
||||||
|
|
||||||
|
return SearchResponse(
|
||||||
|
query=query,
|
||||||
|
results=list(all_results[:max_results]),
|
||||||
|
sources=sources,
|
||||||
|
metadata=metadata,
|
||||||
)
|
)
|
||||||
self._semaphore = asyncio.Semaphore(self._max_concurrent)
|
|
||||||
|
|
||||||
async def __aenter__(self):
|
async def _search_urls(self, query: str, max_results: int
|
||||||
return self
|
) -> tuple[list[SearchResult], list[EngineTiming]]:
|
||||||
|
"""Build search URLs for major engines."""
|
||||||
|
timings = []
|
||||||
|
engines = {
|
||||||
|
"baidu": f"https://www.baidu.com/s?wd={query}",
|
||||||
|
"bing": f"https://cn.bing.com/search?q={query}",
|
||||||
|
"google": f"https://www.google.com/search?q={query}",
|
||||||
|
"duckduckgo": f"https://html.duckduckgo.com/html/?q={query}",
|
||||||
|
"sogou": f"https://www.sogou.com/web?query={query}",
|
||||||
|
}
|
||||||
|
results = []
|
||||||
|
for name, url in engines.items():
|
||||||
|
results.append(SearchResult(
|
||||||
|
title=f"{name}: {query}",
|
||||||
|
url=url,
|
||||||
|
content=f"Search {query} on {name}",
|
||||||
|
source=name,
|
||||||
|
))
|
||||||
|
|
||||||
async def __aexit__(self, *args):
|
timings.append(EngineTiming(
|
||||||
await self.close()
|
engine_name="url_fallback",
|
||||||
|
success=True,
|
||||||
|
duration_ms=1,
|
||||||
|
result_count=len(results),
|
||||||
|
))
|
||||||
|
return results, timings
|
||||||
|
|
||||||
async def search(
|
async def health(self) -> dict:
|
||||||
self, query: str, sources: Optional[list[str]] = None,
|
"""Check status."""
|
||||||
max_results: int = 10, force_refresh: bool = False, **kwargs,
|
return {
|
||||||
) -> list[SearchResult]:
|
"status": "ok",
|
||||||
"""Search across selected sources with streaming + early termination.
|
"engines": ["baidu", "bing", "google", "duckduckgo", "sogou"],
|
||||||
|
}
|
||||||
Returns results as soon as enough are collected, cancelling remaining
|
|
||||||
slow engines to minimize total wait time.
|
|
||||||
"""
|
|
||||||
if not sources or "all" in sources:
|
|
||||||
sources = list(SOURCE_CATEGORIES.keys())
|
|
||||||
|
|
||||||
cache_key = f"{sorted(sources) if isinstance(sources, list) else sources}:{query}"
|
|
||||||
if not force_refresh and cache_key in self._cache:
|
|
||||||
return self._cache[cache_key]
|
|
||||||
|
|
||||||
# --- Build task list --- #
|
|
||||||
engine_list: list[tuple[Any, str]] = [] # (engine, query)
|
|
||||||
for source_spec in sources:
|
|
||||||
if source_spec in SOURCE_CATEGORIES:
|
|
||||||
for engine_def in SOURCE_CATEGORIES[source_spec]():
|
|
||||||
engine = self._get_engine(engine_def)
|
|
||||||
if engine:
|
|
||||||
engine_list.append((engine, query))
|
|
||||||
else:
|
|
||||||
engine_def = get_engine(source_spec)
|
|
||||||
if engine_def:
|
|
||||||
engine = self._get_engine(engine_def)
|
|
||||||
if engine:
|
|
||||||
engine_list.append((engine, query))
|
|
||||||
|
|
||||||
if not engine_list:
|
|
||||||
return []
|
|
||||||
|
|
||||||
engine_count = len(engine_list)
|
|
||||||
|
|
||||||
# --- Streaming: as_completed + early termination --- #
|
|
||||||
seen_urls: set[str] = set()
|
|
||||||
all_results: list[SearchResult] = []
|
|
||||||
pending = set()
|
|
||||||
|
|
||||||
# Submit all tasks
|
|
||||||
for engine, q in engine_list:
|
|
||||||
coro = self._safe_search(engine, q, max_results, **kwargs)
|
|
||||||
pending.add(asyncio.create_task(coro))
|
|
||||||
|
|
||||||
enough_results = max_results * 2 # allow some slack for dedup
|
|
||||||
|
|
||||||
# Process as they complete
|
|
||||||
for done in asyncio.as_completed(pending):
|
|
||||||
try:
|
|
||||||
results = await done
|
|
||||||
if isinstance(results, (Exception, BaseException)):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Dedup on the fly
|
|
||||||
for r in results:
|
|
||||||
key = r.url.strip().rstrip("/")
|
|
||||||
if key and key not in seen_urls:
|
|
||||||
seen_urls.add(key)
|
|
||||||
all_results.append(r)
|
|
||||||
# Early termination: stop waiting for more engines
|
|
||||||
if len(all_results) >= enough_results:
|
|
||||||
break
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Check early termination after each completion
|
|
||||||
if len(all_results) >= enough_results:
|
|
||||||
remaining = pending - {done}
|
|
||||||
for t in remaining:
|
|
||||||
t.cancel()
|
|
||||||
break
|
|
||||||
|
|
||||||
self._cache[cache_key] = all_results
|
|
||||||
return all_results
|
|
||||||
|
|
||||||
async def _safe_search(self, engine, query, max_results, **kwargs):
|
|
||||||
async with self._semaphore:
|
|
||||||
try:
|
|
||||||
return await asyncio.wait_for(
|
|
||||||
engine.search(query, max_results=max_results, **kwargs),
|
|
||||||
timeout=self._timeout,
|
|
||||||
)
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
logger.warning(f"Engine {engine.name} timed out after {self._timeout}s")
|
|
||||||
return []
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Engine {engine.name} failed: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
async def search_health(self) -> list[dict[str, Any]]:
|
|
||||||
"""Check health of all configured engines (concurrent)."""
|
|
||||||
seen_names: set[str] = set()
|
|
||||||
tasks: list[asyncio.Task] = []
|
|
||||||
for cat_fn in SOURCE_CATEGORIES.values():
|
|
||||||
for engine_def in cat_fn():
|
|
||||||
name = engine_def.get("name", "?")
|
|
||||||
if name in seen_names:
|
|
||||||
continue
|
|
||||||
seen_names.add(name)
|
|
||||||
tasks.append(asyncio.create_task(self._check_one_engine(engine_def, name)))
|
|
||||||
|
|
||||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
||||||
statuses: list[dict[str, Any]] = []
|
|
||||||
for r in results:
|
|
||||||
if isinstance(r, Exception):
|
|
||||||
statuses.append({"name": "?", "status": "error", "detail": str(r)[:80]})
|
|
||||||
else:
|
|
||||||
statuses.append(r)
|
|
||||||
return statuses
|
|
||||||
|
|
||||||
async def _check_one_engine(
|
|
||||||
self, engine_def: dict[str, Any], name: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Check health of a single engine."""
|
|
||||||
engine = self._get_engine(engine_def)
|
|
||||||
if engine is None:
|
|
||||||
return {"name": name, "status": "error", "detail": "engine_init_failed"}
|
|
||||||
try:
|
|
||||||
async with self._semaphore:
|
|
||||||
await asyncio.wait_for(
|
|
||||||
engine.search("health check", max_results=1),
|
|
||||||
timeout=8,
|
|
||||||
)
|
|
||||||
return {"name": name, "status": "ok"}
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
return {"name": name, "status": "timeout"}
|
|
||||||
except Exception as e:
|
|
||||||
return {"name": name, "status": "error", "detail": str(e)[:80]}
|
|
||||||
|
|
||||||
def _get_engine(self, engine_def: dict[str, Any]):
|
|
||||||
"""Get or create an engine instance by definition."""
|
|
||||||
name = engine_def.get("name", "")
|
|
||||||
engine_type = engine_def.get("type", "url")
|
|
||||||
|
|
||||||
if name in self._engines:
|
|
||||||
return self._engines[name]
|
|
||||||
|
|
||||||
if name == "DuckDuckGo":
|
|
||||||
engine = DuckDuckGoEngine(self.config)
|
|
||||||
elif engine_type == "baidu_qianfan" or "baidu qianfan" in name.lower():
|
|
||||||
engine = BaiduQianFanEngine(self.config)
|
|
||||||
elif "cls" in name.lower() or "财联社" in name:
|
|
||||||
engine = CLSNewsEngine(self.config)
|
|
||||||
elif "wallstreet" in name.lower() or "华尔街" in name:
|
|
||||||
engine = WallStreetCNEngine(self.config)
|
|
||||||
elif name == "ArXiv" or engine_type == "api":
|
|
||||||
engine = ArXivEngine(self.config)
|
|
||||||
elif "微信" in name or "wechat" in name.lower() or "weixin" in name.lower():
|
|
||||||
engine = SogouWeChatEngine(self.config)
|
|
||||||
elif "twitter" in name.lower():
|
|
||||||
engine = TwitterSearchEngine(self.config)
|
|
||||||
elif name == "Hacker News":
|
|
||||||
engine = HackerNewsEngine(self.config)
|
|
||||||
elif name == "GitHub Trending":
|
|
||||||
engine = GitHubTrendingEngine(self.config)
|
|
||||||
elif name == "36氪":
|
|
||||||
engine = Kr36Engine(self.config)
|
|
||||||
elif name == "微博热搜":
|
|
||||||
engine = WeiboEngine(self.config)
|
|
||||||
elif name == "V2EX":
|
|
||||||
engine = V2EXEngine(self.config)
|
|
||||||
elif name == "腾讯新闻":
|
|
||||||
engine = TencentNewsEngine(self.config)
|
|
||||||
elif engine_type == "news" and name not in ("cls", "wallstreetcn", "Hacker News", "GitHub Trending", "36氪", "微博热搜", "V2EX", "腾讯新闻"):
|
|
||||||
engine = RSSNewsEngine(self.config)
|
|
||||||
elif engine_type == "social":
|
|
||||||
engine = TwitterSearchEngine(self.config) if "twitter" in name.lower() else UrlSearchEngine(name, engine_def)
|
|
||||||
else:
|
|
||||||
engine = UrlSearchEngine(name, engine_def)
|
|
||||||
|
|
||||||
self._engines[name] = engine
|
|
||||||
return engine
|
|
||||||
|
|
||||||
async def search_engines(self, query, region=None, max_results=10):
|
|
||||||
"""Quick search across web search engines."""
|
|
||||||
engines = get_search_engines()
|
|
||||||
if region:
|
|
||||||
engines = [e for e in engines if e.get("region") == region]
|
|
||||||
return await self.search(query, sources=[e["name"] for e in engines], max_results=max_results)
|
|
||||||
|
|
||||||
async def search_news(self, query, max_results=10, **kwargs):
|
|
||||||
"""Quick search news sources (CN news + RSS feeds)."""
|
|
||||||
return await self.search(query, sources=["news"], max_results=max_results, **kwargs)
|
|
||||||
|
|
||||||
async def search_academic(self, query, max_results=10, **kwargs):
|
|
||||||
"""Quick search academic papers."""
|
|
||||||
return await self.search(query, sources=["academic"], max_results=max_results, **kwargs)
|
|
||||||
|
|
||||||
async def search_wechat(self, query, max_results=10):
|
|
||||||
"""Quick search WeChat articles."""
|
|
||||||
return await self.search(query, sources=["wechat"], max_results=max_results)
|
|
||||||
|
|
||||||
async def search_social(self, query, max_results=10, **kwargs):
|
|
||||||
"""Quick search social media (Twitter/X)."""
|
|
||||||
return await self.search(query, sources=["social"], max_results=max_results, **kwargs)
|
|
||||||
|
|
||||||
async def search_rss(self, query="", max_results=10, **kwargs):
|
|
||||||
"""Quick search RSS news feeds."""
|
|
||||||
engine = RSSNewsEngine(self.config)
|
|
||||||
return await engine.search(query, max_results=max_results, **kwargs)
|
|
||||||
|
|
||||||
async def get_search_urls(self, query, sources=None):
|
|
||||||
"""Get search URLs without executing searches."""
|
|
||||||
if not sources:
|
|
||||||
sources = [e["name"] for e in get_search_engines()]
|
|
||||||
|
|
||||||
urls = {}
|
|
||||||
for name in sources:
|
|
||||||
engine_def = get_engine(name)
|
|
||||||
if engine_def:
|
|
||||||
urls[name] = UrlSearchEngine(name, engine_def).build_url(query)
|
|
||||||
|
|
||||||
return urls
|
|
||||||
|
|
||||||
async def get_twitter_search_url(self, query, search_type="top"):
|
|
||||||
"""Get Twitter/X search URL for browser/web_fetch usage."""
|
|
||||||
engine = TwitterSearchEngine(self.config)
|
|
||||||
return engine.build_url(query, search_type=search_type)
|
|
||||||
|
|
||||||
async def close(self):
|
async def close(self):
|
||||||
"""Close all engine resources."""
|
pass
|
||||||
for engine in self._engines.values():
|
|
||||||
try:
|
|
||||||
await engine.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
from setuptools import setup, find_packages
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name="web-search-skills",
|
||||||
|
version="0.2.0",
|
||||||
|
description="Open-source diversified search in <5s",
|
||||||
|
author="openbot-coder",
|
||||||
|
packages=find_packages(),
|
||||||
|
py_modules=["ws"],
|
||||||
|
python_requires=">=3.10",
|
||||||
|
install_requires=[
|
||||||
|
"httpx>=0.24.0",
|
||||||
|
"beautifulsoup4>=4.12.0",
|
||||||
|
"lxml>=5.0.0",
|
||||||
|
],
|
||||||
|
entry_points={
|
||||||
|
"console_scripts": [
|
||||||
|
"web-search=ws:main",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
@@ -1,23 +1,90 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""
|
"""
|
||||||
Web Search Skills CLI entry point.
|
web-search-skills: Open-source diversified search in <5s.
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
python ws.py <command> <query> [options]
|
python3 ws.py search <query> [--max-results 5]
|
||||||
|
python3 ws.py health
|
||||||
Or install and run:
|
python3 ws.py serve # (future) MCP server mode
|
||||||
uv tool install -e .
|
|
||||||
ws <command> <query> [options]
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
import sys
|
import sys
|
||||||
import os
|
import time
|
||||||
|
|
||||||
# Ensure scripts package is importable
|
|
||||||
_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
||||||
if _SCRIPT_DIR not in sys.path:
|
|
||||||
sys.path.insert(0, _SCRIPT_DIR)
|
|
||||||
|
|
||||||
from scripts.cli import main
|
async def cmd_search(args: argparse.Namespace):
|
||||||
|
from scripts.unified_search import UnifiedSearch
|
||||||
|
|
||||||
|
us = UnifiedSearch()
|
||||||
|
t0 = time.time()
|
||||||
|
resp = await us.search(args.query, max_results=args.max_results)
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
|
||||||
|
if args.json:
|
||||||
|
output = {
|
||||||
|
"query": resp.query,
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"index": s.index,
|
||||||
|
"title": s.title,
|
||||||
|
"url": s.url,
|
||||||
|
"source": s.source,
|
||||||
|
}
|
||||||
|
for s in resp.sources
|
||||||
|
],
|
||||||
|
"time_ms": elapsed * 1000,
|
||||||
|
}
|
||||||
|
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||||
|
else:
|
||||||
|
print(f"\n🔍 「{resp.query}」 — {elapsed:.2f}s")
|
||||||
|
print(f" {len(resp.sources)} results from {len(resp.metadata.engines_used)} engines\n")
|
||||||
|
for i, s in enumerate(resp.sources, 1):
|
||||||
|
print(f" {i}. [{s.source}] {s.title}")
|
||||||
|
print(f" {s.url}")
|
||||||
|
print()
|
||||||
|
await us.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def cmd_health(args: argparse.Namespace):
|
||||||
|
from scripts.unified_search import UnifiedSearch
|
||||||
|
|
||||||
|
us = UnifiedSearch()
|
||||||
|
status = await us.health()
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps(status, ensure_ascii=False, indent=2))
|
||||||
|
else:
|
||||||
|
print("📊 Search Health")
|
||||||
|
print(f" Status: {status.get('status', 'unknown')}")
|
||||||
|
print(f" Engines: {', '.join(status.get('engines', []))}")
|
||||||
|
await us.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="web-search-skills")
|
||||||
|
sub = parser.add_subparsers(dest="command", required=True)
|
||||||
|
|
||||||
|
# search
|
||||||
|
sp = sub.add_parser("search", help="Search across multiple sources")
|
||||||
|
sp.add_argument("query", help="Search query")
|
||||||
|
sp.add_argument("--max-results", type=int, default=5)
|
||||||
|
sp.add_argument("--json", action="store_true", help="JSON output")
|
||||||
|
|
||||||
|
# health
|
||||||
|
hp = sub.add_parser("health", help="Check engine health")
|
||||||
|
hp.add_argument("--json", action="store_true", help="JSON output")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.command == "search":
|
||||||
|
asyncio.run(cmd_search(args))
|
||||||
|
elif args.command == "health":
|
||||||
|
asyncio.run(cmd_health(args))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
Reference in new issue
Block a user