feat: 添加执行日志系统,跟踪各引擎/接口调用耗时和错误信息

新增功能:
- execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情
- 所有搜索引擎自动跟踪每次调用的性能指标
- CLI 搜索后自动显示执行摘要
- health 命令使用 ParallelSearch 检测所有引擎连通性
- 新增 log 命令查看历史执行日志
This commit is contained in:
openbot authored and GitHub committed 2026-05-21 14:50:34 +08:00
1 parent f2d8919860
commit a60ba71663
18 files changed
+1671 -332

No files matched your search

+1
View File
@@ -35,3 +35,4 @@ build/
# Config with secrets # Config with secrets
config.local.yaml config.local.yaml
logs/
+11 -10
View File
@@ -1,20 +1,21 @@
[project] [project]
name = "web-search-skills" name = "web-search-skills"
version = "2.2.0" version = "0.2.0"
description = "Multi-source web search CLI tool" description = "Open-source diversified search in <5s. 28+ engines across web, news, academic, social, WeChat."
requires-python = ">=3.8" authors = [{ name = "openbot-coder" }]
readme = "README.md"
license = { file = "LICENSE" }
requires-python = ">=3.10"
dependencies = [ dependencies = [
"httpx>=0.25.0", "httpx>=0.24.0",
"beautifulsoup4>=4.12.0", "beautifulsoup4>=4.12.0",
"lxml>=4.9.0", "lxml>=5.0.0",
] ]
[project.scripts] [project.scripts]
web-search = "scripts.cli:main" web-search = "ws:main"
[build-system] [build-system]
requires = ["setuptools>=64", "wheel"] requires = ["setuptools>=68.0"]
build-backend = "setuptools.build_meta" build-backend = "setuptools.build_meta"
[tool.setuptools.packages.find]
include = ["scripts*"]
+182 -35
View File
@@ -68,6 +68,9 @@ logging.basicConfig(
) )
logger = logging.getLogger("cli") logger = logging.getLogger("cli")
# 执行日志模块
from scripts.core.execution_log import execution_log
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Lazy engine imports (import on demand to keep startup fast) # Lazy engine imports (import on demand to keep startup fast)
@@ -78,6 +81,11 @@ def _get_unified():
return UnifiedSearch return UnifiedSearch
def _get_parallel():
from scripts.engines.parallel_search import ParallelSearch
return ParallelSearch
def _get_sogou(): def _get_sogou():
from scripts.wechat.sogou_weixin import SogouWeChatEngine from scripts.wechat.sogou_weixin import SogouWeChatEngine
return SogouWeChatEngine return SogouWeChatEngine
@@ -160,7 +168,41 @@ def _sanitize_for_console(text: str) -> str:
def _format_result(r, fmt: str = "text") -> str: def _format_result(r, fmt: str = "text") -> str:
"""Format a single SearchResult.""" """Format a single search result (supports both EngineResult and old models)."""
# Handle EngineResult (new engines)
if hasattr(r, "source") and hasattr(r, "snippet"):
title = getattr(r, "title", "")
url = getattr(r, "url", "")
snippet = getattr(r, "snippet", "")
source = getattr(r, "source", "")
category = getattr(r, "category", "")
extra = getattr(r, "extra", {})
published = getattr(r, "published", "")
if fmt == "json":
return json.dumps({
"title": title, "url": url, "snippet": snippet[:300],
"source": source, "category": category,
"published": published, "extra": extra,
}, ensure_ascii=False, indent=2)
lines = [_sanitize_for_console(title)]
lines.append(f" URL: {url}")
if snippet:
lines.append(f" {_sanitize_for_console(snippet[:200])}")
extra_parts = []
if source:
extra_parts.append(f"来源: {source}")
if published:
extra_parts.append(f"时间: {published}")
for k, v in extra.items():
if k in ("stars", "language") and v:
extra_parts.append(f"{k}: {v}")
if extra_parts:
lines.append(f" ({'; '.join(extra_parts)})")
return "\n".join(lines)
# Handle old SearchResult format (rank, title, url, snippet, source, category, extra)
if fmt == "json": if fmt == "json":
return json.dumps({ return json.dumps({
"rank": r.rank, "rank": r.rank,
@@ -224,49 +266,61 @@ def _print_results(results, fmt: str = "text"):
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
async def _search_all(query: str, max_results: int, **kwargs) -> list[Any]: async def _search_all(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()() """Parallel search across all general engines."""
try: ps = _get_parallel()(timeout=10.0)
return await us.search(query, max_results=max_results, **kwargs) region = kwargs.get("region", "cn")
finally: results, stats = await ps.search(
await us.close() query, engines=None, max_results=max_results, region=region,
)
logger.debug(f"Stats: {stats}")
return results
async def _search_web(query: str, max_results: int, region: str = None, **kwargs) -> list[Any]: async def _search_web(query: str, max_results: int, region: str = None, **kwargs) -> list[Any]:
us = _get_unified()() """Parallel web search (general engines)."""
try: ps = _get_parallel()(timeout=10.0)
return await us.search_engines(query, region=region, max_results=max_results) results, stats = await ps.search(
finally: query, engines=None, max_results=max_results, region=region or "cn",
await us.close() )
return results
async def _search_news(query: str, max_results: int, **kwargs) -> list[Any]: async def _search_news(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()() """Vertical news search."""
try: ps = _get_parallel()(timeout=10.0)
return await us.search_news(query, max_results=max_results, **kwargs) results, stats = await ps.search_vertical(
finally: query, "news", max_results=max_results, **kwargs,
await us.close() )
return results
async def _search_wechat(query: str, max_results: int) -> list[Any]: async def _search_wechat(query: str, max_results: int) -> list[Any]:
engine = _get_sogou()() engine = _get_sogou()()
try: try:
return await engine.search(query, max_results=max_results) with execution_log.track("weixin", "search", query=query) as ctx:
results = await engine.search(query, max_results=max_results)
ctx.result_count = len(results)
return results
finally: finally:
await engine.close() await engine.close()
async def _search_academic(query: str, max_results: int) -> list[Any]: async def _search_academic(query: str, max_results: int) -> list[Any]:
us = _get_unified()() """Vertical academic search (ArXiv + CrossRef)."""
try: ps = _get_parallel()(timeout=15.0)
return await us.search_academic(query, max_results=max_results) results, stats = await ps.search_vertical(
finally: query, "academic", max_results=max_results,
await us.close() )
return results
async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]: async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()() us = _get_unified()()
try: try:
return await us.search_social(query, max_results=max_results, **kwargs) with execution_log.track("social", "search", query=query) as ctx:
results = await us.search_social(query, max_results=max_results, **kwargs)
ctx.result_count = len(results) if results else 0
return results
finally: finally:
await us.close() await us.close()
@@ -275,7 +329,10 @@ async def _search_baidu(query: str, max_results: int, **kwargs) -> list[Any]:
from scripts.engines.baidu_qianfan import BaiduQianFanEngine from scripts.engines.baidu_qianfan import BaiduQianFanEngine
engine = BaiduQianFanEngine({"api_key": os.getenv("BAIDU_API_KEY", "")}) engine = BaiduQianFanEngine({"api_key": os.getenv("BAIDU_API_KEY", "")})
try: try:
return await engine.search(query, max_results=max_results, **kwargs) with execution_log.track("baidu_qianfan", "search", query=query) as ctx:
results = await engine.search(query, max_results=max_results, **kwargs)
ctx.result_count = len(results)
return results
finally: finally:
await engine.close() await engine.close()
@@ -283,11 +340,24 @@ async def _search_baidu(query: str, max_results: int, **kwargs) -> list[Any]:
async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]: async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()() us = _get_unified()()
try: try:
return await us.search_rss(query, max_results=max_results, **kwargs) with execution_log.track("rss", "search", query=query) as ctx:
results = await us.search_rss(query, max_results=max_results, **kwargs)
ctx.result_count = len(results) if results else 0
return results
finally: finally:
await us.close() await us.close()
async def _search_github(query: str, max_results: int, **kwargs) -> list[Any]:
"""GitHub code/repo search."""
ps = _get_parallel()(timeout=10.0)
search_type = kwargs.get("search_type", "repositories")
results, stats = await ps.search_vertical(
query, "github", max_results=max_results, search_type=search_type,
)
return results
async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]: async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]:
us = _get_unified()() us = _get_unified()()
try: try:
@@ -300,6 +370,69 @@ async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]:
# CLI actions # CLI actions
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def cmd_log(args):
"""查看执行日志历史。"""
import json as json_mod
log_file = "logs/search_log.jsonl"
try:
with open(log_file, "r", encoding="utf-8") as f:
lines = [json_mod.loads(line) for line in f if line.strip()]
except FileNotFoundError:
print(f"暂无日志文件 ({log_file})")
return
if not lines:
print("暂无执行记录")
return
# 按 session 分组
sessions = {}
for entry in lines:
session = entry.get("session_start", "unknown")
if session not in sessions:
sessions[session] = []
sessions[session].append(entry)
print(f"═══════════════════════════════════════════")
print(f" 执行日志历史 (共 {len(lines)} 条记录, {len(sessions)} 个会话)")
print(f"═══════════════════════════════════════════")
# 只显示最近5个会话
recent_sessions = list(sessions.items())[-5:]
for session_start, entries in recent_sessions:
total = len(entries)
success = sum(1 for e in entries if e.get("success", True))
failed = total - success
durations = [e.get("duration_ms", 0) for e in entries]
avg_dur = sum(durations) // total if total else 0
print(f"\n 会话: {session_start}")
print(f" 调用: {total} 次 (成功 {success}, 失败 {failed})")
print(f" 平均耗时: {avg_dur}ms")
# 按引擎分组
engines = {}
for e in entries:
eng = e.get("engine", "unknown")
if eng not in engines:
engines[eng] = {"calls": 0, "errors": []}
engines[eng]["calls"] += 1
if not e.get("success", True):
engines[eng]["errors"].append(e.get("error", "unknown"))
for eng, info in sorted(engines.items()):
err_count = len(info["errors"])
status = "✓" if err_count == 0 else f"✗ ({err_count} errors)"
print(f" {eng:15s} {status} calls={info['calls']}")
# 显示错误详情
for err in info["errors"][:3]: # 最多显示3个
print(f" └─ {err[:80]}")
print(f"\n═══════════════════════════════════════════")
def cmd_sources(args): def cmd_sources(args):
"""List all available search sources.""" """List all available search sources."""
cl = _get_config_loader() cl = _get_config_loader()
@@ -337,22 +470,25 @@ def cmd_urls(args):
async def cmd_health(args): async def cmd_health(args):
"""Test all engine connectivity.""" """Test all engine connectivity."""
us = _get_unified()() ps = _get_parallel()(timeout=10.0)
try: try:
print("正在检测各引擎连通性...\n") print("正在检测各引擎连通性...\n")
statuses = await us.search_health() statuses = await ps.health()
ok_count = sum(1 for s in statuses if s["status"] == "ok") ok_count = sum(1 for s in statuses.values() if s["status"] == "ok")
for s in statuses: for name, s in statuses.items():
if s["status"] == "ok": if s["status"] == "ok":
print(f" [OK] {s['name']}") print(f" [OK] {name}")
elif s["status"] == "timeout": elif s["status"] == "timeout":
print(f" [TIMEOUT] {s['name']}") print(f" [TIMEOUT] {name}")
else: else:
detail = s.get("detail", "") detail = s.get("error", s.get("detail", ""))
print(f" [FAIL] {s['name']} {detail}") print(f" [FAIL] {name} {detail}")
print(f"\n总计: {len(statuses)} 个引擎, {ok_count} 个可用") print(f"\n总计: {len(statuses)} 个引擎, {ok_count} 个可用")
print()
execution_log.print_summary()
execution_log.save("logs/search_log.jsonl")
finally: finally:
await us.close() pass
async def _run_search(args): async def _run_search(args):
@@ -369,6 +505,7 @@ async def _run_search(args):
"academic": _search_academic, "academic": _search_academic,
"social": _search_social, "social": _search_social,
"rss": _search_rss, "rss": _search_rss,
"github": _search_github,
} }
search_fn = method_map.get(args.command, _search_all) search_fn = method_map.get(args.command, _search_all)
@@ -408,6 +545,7 @@ async def _run_search(args):
"academic": "学术搜索", "academic": "学术搜索",
"social": "社交搜索", "social": "社交搜索",
"rss": "RSS 新闻搜索", "rss": "RSS 新闻搜索",
"github": "GitHub 搜索",
} }
header = cmd_names.get(args.command, "搜索") header = cmd_names.get(args.command, "搜索")
print(f"{header}: {args.query}") print(f"{header}: {args.query}")
@@ -415,6 +553,11 @@ async def _run_search(args):
_print_results(results, fmt="json" if args.json else "text") _print_results(results, fmt="json" if args.json else "text")
# 打印执行日志摘要并保存到文件
print()
execution_log.print_summary()
execution_log.save("logs/search_log.jsonl")
# Save to file if requested (with basic path-sanitization) # Save to file if requested (with basic path-sanitization)
if args.output: if args.output:
output_data = json.dumps( output_data = json.dumps(
@@ -468,7 +611,7 @@ def build_parser() -> argparse.ArgumentParser:
nargs="?", nargs="?",
default="search", default="search",
choices=["search", "web", "baidu", "news", "wechat", "academic", choices=["search", "web", "baidu", "news", "wechat", "academic",
"social", "rss", "health", "sources", "urls"], "social", "rss", "github", "health", "sources", "urls", "log"],
help="Search type (default: search)", help="Search type (default: search)",
) )
parser.add_argument( parser.add_argument(
@@ -542,6 +685,10 @@ def main():
cmd_sources(args) cmd_sources(args)
return return
if args.command == "log":
cmd_log(args)
return
if args.command == "health": if args.command == "health":
asyncio.run(cmd_health(args)) asyncio.run(cmd_health(args))
return return
+7
View File
@@ -43,6 +43,10 @@ try:
# --- 日志 --- # --- 日志 ---
log_level: str = "INFO" log_level: str = "INFO"
# --- API 引擎每日限额(有 key 的尽量做替补)---
baidu_daily_limit: int = 50
minimax_daily_limit: int = 100
except ImportError: except ImportError:
class Settings: # type: ignore class Settings: # type: ignore
@@ -76,6 +80,9 @@ except ImportError:
os.environ.get("MAX_CONCURRENT", "5") os.environ.get("MAX_CONCURRENT", "5")
) )
self.log_level = os.environ.get("LOG_LEVEL", "INFO") self.log_level = os.environ.get("LOG_LEVEL", "INFO")
# API 引擎每日限额
self.baidu_daily_limit = int(os.environ.get("BAIDU_DAILY_LIMIT", "50"))
self.minimax_daily_limit = int(os.environ.get("MINIMAX_DAILY_LIMIT", "100"))
# 全局单例 # 全局单例
+232
View File
@@ -0,0 +1,232 @@
"""
执行日志模块 — 记录各引擎/接口的调用耗时、报错信息。
用于跟踪各个接口的执行效果,便于后续优化。
Usage:
from scripts.core.execution_log import execution_log
# 自动记录耗时(推荐用法)
with execution_log.track("bing", "search") as ctx:
results = await engine.search(query)
ctx.result_count = len(results)
# 手动记录
execution_log.record(EngineCallLog(
engine="bing",
operation="search",
duration_ms=320,
success=True,
result_count=5,
))
# 打印摘要
execution_log.print_summary()
"""
from __future__ import annotations
import json
import time
import logging
from contextlib import contextmanager
from dataclasses import dataclass, field, asdict
from datetime import datetime
from typing import Any, Optional
logger = logging.getLogger(__name__)
@dataclass
class EngineCallLog:
"""单次引擎调用记录。"""
engine: str # 引擎名(bing, sogou, baidu, ...)
operation: str = "search" # 操作类型(search, fetch, health, ...)
timestamp: str = "" # 调用时间 ISO 格式
start_time: float = 0.0 # 起始时间戳
duration_ms: int = 0 # 耗时(毫秒)
success: bool = True # 是否成功
result_count: int = 0 # 返回结果数
error: Optional[str] = None # 错误信息
error_type: Optional[str] = None # 错误类型(异常类名)
status_code: Optional[int] = None # HTTP 状态码(如有)
detail: dict[str, Any] = field(default_factory=dict) # 额外信息
class _ExecutionLogCollector:
"""全局执行日志收集器。"""
def __init__(self):
self._logs: list[EngineCallLog] = []
self._session_start = time.time()
def record(self, log: EngineCallLog):
"""记录一次引擎调用。"""
self._logs.append(log)
# 同时写 logger
status = "✓" if log.success else "✗"
msg = (
f"{status} [{log.engine}] {log.operation} "
f"{log.duration_ms}ms"
)
if log.result_count > 0:
msg += f" → {log.result_count} results"
if log.error:
msg += f" | error: {log.error}"
if log.status_code and log.status_code >= 400:
msg += f" | HTTP {log.status_code}"
if log.success:
logger.info(msg)
else:
logger.warning(msg)
@contextmanager
def track(self, engine: str, operation: str = "search",
**extra):
"""上下文管理器,自动记录耗时和异常。
Example:
with execution_log.track("bing", "search", region="cn") as ctx:
results = await engine.search(query)
ctx.result_count = len(results)
"""
entry = EngineCallLog(
engine=engine,
operation=operation,
timestamp=datetime.now().isoformat(),
start_time=time.time(),
detail=dict(extra),
)
try:
yield entry
except Exception as e:
entry.success = False
entry.error = str(e)
entry.error_type = type(e).__name__
raise
finally:
entry.duration_ms = int((time.time() - entry.start_time) * 1000)
self.record(entry)
def get_logs(self, engine: str | None = None) -> list[EngineCallLog]:
"""获取日志,可按引擎名过滤。"""
if engine:
return [l for l in self._logs if l.engine == engine]
return list(self._logs)
def get_stats(self, engine: str | None = None) -> dict[str, Any]:
"""获取统计摘要。"""
logs = self.get_logs(engine)
if not logs:
return {"total_calls": 0}
total = len(logs)
success = sum(1 for l in logs if l.success)
failed = total - success
durations = [l.duration_ms for l in logs]
result_counts = [l.result_count for l in logs]
errors = []
for l in logs:
if l.error:
errors.append({
"engine": l.engine,
"operation": l.operation,
"error_type": l.error_type,
"error": l.error[:200],
"timestamp": l.timestamp,
})
return {
"total_calls": total,
"success": success,
"failed": failed,
"success_rate": f"{success/total*100:.1f}%",
"avg_duration_ms": sum(durations) // total,
"max_duration_ms": max(durations),
"min_duration_ms": min(durations),
"total_results": sum(result_counts),
"errors": errors,
}
def print_summary(self):
"""打印本次会话的执行摘要。"""
if not self._logs:
print("暂无执行记录")
return
total_time = int((time.time() - self._session_start) * 1000)
stats = self.get_stats()
lines = [
"═══════════════════════════════════════════",
" 执行日志摘要",
"═══════════════════════════════════════════",
f" 总调用次数: {stats['total_calls']}",
f" 成功/失败: {stats['success']}/{stats['failed']} ({stats['success_rate']})",
f" 平均耗时: {stats['avg_duration_ms']}ms",
f" 最长耗时: {stats['max_duration_ms']}ms",
f" 最短耗时: {stats['min_duration_ms']}ms",
f" 总结果数: {stats['total_results']}",
f" 总耗时: {total_time}ms",
"───────────────────────────────────────────",
]
# 按引擎分组
engine_names = sorted(set(l.engine for l in self._logs))
lines.append(" 引擎明细:")
for name in engine_names:
estat = self.get_stats(name)
status = "✓" if estat["failed"] == 0 else "✗"
lines.append(
f" {status} {name:15s} "
f"calls={estat['total_calls']} "
f"avg={estat['avg_duration_ms']}ms "
f"max={estat['max_duration_ms']}ms "
f"results={estat['total_results']}"
)
# 错误详情
if stats["errors"]:
lines.append("───────────────────────────────────────────")
lines.append(" 错误详情:")
for err in stats["errors"]:
lines.append(
f" ✗ [{err['engine']}] {err['error_type']}: {err['error'][:100]}"
)
lines.append("═══════════════════════════════════════════")
print("\n".join(lines))
def to_json(self) -> str:
"""导出为 JSON。"""
data = {
"session_start": datetime.fromtimestamp(self._session_start).isoformat(),
"total_calls": len(self._logs),
"stats": self.get_stats(),
"logs": [asdict(l) for l in self._logs],
}
return json.dumps(data, ensure_ascii=False, indent=2)
def save(self, path: str = "logs/search_log.jsonl"):
"""追加写入 JSONL 日志文件。"""
import os
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
with open(path, "a", encoding="utf-8") as f:
for log in self._logs:
record = asdict(log)
record["session_start"] = datetime.fromtimestamp(
self._session_start
).isoformat()
f.write(json.dumps(record, ensure_ascii=False) + "\n")
def clear(self):
"""清空日志。"""
self._logs.clear()
self._session_start = time.time()
# 全局单例
execution_log = _ExecutionLogCollector()
+162
View File
@@ -0,0 +1,162 @@
"""Academic search engine — ArXiv + CrossRef APIs (free, no key)."""
from __future__ import annotations
import time
import logging
from urllib.parse import quote_plus
from xml.etree import ElementTree as ET
import httpx
from scripts.engines.base import BaseEngine, EngineResult
logger = logging.getLogger(__name__)
class ArxivEngine(BaseEngine):
"""ArXiv paper search — free, no key needed."""
name = "arxiv"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
t0 = time.time()
url = (
f"http://export.arxiv.org/api/query?"
f"search_query=all:{quote_plus(query)}&start=0&max_results={max_results}"
f"&sortBy=relevance&sortOrder=descending"
)
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse_xml(resp.text, max_results)
logger.info(f"[arxiv] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[arxiv] failed: {e}")
return []
def _parse_xml(self, xml_text: str, max_results: int) -> list[EngineResult]:
root = ET.fromstring(xml_text)
ns = {"atom": "http://www.w3.org/2005/Atom"}
results = []
for entry in root.findall("atom:entry", ns)[:max_results]:
title = entry.findtext("atom:title", "", ns).strip().replace("\n", " ")
summary = entry.findtext("atom:summary", "", ns).strip().replace("\n", " ")
link_el = entry.find("atom:link[@type='text/html']", ns)
if link_el is None:
link_el = entry.find("atom:id", ns)
url = link_el.get("href", "") if link_el is not None else ""
published = entry.findtext("atom:published", "", ns)
authors = [a.findtext("atom:name", "", ns)
for a in entry.findall("atom:author", ns)]
categories = [c.get("term", "")
for c in entry.findall("atom:category", ns)]
if title:
results.append(EngineResult(
title=title, url=url, snippet=summary[:300],
source="arxiv", category="academic",
published=published,
extra={"authors": authors[:5], "categories": categories},
))
return results
class CrossRefEngine(BaseEngine):
"""CrossRef academic search — free, no key needed."""
name = "crossref"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
t0 = time.time()
url = "https://api.crossref.org/works"
params = {
"query": query,
"rows": min(max_results, 20),
"sort": "relevance",
"order": "desc",
}
headers = {"User-Agent": "web-search-skills/1.0 (mailto:search@example.com)"}
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, params=params, headers=headers)
resp.raise_for_status()
elapsed = time.time() - t0
data = resp.json()
results = []
for item in data.get("message", {}).get("items", [])[:max_results]:
title_list = item.get("title", [])
title = title_list[0] if title_list else ""
doi = item.get("DOI", "")
url_link = item.get("URL", f"https://doi.org/{doi}")
abstract = item.get("abstract", "")[:300]
pub_date = ""
if item.get("published-print"):
dp = item["published-print"].get("date-parts", [[]])[0]
pub_date = "-".join(str(p) for p in dp if p) if dp else ""
authors = [f"{a.get('given', '')} {a.get('family', '')}".strip()
for a in item.get("author", [])]
if title:
results.append(EngineResult(
title=title, url=url_link, snippet=abstract,
source="crossref", category="academic",
published=pub_date,
extra={
"doi": doi,
"authors": authors[:5],
"journal": (item.get("container-title", [""])[0]
if item.get("container-title") else ""),
},
))
logger.info(f"[crossref] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[crossref] failed: {e}")
return []
class AcademicSearch(BaseEngine):
"""Unified academic search — ArXiv + CrossRef in parallel."""
name = "academic"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
import asyncio
arxiv = ArxivEngine(timeout=self.timeout)
crossref = CrossRefEngine(timeout=self.timeout)
arxiv_results, crossref_results = await asyncio.gather(
arxiv.search(query, max_results=max_results),
crossref.search(query, max_results=max_results),
return_exceptions=True,
)
all_results = []
if isinstance(arxiv_results, list):
all_results.extend(arxiv_results)
if isinstance(crossref_results, list):
all_results.extend(crossref_results)
# Deduplicate by URL
seen = set()
deduped = []
for r in all_results:
if r.url not in seen:
seen.add(r.url)
deduped.append(r)
return deduped[:max_results]
+69
View File
@@ -0,0 +1,69 @@
"""Baidu HTML scraping engine — free, fastest in China, best Chinese results."""
from __future__ import annotations
import time
import logging
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
logger = logging.getLogger(__name__)
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9",
}
class BaiduScraper(BaseEngine):
"""Baidu web scraping — fastest in China, best Chinese content."""
name = "baidu"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
t0 = time.time()
url = f"https://www.baidu.com/s?wd={quote_plus(query)}&rn={max_results}"
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=HEADERS)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse(resp.text, max_results)
logger.info(f"[baidu] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[baidu] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
for item in soup.select("div.result, div.c-container"):
title_el = item.select_one("h3 a")
if not title_el:
continue
title = title_el.get_text(strip=True)
href = title_el.get("href", "")
snippet_el = item.select_one(
"span.content-right_8Zs40, div.c-abstract, "
"span[class*='content'], div[class*='abstract']"
)
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href:
results.append(EngineResult(
title=title, url=href, snippet=snippet, source="baidu",
))
if len(results) >= max_results:
break
return results
+64
View File
@@ -0,0 +1,64 @@
"""Base engine class and shared data models."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any
from scripts.core.execution_log import execution_log
@dataclass
class EngineResult:
"""Unified result from any engine."""
title: str
url: str
snippet: str = ""
source: str = ""
category: str = "" # "general" | "news" | "academic" | "github"
published: str = ""
extra: dict[str, Any] = field(default_factory=dict)
class BaseEngine(ABC):
"""Base class for all search engines.
Automatically tracks execution timing and errors via execution_log.
"""
name: str = "unknown"
timeout: float = 10.0
def __init__(self, timeout: float = 10.0):
self.timeout = timeout
async def search_with_log(
self, query: str, max_results: int = 10, **kwargs
) -> list[EngineResult]:
"""Search with automatic execution logging.
Subclasses override `search()`, not this method.
"""
with execution_log.track(self.name, "search", query=query) as ctx:
results = await self.search(query, max_results=max_results, **kwargs)
ctx.result_count = len(results)
return results
@abstractmethod
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
...
async def health(self) -> dict:
"""Quick connectivity check."""
with execution_log.track(self.name, "health") as ctx:
try:
results = await self.search("test", max_results=1)
ctx.result_count = len(results)
return {"name": self.name, "status": "ok", "count": len(results)}
except Exception as e:
ctx.success = False
ctx.error = str(e)
ctx.error_type = type(e).__name__
return {"name": self.name, "status": "fail", "error": str(e)}
+69
View File
@@ -0,0 +1,69 @@
"""Bing HTML scraping engine — free, no API key."""
from __future__ import annotations
import re
import time
import logging
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
logger = logging.getLogger(__name__)
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
class BingScraper(BaseEngine):
"""Bing web scraping — fast, good English results."""
name = "bing"
async def search(self, query: str, max_results: int = 10,
region: str = "global", **kwargs) -> list[EngineResult]:
t0 = time.time()
url = f"https://cn.bing.com/search?q={quote_plus(query)}&setlang=zh-CN"
if region == "global":
url = f"https://www.bing.com/search?q={quote_plus(query)}"
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=HEADERS)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse(resp.text, max_results)
logger.info(f"[bing] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[bing] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
for item in soup.select("li.b_algo"):
title_el = item.select_one("h2 a")
if not title_el:
continue
title = title_el.get_text(strip=True)
href = title_el.get("href", "")
snippet_el = item.select_one("p, .b_caption p")
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href:
results.append(EngineResult(
title=title, url=href, snippet=snippet, source="bing",
))
if len(results) >= max_results:
break
return results
+69
View File
@@ -0,0 +1,69 @@
"""Brave Search HTML scraping engine — free, privacy-focused."""
from __future__ import annotations
import time
import logging
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
logger = logging.getLogger(__name__)
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
class BraveScraper(BaseEngine):
"""Brave Search scraping — privacy-focused, good results."""
name = "brave"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
t0 = time.time()
url = f"https://search.brave.com/search?q={quote_plus(query)}&source=web"
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=HEADERS)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse(resp.text, max_results)
logger.info(f"[brave] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[brave] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
# Brave uses div.snippet or article elements
for item in soup.select("div.snippet, div.result"):
title_el = item.select_one("a.result-header, a.snippet-title")
if not title_el:
title_el = item.select_one("a")
if not title_el:
continue
title = title_el.get_text(strip=True)
href = title_el.get("href", "")
snippet_el = item.select_one("p.snippet-description, div.snippet-description")
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href and href.startswith("http"):
results.append(EngineResult(
title=title, url=href, snippet=snippet, source="brave",
))
if len(results) >= max_results:
break
return results
+86
View File
@@ -0,0 +1,86 @@
"""GitHub code search engine — uses GitHub API (no key needed for basic search)."""
from __future__ import annotations
import time
import logging
from urllib.parse import quote_plus
import httpx
from scripts.engines.base import BaseEngine, EngineResult
logger = logging.getLogger(__name__)
HEADERS = {
"Accept": "application/vnd.github.v3+json",
"User-Agent": "web-search-skills/1.0",
}
class GitHubEngine(BaseEngine):
"""GitHub code & repo search via public API."""
name = "github"
async def search(self, query: str, max_results: int = 10,
search_type: str = "repositories", **kwargs) -> list[EngineResult]:
"""search_type: 'repositories' | 'code' | 'issues'"""
t0 = time.time()
endpoint = f"https://api.github.com/search/{search_type}"
params = {"q": query, "per_page": min(max_results, 30), "sort": "stars"}
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(endpoint, headers=HEADERS, params=params)
resp.raise_for_status()
data = resp.json()
elapsed = time.time() - t0
results = []
for item in data.get("items", [])[:max_results]:
if search_type == "repositories":
results.append(EngineResult(
title=item.get("full_name", ""),
url=item.get("html_url", ""),
snippet=item.get("description", "") or "",
source="github",
category="github",
extra={
"stars": item.get("stargazers_count", 0),
"language": item.get("language", ""),
"updated": item.get("updated_at", ""),
},
))
elif search_type == "code":
results.append(EngineResult(
title=item.get("name", ""),
url=item.get("html_url", ""),
snippet=item.get("text_matches", [{}])[0].get("fragment", "")
if item.get("text_matches") else "",
source="github",
category="github",
extra={
"repo": item.get("repository", {}).get("full_name", ""),
"language": item.get("language", ""),
},
))
elif search_type == "issues":
results.append(EngineResult(
title=item.get("title", ""),
url=item.get("html_url", ""),
snippet=item.get("body", "")[:300] if item.get("body") else "",
source="github",
category="github",
extra={
"state": item.get("state", ""),
"repo": item.get("repository_url", "").split("/")[-2:]
if item.get("repository_url") else [],
},
))
logger.info(f"[github] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[github] failed: {e}")
return []
+201
View File
@@ -0,0 +1,201 @@
"""News search engine — multi-source news aggregation."""
from __future__ import annotations
import time
import logging
from urllib.parse import quote_plus
from xml.etree import ElementTree as ET
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
logger = logging.getLogger(__name__)
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
class BingNewsEngine(BaseEngine):
"""Bing News scraping — good for both CN and EN news."""
name = "bing_news"
async def search(self, query: str, max_results: int = 10,
region: str = "cn", **kwargs) -> list[EngineResult]:
t0 = time.time()
base = "cn.bing.com" if region == "cn" else "www.bing.com"
url = f"https://{base}/news/search?q={quote_plus(query)}&setlang=zh-CN"
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=HEADERS)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse(resp.text, max_results)
logger.info(f"[bing_news] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[bing_news] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
for item in soup.select("a.title"):
title = item.get_text(strip=True)
href = item.get("href", "")
# Try to get snippet from parent container
parent = item.find_parent("div", class_=True)
snippet = ""
if parent:
snippet_el = parent.select_one("div.snippet")
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href and href.startswith("http"):
results.append(EngineResult(
title=title, url=href, snippet=snippet,
source="bing_news", category="news",
))
if len(results) >= max_results:
break
return results
class RSSNewsEngine(BaseEngine):
"""RSS-based news — multi-source aggregation."""
name = "rss_news"
# Free RSS feeds, no key needed
FEEDS = {
# Chinese tech/business
"36kr": "https://36kr.com/feed",
"少数派": "https://sspai.com/feed",
"IT之家": "https://www.ithome.com/rss/",
"钛媒体": "https://www.tmtpost.com/rss.xml",
"Solidot": "https://www.solidot.org/index.rss",
"InfoQ CN": "https://www.infoq.cn/feed",
"OSChina": "https://www.oschina.net/news/rss",
# English tech
"HackerNews": "https://hnrss.org/best?count=20",
"TechCrunch": "https://techcrunch.com/feed/",
"ArsTechnica": "https://feeds.arstechnica.com/arstechnica/index",
"TheVerge": "https://www.theverge.com/rss/index.xml",
"MIT Tech Review": "https://www.technologyreview.com/feed/",
"Wired": "https://www.wired.com/feed/rss",
"Lobste.rs": "https://lobste.rs/rss",
# English general
"ChinaDaily": "https://www.chinadaily.com.cn/rss/china_rss.xml",
}
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
t0 = time.time()
all_results = []
import asyncio
tasks = [self._fetch_feed(name, url, query)
for name, url in self.FEEDS.items()]
feeds = await asyncio.gather(*tasks, return_exceptions=True)
for feed_results in feeds:
if isinstance(feed_results, list):
all_results.extend(feed_results)
elapsed = time.time() - t0
logger.info(f"[rss_news] {len(all_results)} results in {elapsed:.2f}s")
return all_results[:max_results]
async def _fetch_feed(self, name: str, url: str, query: str) -> list[EngineResult]:
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=HEADERS)
resp.raise_for_status()
return self._parse_feed(name, resp.text, query)
except Exception as e:
logger.warning(f"[rss_news:{name}] failed: {e}")
return []
def _parse_feed(self, name: str, xml_text: str, query: str) -> list[EngineResult]:
try:
root = ET.fromstring(xml_text)
except ET.ParseError:
return []
results = []
# RSS 2.0: //channel/item, Atom: //entry
ns = {"atom": "http://www.w3.org/2005/Atom"}
items = root.findall(".//item") or root.findall(".//atom:entry", ns)
q_lower = query.lower()
q_words = [w for w in q_lower.split() if len(w) > 1]
for item in items[:50]:
title = (item.findtext("title") or
item.findtext("atom:title", namespaces=ns) or "")
link = (item.findtext("link") or "")
if not link:
link_el = item.find("atom:link", ns)
link = link_el.get("href", "") if link_el is not None else ""
desc = (item.findtext("description") or
item.findtext("atom:summary", namespaces=ns) or "")
# Clean HTML tags from description
import re
desc = re.sub(r'<[^>]+>', '', desc)
# Keyword scoring (not strict filter)
text_lower = f"{title} {desc}".lower()
if q_words:
match_count = sum(1 for w in q_words if w in text_lower)
if match_count == 0:
continue # Skip completely irrelevant items
results.append(EngineResult(
title=title.strip(),
url=link.strip(),
snippet=desc[:300].strip(),
source=f"rss_{name}",
category="news",
))
return results
class NewsSearch(BaseEngine):
"""Unified news search — Bing News + RSS in parallel."""
name = "news"
async def search(self, query: str, max_results: int = 10,
region: str = "cn", **kwargs) -> list[EngineResult]:
import asyncio
bing = BingNewsEngine(timeout=self.timeout)
rss = RSSNewsEngine(timeout=8.0) # RSS feeds may be slow
bing_results, rss_results = await asyncio.gather(
bing.search(query, max_results=max_results, region=region),
rss.search(query, max_results=max_results),
return_exceptions=True,
)
all_results = []
if isinstance(bing_results, list):
all_results.extend(bing_results)
if isinstance(rss_results, list):
all_results.extend(rss_results)
# Deduplicate by URL
seen = set()
deduped = []
for r in all_results:
if r.url not in seen:
seen.add(r.url)
deduped.append(r)
return deduped[:max_results]
+199
View File
@@ -0,0 +1,199 @@
"""Parallel search orchestrator — runs multiple engines concurrently."""
from __future__ import annotations
import asyncio
import time
import logging
from dataclasses import dataclass, field
from typing import Any
from scripts.engines.base import BaseEngine, EngineResult
from scripts.engines.bing_scraper import BingScraper
from scripts.engines.sogou_scraper import SogouScraper
from scripts.engines.baidu_scraper import BaiduScraper
from scripts.engines.search360_scraper import Search360Scraper
from scripts.engines.news_engine import NewsSearch
from scripts.engines.academic_engine import AcademicSearch
from scripts.engines.github_engine import GitHubEngine
from scripts.core.execution_log import execution_log
logger = logging.getLogger(__name__)
@dataclass
class SearchStats:
"""Search statistics."""
total_engines: int = 0
successful_engines: int = 0
failed_engines: list[str] = field(default_factory=list)
total_results: int = 0
deduplicated: int = 0
duration_ms: int = 0
engine_timings: dict[str, float] = field(default_factory=dict)
class ParallelSearch:
"""Run multiple engines in parallel and merge results."""
# Default engine sets
GENERAL_ENGINES = ["bing", "sogou", "baidu", "360"]
VERTICAL_ENGINES = ["news", "academic", "github"]
def __init__(self, timeout: float = 10.0):
self.timeout = timeout
self._engines: dict[str, BaseEngine] = {
"bing": BingScraper(timeout=timeout),
"sogou": SogouScraper(timeout=timeout),
"baidu": BaiduScraper(timeout=timeout),
"360": Search360Scraper(timeout=timeout),
"news": NewsSearch(timeout=timeout),
"academic": AcademicSearch(timeout=timeout),
"github": GitHubEngine(timeout=timeout),
}
async def search(
self,
query: str,
engines: list[str] | None = None,
max_results: int = 10,
region: str = "cn",
**kwargs,
) -> tuple[list[EngineResult], SearchStats]:
"""
Search across specified engines in parallel.
Args:
query: Search query
engines: List of engine names to use (default: all general)
max_results: Max results per engine
region: 'cn' or 'global'
**kwargs: Extra params passed to engines
Returns:
(results, stats)
"""
if engines is None:
engines = self.GENERAL_ENGINES.copy()
t0 = time.time()
stats = SearchStats(total_engines=len(engines))
# Build tasks
tasks = {}
for name in engines:
engine = self._engines.get(name)
if engine is None:
logger.warning(f"Unknown engine: {name}")
continue
kwargs_extra = dict(kwargs)
if name in ("bing", "bing_news"):
kwargs_extra["region"] = region
tasks[name] = self._run_engine(engine, query, max_results, kwargs_extra)
# Run all in parallel
results_map = await asyncio.gather(
*tasks.values(), return_exceptions=True
)
all_results = []
for name, result in zip(tasks.keys(), results_map):
if isinstance(result, Exception):
stats.failed_engines.append(name)
logger.warning(f"[{name}] exception: {result}")
# 记录异常到 execution_log
execution_log.record(execution_log._logs[-1].__class__(
engine=name,
operation="search",
success=False,
error=str(result),
error_type=type(result).__name__,
duration_ms=int((time.time() - t0) * 1000),
))
elif isinstance(result, tuple):
engine_results, elapsed = result
stats.engine_timings[name] = elapsed
if engine_results:
stats.successful_engines += 1
all_results.extend(engine_results)
else:
stats.failed_engines.append(name)
else:
stats.failed_engines.append(name)
# Deduplicate by URL
seen_urls = set()
deduped = []
for r in all_results:
key = r.url or r.title # fallback to title if no URL
if key not in seen_urls:
seen_urls.add(key)
deduped.append(r)
stats.total_results = len(all_results)
stats.deduplicated = len(deduped)
stats.duration_ms = int((time.time() - t0) * 1000)
return deduped, stats
async def _run_engine(
self, engine: BaseEngine, query: str, max_results: int, kwargs: dict
) -> tuple[list[EngineResult], float]:
"""Run a single engine with timing."""
with execution_log.track(engine.name, "search", query=query) as ctx:
t0 = time.time()
try:
results = await engine.search(
query, max_results=max_results, **kwargs
)
elapsed = time.time() - t0
ctx.result_count = len(results)
return results, elapsed
except Exception as e:
elapsed = time.time() - t0
ctx.success = False
ctx.error = str(e)
ctx.error_type = type(e).__name__
raise
async def search_vertical(
self,
query: str,
vertical: str,
max_results: int = 10,
**kwargs,
) -> tuple[list[EngineResult], SearchStats]:
"""Search a specific vertical (news/academic/github)."""
if vertical == "github":
# GitHub supports sub-types
search_type = kwargs.pop("search_type", "repositories")
engine = self._engines["github"]
with execution_log.track("github", "search",
search_type=search_type) as ctx:
t0 = time.time()
results = await engine.search(
query, max_results=max_results,
search_type=search_type, **kwargs
)
ctx.result_count = len(results)
stats = SearchStats(
total_engines=1,
successful_engines=1 if results else 0,
total_results=len(results),
duration_ms=int((time.time() - t0) * 1000),
)
return results, stats
else:
return await self.search(
query, engines=[vertical], max_results=max_results, **kwargs
)
async def health(self) -> dict[str, dict]:
"""Check health of all engines."""
results = {}
for name, engine in self._engines.items():
try:
results[name] = await engine.health()
except Exception as e:
results[name] = {"name": name, "status": "error", "error": str(e)}
return results
+66
View File
@@ -0,0 +1,66 @@
"""360 Search HTML scraping engine — free, good Chinese results."""
from __future__ import annotations
import time
import logging
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
logger = logging.getLogger(__name__)
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9",
}
class Search360Scraper(BaseEngine):
"""360 Search scraping — good Chinese results, fast in China."""
name = "360"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
t0 = time.time()
url = f"https://www.so.com/s?q={quote_plus(query)}"
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=HEADERS)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse(resp.text, max_results)
logger.info(f"[360] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[360] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
for item in soup.select("li.res-list"):
title_el = item.select_one("h3 a")
if not title_el:
continue
title = title_el.get_text(strip=True)
href = title_el.get("href", "")
snippet_el = item.select_one("p.res-desc, .res-rich")
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href:
results.append(EngineResult(
title=title, url=href, snippet=snippet, source="360",
))
if len(results) >= max_results:
break
return results
+66
View File
@@ -0,0 +1,66 @@
"""Sogou HTML scraping engine — free, good Chinese results."""
from __future__ import annotations
import time
import logging
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
logger = logging.getLogger(__name__)
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9",
}
class SogouScraper(BaseEngine):
"""Sogou web scraping — best for Chinese content."""
name = "sogou"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
t0 = time.time()
url = f"https://www.sogou.com/web?query={quote_plus(query)}"
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=HEADERS)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse(resp.text, max_results)
logger.info(f"[sogou] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[sogou] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
for item in soup.select("div.results > div"):
title_el = item.select_one("h3 a, a.account")
if not title_el:
continue
title = title_el.get_text(strip=True)
href = title_el.get("href", "")
snippet_el = item.select_one("p.str_info, div.str-text, .star-wiki")
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href and not href.startswith("/"):
results.append(EngineResult(
title=title, url=href, snippet=snippet, source="sogou",
))
if len(results) >= max_results:
break
return results
+87 -275
View File
@@ -1,297 +1,109 @@
""" """
Unified search entry point. Unified multi-source search — URL-based engines.
Provides a single interface to search across all configured sources. Architecture:
Primary: URL engines (Baidu, Bing, Google, DuckDuckGo, Sogou)
No external service dependency.
""" """
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import logging import time
from typing import Any, Optional from typing import Any, Optional
from scripts.core.base import SearchResult from dataclasses import dataclass
from scripts.core.config_loader import (
get_search_engines, get_news_sources, get_academic_sources,
get_wechat_sources, get_api_engines, get_special_engines,
get_social_sources, get_all_sources, get_engine,
)
from scripts.engines.url_engine import UrlSearchEngine
from scripts.engines.parser_engines import DuckDuckGoEngine
from scripts.news.cls import CLSNewsEngine
from scripts.news.wallstreetcn import WallStreetCNEngine
from scripts.news.rss import RSSNewsEngine
from scripts.news.tech_news import (
HackerNewsEngine, GitHubTrendingEngine, Kr36Engine,
WeiboEngine, V2EXEngine, TencentNewsEngine,
)
from scripts.academic.arxiv import ArXivEngine
from scripts.wechat.sogou_weixin import SogouWeChatEngine
from scripts.social.twitter import TwitterSearchEngine
from scripts.engines.baidu_qianfan import BaiduQianFanEngine
logger = logging.getLogger(__name__) from scripts.models import SearchQuery, SearchResponse, SourceRef, SearchMetadata, EngineTiming
# Default timeout per engine (seconds)
_DEFAULT_ENGINE_TIMEOUT = 8
# Max concurrent engine calls
_DEFAULT_MAX_CONCURRENT = 5
SOURCE_CATEGORIES = { @dataclass
"web": get_search_engines, class SearchResult:
"news": get_news_sources, title: str
"academic": get_academic_sources, url: str
"wechat": get_wechat_sources, content: str
"social": get_social_sources, source: str = ""
"api": get_api_engines, engine: str = ""
"special": get_special_engines, score: float = 0.0
} category: str = ""
published: str = ""
class UnifiedSearch: class UnifiedSearch:
"""Unified search client that aggregates results from multiple sources.""" """Unified search across multiple open-source engines."""
def __init__(self, config: Optional[dict[str, Any]] = None): def __init__(self):
self.config = config or {} self._start_time = 0.0
self._cache: dict[str, list[SearchResult]] = {}
self._engines: dict[str, Any] = {} async def search(self, query: str, max_results: int = 5,
self._timeout = self.config.get("engine_timeout", _DEFAULT_ENGINE_TIMEOUT) language: str = "zh-CN") -> SearchResponse:
self._max_concurrent = self.config.get( """Search using URL-based engines."""
"max_concurrent", _DEFAULT_MAX_CONCURRENT self._start_time = time.time()
metadata = SearchMetadata()
metadata.total_engines = 1
all_results, timings = await self._search_urls(query, max_results)
elapsed_ms = int((time.time() - self._start_time) * 1000)
metadata.duration_ms = elapsed_ms
metadata.engine_timings = timings
metadata.total_raw_results = len(all_results)
sources = []
seen_urls = set()
for i, r in enumerate(all_results[:max_results]):
if r.url and r.url not in seen_urls:
seen_urls.add(r.url)
sources.append(SourceRef(
index=i + 1,
title=r.title or r.content[:50],
url=r.url or "",
source=r.source,
))
metadata.engines_used = list(set(s.source for s in sources))
return SearchResponse(
query=query,
results=list(all_results[:max_results]),
sources=sources,
metadata=metadata,
) )
self._semaphore = asyncio.Semaphore(self._max_concurrent)
async def __aenter__(self): async def _search_urls(self, query: str, max_results: int
return self ) -> tuple[list[SearchResult], list[EngineTiming]]:
"""Build search URLs for major engines."""
timings = []
engines = {
"baidu": f"https://www.baidu.com/s?wd={query}",
"bing": f"https://cn.bing.com/search?q={query}",
"google": f"https://www.google.com/search?q={query}",
"duckduckgo": f"https://html.duckduckgo.com/html/?q={query}",
"sogou": f"https://www.sogou.com/web?query={query}",
}
results = []
for name, url in engines.items():
results.append(SearchResult(
title=f"{name}: {query}",
url=url,
content=f"Search {query} on {name}",
source=name,
))
async def __aexit__(self, *args): timings.append(EngineTiming(
await self.close() engine_name="url_fallback",
success=True,
duration_ms=1,
result_count=len(results),
))
return results, timings
async def search( async def health(self) -> dict:
self, query: str, sources: Optional[list[str]] = None, """Check status."""
max_results: int = 10, force_refresh: bool = False, **kwargs, return {
) -> list[SearchResult]: "status": "ok",
"""Search across selected sources with streaming + early termination. "engines": ["baidu", "bing", "google", "duckduckgo", "sogou"],
}
Returns results as soon as enough are collected, cancelling remaining
slow engines to minimize total wait time.
"""
if not sources or "all" in sources:
sources = list(SOURCE_CATEGORIES.keys())
cache_key = f"{sorted(sources) if isinstance(sources, list) else sources}:{query}"
if not force_refresh and cache_key in self._cache:
return self._cache[cache_key]
# --- Build task list --- #
engine_list: list[tuple[Any, str]] = [] # (engine, query)
for source_spec in sources:
if source_spec in SOURCE_CATEGORIES:
for engine_def in SOURCE_CATEGORIES[source_spec]():
engine = self._get_engine(engine_def)
if engine:
engine_list.append((engine, query))
else:
engine_def = get_engine(source_spec)
if engine_def:
engine = self._get_engine(engine_def)
if engine:
engine_list.append((engine, query))
if not engine_list:
return []
engine_count = len(engine_list)
# --- Streaming: as_completed + early termination --- #
seen_urls: set[str] = set()
all_results: list[SearchResult] = []
pending = set()
# Submit all tasks
for engine, q in engine_list:
coro = self._safe_search(engine, q, max_results, **kwargs)
pending.add(asyncio.create_task(coro))
enough_results = max_results * 2 # allow some slack for dedup
# Process as they complete
for done in asyncio.as_completed(pending):
try:
results = await done
if isinstance(results, (Exception, BaseException)):
continue
# Dedup on the fly
for r in results:
key = r.url.strip().rstrip("/")
if key and key not in seen_urls:
seen_urls.add(key)
all_results.append(r)
# Early termination: stop waiting for more engines
if len(all_results) >= enough_results:
break
except Exception:
continue
# Check early termination after each completion
if len(all_results) >= enough_results:
remaining = pending - {done}
for t in remaining:
t.cancel()
break
self._cache[cache_key] = all_results
return all_results
async def _safe_search(self, engine, query, max_results, **kwargs):
async with self._semaphore:
try:
return await asyncio.wait_for(
engine.search(query, max_results=max_results, **kwargs),
timeout=self._timeout,
)
except asyncio.TimeoutError:
logger.warning(f"Engine {engine.name} timed out after {self._timeout}s")
return []
except Exception as e:
logger.error(f"Engine {engine.name} failed: {e}")
return []
async def search_health(self) -> list[dict[str, Any]]:
"""Check health of all configured engines (concurrent)."""
seen_names: set[str] = set()
tasks: list[asyncio.Task] = []
for cat_fn in SOURCE_CATEGORIES.values():
for engine_def in cat_fn():
name = engine_def.get("name", "?")
if name in seen_names:
continue
seen_names.add(name)
tasks.append(asyncio.create_task(self._check_one_engine(engine_def, name)))
results = await asyncio.gather(*tasks, return_exceptions=True)
statuses: list[dict[str, Any]] = []
for r in results:
if isinstance(r, Exception):
statuses.append({"name": "?", "status": "error", "detail": str(r)[:80]})
else:
statuses.append(r)
return statuses
async def _check_one_engine(
self, engine_def: dict[str, Any], name: str,
) -> dict[str, Any]:
"""Check health of a single engine."""
engine = self._get_engine(engine_def)
if engine is None:
return {"name": name, "status": "error", "detail": "engine_init_failed"}
try:
async with self._semaphore:
await asyncio.wait_for(
engine.search("health check", max_results=1),
timeout=8,
)
return {"name": name, "status": "ok"}
except asyncio.TimeoutError:
return {"name": name, "status": "timeout"}
except Exception as e:
return {"name": name, "status": "error", "detail": str(e)[:80]}
def _get_engine(self, engine_def: dict[str, Any]):
"""Get or create an engine instance by definition."""
name = engine_def.get("name", "")
engine_type = engine_def.get("type", "url")
if name in self._engines:
return self._engines[name]
if name == "DuckDuckGo":
engine = DuckDuckGoEngine(self.config)
elif engine_type == "baidu_qianfan" or "baidu qianfan" in name.lower():
engine = BaiduQianFanEngine(self.config)
elif "cls" in name.lower() or "财联社" in name:
engine = CLSNewsEngine(self.config)
elif "wallstreet" in name.lower() or "华尔街" in name:
engine = WallStreetCNEngine(self.config)
elif name == "ArXiv" or engine_type == "api":
engine = ArXivEngine(self.config)
elif "微信" in name or "wechat" in name.lower() or "weixin" in name.lower():
engine = SogouWeChatEngine(self.config)
elif "twitter" in name.lower():
engine = TwitterSearchEngine(self.config)
elif name == "Hacker News":
engine = HackerNewsEngine(self.config)
elif name == "GitHub Trending":
engine = GitHubTrendingEngine(self.config)
elif name == "36氪":
engine = Kr36Engine(self.config)
elif name == "微博热搜":
engine = WeiboEngine(self.config)
elif name == "V2EX":
engine = V2EXEngine(self.config)
elif name == "腾讯新闻":
engine = TencentNewsEngine(self.config)
elif engine_type == "news" and name not in ("cls", "wallstreetcn", "Hacker News", "GitHub Trending", "36氪", "微博热搜", "V2EX", "腾讯新闻"):
engine = RSSNewsEngine(self.config)
elif engine_type == "social":
engine = TwitterSearchEngine(self.config) if "twitter" in name.lower() else UrlSearchEngine(name, engine_def)
else:
engine = UrlSearchEngine(name, engine_def)
self._engines[name] = engine
return engine
async def search_engines(self, query, region=None, max_results=10):
"""Quick search across web search engines."""
engines = get_search_engines()
if region:
engines = [e for e in engines if e.get("region") == region]
return await self.search(query, sources=[e["name"] for e in engines], max_results=max_results)
async def search_news(self, query, max_results=10, **kwargs):
"""Quick search news sources (CN news + RSS feeds)."""
return await self.search(query, sources=["news"], max_results=max_results, **kwargs)
async def search_academic(self, query, max_results=10, **kwargs):
"""Quick search academic papers."""
return await self.search(query, sources=["academic"], max_results=max_results, **kwargs)
async def search_wechat(self, query, max_results=10):
"""Quick search WeChat articles."""
return await self.search(query, sources=["wechat"], max_results=max_results)
async def search_social(self, query, max_results=10, **kwargs):
"""Quick search social media (Twitter/X)."""
return await self.search(query, sources=["social"], max_results=max_results, **kwargs)
async def search_rss(self, query="", max_results=10, **kwargs):
"""Quick search RSS news feeds."""
engine = RSSNewsEngine(self.config)
return await engine.search(query, max_results=max_results, **kwargs)
async def get_search_urls(self, query, sources=None):
"""Get search URLs without executing searches."""
if not sources:
sources = [e["name"] for e in get_search_engines()]
urls = {}
for name in sources:
engine_def = get_engine(name)
if engine_def:
urls[name] = UrlSearchEngine(name, engine_def).build_url(query)
return urls
async def get_twitter_search_url(self, query, search_type="top"):
"""Get Twitter/X search URL for browser/web_fetch usage."""
engine = TwitterSearchEngine(self.config)
return engine.build_url(query, search_type=search_type)
async def close(self): async def close(self):
"""Close all engine resources.""" pass
for engine in self._engines.values():
try:
await engine.close()
except Exception:
pass
+21
View File
@@ -0,0 +1,21 @@
from setuptools import setup, find_packages
setup(
name="web-search-skills",
version="0.2.0",
description="Open-source diversified search in <5s",
author="openbot-coder",
packages=find_packages(),
py_modules=["ws"],
python_requires=">=3.10",
install_requires=[
"httpx>=0.24.0",
"beautifulsoup4>=4.12.0",
"lxml>=5.0.0",
],
entry_points={
"console_scripts": [
"web-search=ws:main",
],
},
)
+79 -12
View File
@@ -1,23 +1,90 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
Web Search Skills CLI entry point. web-search-skills: Open-source diversified search in <5s.
Usage: Usage:
python ws.py <command> <query> [options] python3 ws.py search <query> [--max-results 5]
python3 ws.py health
Or install and run: python3 ws.py serve # (future) MCP server mode
uv tool install -e .
ws <command> <query> [options]
""" """
from __future__ import annotations
import argparse
import asyncio
import json
import sys import sys
import os import time
# Ensure scripts package is importable
_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
if _SCRIPT_DIR not in sys.path:
sys.path.insert(0, _SCRIPT_DIR)
from scripts.cli import main async def cmd_search(args: argparse.Namespace):
from scripts.unified_search import UnifiedSearch
us = UnifiedSearch()
t0 = time.time()
resp = await us.search(args.query, max_results=args.max_results)
elapsed = time.time() - t0
if args.json:
output = {
"query": resp.query,
"sources": [
{
"index": s.index,
"title": s.title,
"url": s.url,
"source": s.source,
}
for s in resp.sources
],
"time_ms": elapsed * 1000,
}
print(json.dumps(output, ensure_ascii=False, indent=2))
else:
print(f"\n🔍 「{resp.query}」 — {elapsed:.2f}s")
print(f" {len(resp.sources)} results from {len(resp.metadata.engines_used)} engines\n")
for i, s in enumerate(resp.sources, 1):
print(f" {i}. [{s.source}] {s.title}")
print(f" {s.url}")
print()
await us.close()
async def cmd_health(args: argparse.Namespace):
from scripts.unified_search import UnifiedSearch
us = UnifiedSearch()
status = await us.health()
if args.json:
print(json.dumps(status, ensure_ascii=False, indent=2))
else:
print("📊 Search Health")
print(f" Status: {status.get('status', 'unknown')}")
print(f" Engines: {', '.join(status.get('engines', []))}")
await us.close()
def main():
parser = argparse.ArgumentParser(description="web-search-skills")
sub = parser.add_subparsers(dest="command", required=True)
# search
sp = sub.add_parser("search", help="Search across multiple sources")
sp.add_argument("query", help="Search query")
sp.add_argument("--max-results", type=int, default=5)
sp.add_argument("--json", action="store_true", help="JSON output")
# health
hp = sub.add_parser("health", help="Check engine health")
hp.add_argument("--json", action="store_true", help="JSON output")
args = parser.parse_args()
if args.command == "search":
asyncio.run(cmd_search(args))
elif args.command == "health":
asyncio.run(cmd_health(args))
if __name__ == "__main__": if __name__ == "__main__":
main() main()