feat: 优化搜索广度、深度、效率及可靠性
- 新增百度千帆 AI 搜索 API 引擎 (scripts/engines/baidu_qianfan.py) - CLI 新增 baidu 子命令,支持 --site 站点限定和 --recency 时间过滤 - UnifiedSearch: 引擎并发控制 (Semaphore) + 超时熔断 (wait_for) - UnifiedSearch: 搜索结果按 URL 去重 - CLI 新增 health 健康检查命令 - sources 命令新增 API 引擎类别展示 - 支持 .env 文件读取 BAIDU_API_KEY
This commit is contained in:
1 parent
87935b59ce
commit
7eb1467e07
5 files changed
+322
-17
No files matched your search
@@ -15,6 +15,9 @@
|
|||||||
{"name": "Ecosia", "url": "https://www.ecosia.org/search?q={keyword}", "region": "global", "type": "url"},
|
{"name": "Ecosia", "url": "https://www.ecosia.org/search?q={keyword}", "region": "global", "type": "url"},
|
||||||
{"name": "Qwant", "url": "https://www.qwant.com/?q={keyword}", "region": "global", "type": "url"}
|
{"name": "Qwant", "url": "https://www.qwant.com/?q={keyword}", "region": "global", "type": "url"}
|
||||||
],
|
],
|
||||||
|
"api_engines": [
|
||||||
|
{"name": "Baidu Qianfan", "type": "baidu_qianfan", "region": "cn"}
|
||||||
|
],
|
||||||
"special_engines": [
|
"special_engines": [
|
||||||
{"name": "WolframAlpha", "url": "https://www.wolframalpha.com/input?i={keyword}", "region": "global", "type": "url"}
|
{"name": "WolframAlpha", "url": "https://www.wolframalpha.com/input?i={keyword}", "region": "global", "type": "url"}
|
||||||
],
|
],
|
||||||
|
|||||||
+65
-5
@@ -8,11 +8,13 @@ Usage:
|
|||||||
Commands:
|
Commands:
|
||||||
search Search all sources (default)
|
search Search all sources (default)
|
||||||
web Search web search engines (Google, Baidu, DuckDuckGo, etc.)
|
web Search web search engines (Google, Baidu, DuckDuckGo, etc.)
|
||||||
|
baidu Search via Baidu Qianfan AI Search API (需配置 BAIDU_API_KEY)
|
||||||
news Search news (财联社, 华尔街见闻, RSS)
|
news Search news (财联社, 华尔街见闻, RSS)
|
||||||
wechat Search WeChat public account articles
|
wechat Search WeChat public account articles (via Sougou)
|
||||||
academic Search academic papers (ArXiv)
|
academic Search academic papers (ArXiv)
|
||||||
social Search social media (Twitter/X)
|
social Search social media (Twitter/X)
|
||||||
rss Search RSS news feeds
|
rss Search RSS news feeds
|
||||||
|
health Test engine connectivity
|
||||||
sources List all available sources
|
sources List all available sources
|
||||||
urls Get search URLs without executing
|
urls Get search URLs without executing
|
||||||
|
|
||||||
@@ -20,16 +22,21 @@ Options:
|
|||||||
-n, --max-results N Max results per source (default: 10)
|
-n, --max-results N Max results per source (default: 10)
|
||||||
-j, --json Output as JSON
|
-j, --json Output as JSON
|
||||||
-o, --output FILE Save results to file
|
-o, --output FILE Save results to file
|
||||||
-s, --source NAME Specific source name
|
-s, --source NAME Specific source name(s)
|
||||||
-r, --region REGION Region filter (cn/global)
|
-r, --region REGION Region filter (cn/global)
|
||||||
|
--site SITE Restrict to specific site (baidu command only, e.g. mp.weixin.qq.com)
|
||||||
|
--recency RECENCY Time filter: day, week, month, semiyear, year (baidu command only)
|
||||||
-v, --verbose Show debug logs
|
-v, --verbose Show debug logs
|
||||||
-h, --help Show this help
|
-h, --help Show this help
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
web-search wechat 蒙面财经
|
web-search wechat 蒙面财经
|
||||||
|
web-search baidu 今日新闻 --recency week
|
||||||
|
web-search baidu "央视新闻 中美元首会晤" --site mp.weixin.qq.com --recency week
|
||||||
web-search news "人工智能" -n 5
|
web-search news "人工智能" -n 5
|
||||||
web-search web "Python async" -r global -j
|
web-search web "Python async" -r global -j
|
||||||
web-search search "量子计算" -o results.json
|
web-search search "量子计算" -o results.json
|
||||||
|
web-search health
|
||||||
web-search sources
|
web-search sources
|
||||||
web-search urls "机器学习"
|
web-search urls "机器学习"
|
||||||
"""
|
"""
|
||||||
@@ -226,6 +233,15 @@ async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]:
|
|||||||
await us.close()
|
await us.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def _search_baidu(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||||
|
from scripts.engines.baidu_qianfan import BaiduQianFanEngine
|
||||||
|
engine = BaiduQianFanEngine({"api_key": os.getenv("BAIDU_API_KEY", "")})
|
||||||
|
try:
|
||||||
|
return await engine.search(query, max_results=max_results, **kwargs)
|
||||||
|
finally:
|
||||||
|
await engine.close()
|
||||||
|
|
||||||
|
|
||||||
async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]:
|
async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]:
|
||||||
us = _get_unified()()
|
us = _get_unified()()
|
||||||
try:
|
try:
|
||||||
@@ -258,6 +274,7 @@ def cmd_sources(args):
|
|||||||
("academic_sources", "学术源"),
|
("academic_sources", "学术源"),
|
||||||
("wechat_sources", "微信源"),
|
("wechat_sources", "微信源"),
|
||||||
("social_sources", "社交源"),
|
("social_sources", "社交源"),
|
||||||
|
("api_engines", "API 引擎"),
|
||||||
("special_engines", "特殊源"),
|
("special_engines", "特殊源"),
|
||||||
]:
|
]:
|
||||||
count = categories.get(cat_key, 0)
|
count = categories.get(cat_key, 0)
|
||||||
@@ -280,6 +297,26 @@ def cmd_urls(args):
|
|||||||
print()
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
async def cmd_health(args):
|
||||||
|
"""Test all engine connectivity."""
|
||||||
|
us = _get_unified()()
|
||||||
|
try:
|
||||||
|
print("正在检测各引擎连通性...\n")
|
||||||
|
statuses = await us.search_health()
|
||||||
|
ok_count = sum(1 for s in statuses if s["status"] == "ok")
|
||||||
|
for s in statuses:
|
||||||
|
if s["status"] == "ok":
|
||||||
|
print(f" ✅ {s['name']}")
|
||||||
|
elif s["status"] == "timeout":
|
||||||
|
print(f" ⏱️ {s['name']} 超时")
|
||||||
|
else:
|
||||||
|
detail = s.get("detail", "")
|
||||||
|
print(f" ❌ {s['name']} {detail}")
|
||||||
|
print(f"\n总计: {len(statuses)} 个引擎, {ok_count} 个可用")
|
||||||
|
finally:
|
||||||
|
await us.close()
|
||||||
|
|
||||||
|
|
||||||
async def _run_search(args):
|
async def _run_search(args):
|
||||||
"""Route search to the right engine(s)."""
|
"""Route search to the right engine(s)."""
|
||||||
if args.verbose:
|
if args.verbose:
|
||||||
@@ -288,6 +325,7 @@ async def _run_search(args):
|
|||||||
method_map = {
|
method_map = {
|
||||||
"search": _search_all,
|
"search": _search_all,
|
||||||
"web": _search_web,
|
"web": _search_web,
|
||||||
|
"baidu": _search_baidu,
|
||||||
"news": _search_news,
|
"news": _search_news,
|
||||||
"wechat": _search_wechat,
|
"wechat": _search_wechat,
|
||||||
"academic": _search_academic,
|
"academic": _search_academic,
|
||||||
@@ -300,6 +338,11 @@ async def _run_search(args):
|
|||||||
kwargs = {}
|
kwargs = {}
|
||||||
if args.command == "web" and args.region:
|
if args.command == "web" and args.region:
|
||||||
kwargs["region"] = args.region
|
kwargs["region"] = args.region
|
||||||
|
if args.command == "baidu":
|
||||||
|
if args.site:
|
||||||
|
kwargs["site"] = args.site
|
||||||
|
if args.recency:
|
||||||
|
kwargs["recency"] = args.recency
|
||||||
|
|
||||||
try:
|
try:
|
||||||
results = await search_fn(
|
results = await search_fn(
|
||||||
@@ -318,6 +361,7 @@ async def _run_search(args):
|
|||||||
cmd_names = {
|
cmd_names = {
|
||||||
"search": "全源搜索",
|
"search": "全源搜索",
|
||||||
"web": "网页搜索",
|
"web": "网页搜索",
|
||||||
|
"baidu": "百度千帆 AI 搜索",
|
||||||
"news": "新闻搜索",
|
"news": "新闻搜索",
|
||||||
"wechat": "微信公众号搜索",
|
"wechat": "微信公众号搜索",
|
||||||
"academic": "学术搜索",
|
"academic": "学术搜索",
|
||||||
@@ -376,8 +420,8 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
"command",
|
"command",
|
||||||
nargs="?",
|
nargs="?",
|
||||||
default="search",
|
default="search",
|
||||||
choices=["search", "web", "news", "wechat", "academic", "social",
|
choices=["search", "web", "baidu", "news", "wechat", "academic",
|
||||||
"rss", "sources", "urls"],
|
"social", "rss", "health", "sources", "urls"],
|
||||||
help="Search type (default: search)",
|
help="Search type (default: search)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
@@ -414,6 +458,18 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
choices=["cn", "global"],
|
choices=["cn", "global"],
|
||||||
help="Region filter (web search only)",
|
help="Region filter (web search only)",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--site",
|
||||||
|
type=str,
|
||||||
|
help="Restrict to specific site (baidu command only, e.g. mp.weixin.qq.com)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--recency",
|
||||||
|
type=str,
|
||||||
|
choices=["day", "week", "month", "semiyear", "year"],
|
||||||
|
default=None,
|
||||||
|
help="Time filter (baidu command only, default: year)",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"-v", "--verbose",
|
"-v", "--verbose",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -434,11 +490,15 @@ def main():
|
|||||||
if args.verbose:
|
if args.verbose:
|
||||||
logging.getLogger().setLevel(logging.DEBUG)
|
logging.getLogger().setLevel(logging.DEBUG)
|
||||||
|
|
||||||
# sources and urls commands don't need a query
|
# sources, urls, health don't need a query
|
||||||
if args.command == "sources":
|
if args.command == "sources":
|
||||||
cmd_sources(args)
|
cmd_sources(args)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if args.command == "health":
|
||||||
|
asyncio.run(cmd_health(args))
|
||||||
|
return
|
||||||
|
|
||||||
if args.command == "urls":
|
if args.command == "urls":
|
||||||
if not args.query:
|
if not args.query:
|
||||||
print("错误: urls 命令需要提供搜索关键词", file=sys.stderr)
|
print("错误: urls 命令需要提供搜索关键词", file=sys.stderr)
|
||||||
|
|||||||
@@ -38,6 +38,11 @@ def get_social_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
|
|||||||
return _load_config(path).get("social_sources", [])
|
return _load_config(path).get("social_sources", [])
|
||||||
|
|
||||||
|
|
||||||
|
def get_api_engines(path: Optional[str] = None) -> list[dict[str, Any]]:
|
||||||
|
"""Get API-based search engines (Baidu Qianfan, etc.)."""
|
||||||
|
return _load_config(path).get("api_engines", [])
|
||||||
|
|
||||||
|
|
||||||
def get_special_engines(path: Optional[str] = None) -> list[dict[str, Any]]:
|
def get_special_engines(path: Optional[str] = None) -> list[dict[str, Any]]:
|
||||||
return _load_config(path).get("special_engines", [])
|
return _load_config(path).get("special_engines", [])
|
||||||
|
|
||||||
@@ -45,7 +50,8 @@ def get_special_engines(path: Optional[str] = None) -> list[dict[str, Any]]:
|
|||||||
def get_engine(name: str, path: Optional[str] = None):
|
def get_engine(name: str, path: Optional[str] = None):
|
||||||
config = _load_config(path)
|
config = _load_config(path)
|
||||||
for category in ["search_engines", "news_sources", "academic_sources",
|
for category in ["search_engines", "news_sources", "academic_sources",
|
||||||
"wechat_sources", "social_sources", "special_engines"]:
|
"wechat_sources", "social_sources", "api_engines",
|
||||||
|
"special_engines"]:
|
||||||
for engine in config.get(category, []):
|
for engine in config.get(category, []):
|
||||||
if engine["name"].lower() == name.lower():
|
if engine["name"].lower() == name.lower():
|
||||||
return engine
|
return engine
|
||||||
@@ -56,7 +62,8 @@ def get_all_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
|
|||||||
config = _load_config(path)
|
config = _load_config(path)
|
||||||
all_sources = []
|
all_sources = []
|
||||||
for category in ["search_engines", "news_sources", "academic_sources",
|
for category in ["search_engines", "news_sources", "academic_sources",
|
||||||
"wechat_sources", "social_sources", "special_engines"]:
|
"wechat_sources", "social_sources", "api_engines",
|
||||||
|
"special_engines"]:
|
||||||
all_sources.extend(config.get(category, []))
|
all_sources.extend(config.get(category, []))
|
||||||
return all_sources
|
return all_sources
|
||||||
|
|
||||||
@@ -65,6 +72,7 @@ def list_source_categories(path: Optional[str] = None) -> dict[str, int]:
|
|||||||
config = _load_config(path)
|
config = _load_config(path)
|
||||||
categories = [
|
categories = [
|
||||||
"search_engines", "news_sources", "academic_sources",
|
"search_engines", "news_sources", "academic_sources",
|
||||||
"wechat_sources", "social_sources", "special_engines",
|
"wechat_sources", "social_sources", "api_engines",
|
||||||
|
"special_engines",
|
||||||
]
|
]
|
||||||
return {cat: len(config.get(cat, [])) for cat in categories}
|
return {cat: len(config.get(cat, [])) for cat in categories}
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""
|
||||||
|
Baidu Qianfan AI Search API engine.
|
||||||
|
|
||||||
|
Uses the official Baidu Qianfan (千帆) AI Search API to get
|
||||||
|
structured search results. Requires BAIDU_API_KEY environment variable
|
||||||
|
or .env file in the project root.
|
||||||
|
|
||||||
|
API docs: https://console.bce.baidu.com/qianfan/ais/console/apiKey
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from scripts.core.base import SearchEngine, SearchResult
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
API_URL = "https://qianfan.baidubce.com/v2/ai_search/web_search"
|
||||||
|
ENV_KEY = "BAIDU_API_KEY"
|
||||||
|
|
||||||
|
|
||||||
|
def _load_env_file() -> dict[str, str]:
|
||||||
|
"""Load .env file from project root (simple parser, no external deps)."""
|
||||||
|
env_path = Path(__file__).resolve().parent.parent.parent / ".env"
|
||||||
|
if not env_path.exists():
|
||||||
|
return {}
|
||||||
|
env_vars = {}
|
||||||
|
with open(env_path, "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
if not line or line.startswith("#") or "=" not in line:
|
||||||
|
continue
|
||||||
|
key, _, val = line.partition("=")
|
||||||
|
env_vars[key.strip()] = val.strip().strip("\"'")
|
||||||
|
return env_vars
|
||||||
|
|
||||||
|
|
||||||
|
def _get_api_key() -> str:
|
||||||
|
"""Get API key: env var > .env file > config."""
|
||||||
|
key = os.getenv(ENV_KEY)
|
||||||
|
if key:
|
||||||
|
return key
|
||||||
|
env_from_file = _load_env_file()
|
||||||
|
return env_from_file.get(ENV_KEY, "")
|
||||||
|
|
||||||
|
|
||||||
|
class BaiduQianFanEngine(SearchEngine):
|
||||||
|
"""百度千帆 AI 搜索 API 引擎."""
|
||||||
|
|
||||||
|
def __init__(self, config: Optional[dict] = None):
|
||||||
|
super().__init__("Baidu Qianfan", config)
|
||||||
|
self.api_key = (config or {}).get("api_key", "") or _get_api_key()
|
||||||
|
if not self.api_key:
|
||||||
|
logger.warning(
|
||||||
|
f"BAIDU_API_KEY not set! Set it in .env file or as environment variable."
|
||||||
|
)
|
||||||
|
|
||||||
|
timeout = (config or {}).get("timeout", 30)
|
||||||
|
self._client = httpx.AsyncClient(
|
||||||
|
timeout=timeout,
|
||||||
|
headers={
|
||||||
|
"Authorization": f"Bearer {self.api_key}",
|
||||||
|
"X-Appbuilder-From": "openclaw",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
follow_redirects=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def search(
|
||||||
|
self, query: str, max_results: int = 10, **kwargs
|
||||||
|
) -> list[SearchResult]:
|
||||||
|
"""Search via Baidu Qianfan AI Search API.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search keywords.
|
||||||
|
max_results: Max results to return.
|
||||||
|
**kwargs:
|
||||||
|
site: Restrict search to a specific site (e.g. "mp.weixin.qq.com").
|
||||||
|
recency: Time filter: "week", "month", "semiyear", "year" (default: "year").
|
||||||
|
edition: "standard" (default) or "lite".
|
||||||
|
safe_search: bool, default False.
|
||||||
|
"""
|
||||||
|
if not self.api_key:
|
||||||
|
return [
|
||||||
|
SearchResult(
|
||||||
|
title="错误: BAIDU_API_KEY 未配置",
|
||||||
|
url="https://console.bce.baidu.com/qianfan/ais/console/apiKey",
|
||||||
|
snippet="请先设置 BAIDU_API_KEY 环境变量",
|
||||||
|
source=self.name,
|
||||||
|
rank=1,
|
||||||
|
category="web",
|
||||||
|
extra={"error": "missing_api_key"},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
# Build resource_type_filter based on max_results
|
||||||
|
top_k = min(max_results, 50)
|
||||||
|
resource_type_filter = [
|
||||||
|
{"type": "web", "top_k": top_k},
|
||||||
|
]
|
||||||
|
|
||||||
|
# Build search_filter if site is specified
|
||||||
|
search_filter = {}
|
||||||
|
site = kwargs.get("site")
|
||||||
|
if site:
|
||||||
|
search_filter["match"] = {"site": [site]}
|
||||||
|
|
||||||
|
request_body = {
|
||||||
|
"messages": [{"content": query, "role": "user"}],
|
||||||
|
"edition": kwargs.get("edition", "standard"),
|
||||||
|
"search_source": "baidu_search_v2",
|
||||||
|
"resource_type_filter": resource_type_filter,
|
||||||
|
"search_recency_filter": kwargs.get("recency", "year"),
|
||||||
|
"safe_search": kwargs.get("safe_search", False),
|
||||||
|
}
|
||||||
|
if search_filter:
|
||||||
|
request_body["search_filter"] = search_filter
|
||||||
|
if kwargs.get("block_websites"):
|
||||||
|
request_body["block_websites"] = kwargs["block_websites"]
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = await self._client.post(API_URL, json=request_body)
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
if "code" in data:
|
||||||
|
logger.error(f"Baidu Qianfan API error: {data.get('message', '')}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
references = data.get("references", [])
|
||||||
|
results = []
|
||||||
|
for i, ref in enumerate(references[:max_results]):
|
||||||
|
results.append(
|
||||||
|
SearchResult(
|
||||||
|
title=ref.get("title", ""),
|
||||||
|
url=ref.get("url", ""),
|
||||||
|
snippet=ref.get("snippet", ""),
|
||||||
|
source=self.name,
|
||||||
|
rank=i + 1,
|
||||||
|
category="web",
|
||||||
|
extra={
|
||||||
|
k: ref[k]
|
||||||
|
for k in ("site", "date", "type", "source")
|
||||||
|
if k in ref
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return results
|
||||||
|
|
||||||
|
except httpx.HTTPStatusError as e:
|
||||||
|
logger.error(f"Baidu Qianfan API HTTP error: {e.response.status_code}")
|
||||||
|
if e.response.status_code == 401:
|
||||||
|
logger.error("API Key 无效或未授权,请检查 BAIDU_API_KEY")
|
||||||
|
elif e.response.status_code == 429:
|
||||||
|
logger.error("API 配额不足")
|
||||||
|
return []
|
||||||
|
except httpx.RequestError as e:
|
||||||
|
logger.error(f"Baidu Qianfan API request failed: {e}")
|
||||||
|
return []
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
logger.error(f"Baidu Qianfan API response parse failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
await self._client.aclose()
|
||||||
@@ -13,8 +13,8 @@ from typing import Any, Optional
|
|||||||
from scripts.core.base import SearchResult
|
from scripts.core.base import SearchResult
|
||||||
from scripts.core.config_loader import (
|
from scripts.core.config_loader import (
|
||||||
get_search_engines, get_news_sources, get_academic_sources,
|
get_search_engines, get_news_sources, get_academic_sources,
|
||||||
get_wechat_sources, get_special_engines, get_social_sources,
|
get_wechat_sources, get_api_engines, get_special_engines,
|
||||||
get_all_sources, get_engine,
|
get_social_sources, get_all_sources, get_engine,
|
||||||
)
|
)
|
||||||
from scripts.engines.url_engine import UrlSearchEngine
|
from scripts.engines.url_engine import UrlSearchEngine
|
||||||
from scripts.engines.parser_engines import DuckDuckGoEngine
|
from scripts.engines.parser_engines import DuckDuckGoEngine
|
||||||
@@ -24,15 +24,22 @@ from scripts.news.rss import RSSNewsEngine
|
|||||||
from scripts.academic.arxiv import ArXivEngine
|
from scripts.academic.arxiv import ArXivEngine
|
||||||
from scripts.wechat.sogou_weixin import SogouWeChatEngine
|
from scripts.wechat.sogou_weixin import SogouWeChatEngine
|
||||||
from scripts.social.twitter import TwitterSearchEngine
|
from scripts.social.twitter import TwitterSearchEngine
|
||||||
|
from scripts.engines.baidu_qianfan import BaiduQianFanEngine
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Default timeout per engine (seconds)
|
||||||
|
_DEFAULT_ENGINE_TIMEOUT = 15
|
||||||
|
# Max concurrent engine calls
|
||||||
|
_DEFAULT_MAX_CONCURRENT = 5
|
||||||
|
|
||||||
SOURCE_CATEGORIES = {
|
SOURCE_CATEGORIES = {
|
||||||
"web": get_search_engines,
|
"web": get_search_engines,
|
||||||
"news": get_news_sources,
|
"news": get_news_sources,
|
||||||
"academic": get_academic_sources,
|
"academic": get_academic_sources,
|
||||||
"wechat": get_wechat_sources,
|
"wechat": get_wechat_sources,
|
||||||
"social": get_social_sources,
|
"social": get_social_sources,
|
||||||
|
"api": get_api_engines,
|
||||||
"special": get_special_engines,
|
"special": get_special_engines,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -44,6 +51,10 @@ class UnifiedSearch:
|
|||||||
self.config = config or {}
|
self.config = config or {}
|
||||||
self._cache: dict[str, list[SearchResult]] = {}
|
self._cache: dict[str, list[SearchResult]] = {}
|
||||||
self._engines: dict[str, Any] = {}
|
self._engines: dict[str, Any] = {}
|
||||||
|
self._timeout = self.config.get("engine_timeout", _DEFAULT_ENGINE_TIMEOUT)
|
||||||
|
self._semaphore = asyncio.Semaphore(
|
||||||
|
self.config.get("max_concurrent", _DEFAULT_MAX_CONCURRENT)
|
||||||
|
)
|
||||||
|
|
||||||
async def __aenter__(self):
|
async def __aenter__(self):
|
||||||
return self
|
return self
|
||||||
@@ -80,6 +91,7 @@ class UnifiedSearch:
|
|||||||
if not tasks:
|
if not tasks:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
# Run all engines with semaphore-based concurrency control
|
||||||
results_lists = await asyncio.gather(*tasks, return_exceptions=True)
|
results_lists = await asyncio.gather(*tasks, return_exceptions=True)
|
||||||
all_results: list[SearchResult] = []
|
all_results: list[SearchResult] = []
|
||||||
for results in results_lists:
|
for results in results_lists:
|
||||||
@@ -88,9 +100,64 @@ class UnifiedSearch:
|
|||||||
continue
|
continue
|
||||||
all_results.extend(results)
|
all_results.extend(results)
|
||||||
|
|
||||||
|
# Dedup by URL, keep first occurrence
|
||||||
|
all_results = self._dedup_results(all_results)
|
||||||
|
|
||||||
self._cache[cache_key] = all_results
|
self._cache[cache_key] = all_results
|
||||||
return all_results
|
return all_results
|
||||||
|
|
||||||
|
def _dedup_results(self, results: list[SearchResult]) -> list[SearchResult]:
|
||||||
|
"""Remove duplicates by URL, keeping first occurrence."""
|
||||||
|
seen: set[str] = set()
|
||||||
|
deduped = []
|
||||||
|
for r in results:
|
||||||
|
key = r.url.strip().rstrip("/")
|
||||||
|
if key and key not in seen:
|
||||||
|
seen.add(key)
|
||||||
|
deduped.append(r)
|
||||||
|
return deduped
|
||||||
|
|
||||||
|
async def _safe_search(self, engine, query, max_results, **kwargs):
|
||||||
|
async with self._semaphore:
|
||||||
|
try:
|
||||||
|
return await asyncio.wait_for(
|
||||||
|
engine.search(query, max_results=max_results, **kwargs),
|
||||||
|
timeout=self._timeout,
|
||||||
|
)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
logger.warning(f"Engine {engine.name} timed out after {self._timeout}s")
|
||||||
|
return []
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Engine {engine.name} failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def search_health(self) -> list[dict[str, Any]]:
|
||||||
|
"""Check health of all configured engines."""
|
||||||
|
statuses = []
|
||||||
|
seen_names = set()
|
||||||
|
for cat_key, cat_fn in SOURCE_CATEGORIES.items():
|
||||||
|
for engine_def in cat_fn():
|
||||||
|
name = engine_def.get("name", "?")
|
||||||
|
if name in seen_names:
|
||||||
|
continue
|
||||||
|
seen_names.add(name)
|
||||||
|
engine = self._get_engine(engine_def)
|
||||||
|
if engine is None:
|
||||||
|
statuses.append({"name": name, "status": "error", "detail": "engine_init_failed"})
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
async with self._semaphore:
|
||||||
|
await asyncio.wait_for(
|
||||||
|
engine.search("health check", max_results=1),
|
||||||
|
timeout=8,
|
||||||
|
)
|
||||||
|
statuses.append({"name": name, "status": "ok"})
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
statuses.append({"name": name, "status": "timeout"})
|
||||||
|
except Exception as e:
|
||||||
|
statuses.append({"name": name, "status": "error", "detail": str(e)[:80]})
|
||||||
|
return statuses
|
||||||
|
|
||||||
def _get_engine(self, engine_def: dict[str, Any]):
|
def _get_engine(self, engine_def: dict[str, Any]):
|
||||||
"""Get or create an engine instance by definition."""
|
"""Get or create an engine instance by definition."""
|
||||||
name = engine_def.get("name", "")
|
name = engine_def.get("name", "")
|
||||||
@@ -101,6 +168,8 @@ class UnifiedSearch:
|
|||||||
|
|
||||||
if name == "DuckDuckGo":
|
if name == "DuckDuckGo":
|
||||||
engine = DuckDuckGoEngine(self.config)
|
engine = DuckDuckGoEngine(self.config)
|
||||||
|
elif engine_type == "baidu_qianfan" or "baidu qianfan" in name.lower():
|
||||||
|
engine = BaiduQianFanEngine(self.config)
|
||||||
elif "cls" in name.lower():
|
elif "cls" in name.lower():
|
||||||
engine = CLSNewsEngine(self.config)
|
engine = CLSNewsEngine(self.config)
|
||||||
elif "wallstreet" in name.lower() or "华尔街" in name:
|
elif "wallstreet" in name.lower() or "华尔街" in name:
|
||||||
@@ -121,13 +190,6 @@ class UnifiedSearch:
|
|||||||
self._engines[name] = engine
|
self._engines[name] = engine
|
||||||
return engine
|
return engine
|
||||||
|
|
||||||
async def _safe_search(self, engine, query, max_results, **kwargs):
|
|
||||||
try:
|
|
||||||
return await engine.search(query, max_results=max_results, **kwargs)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Engine {engine.name} failed: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
async def search_engines(self, query, region=None, max_results=10):
|
async def search_engines(self, query, region=None, max_results=10):
|
||||||
"""Quick search across web search engines."""
|
"""Quick search across web search engines."""
|
||||||
engines = get_search_engines()
|
engines = get_search_engines()
|
||||||
|
|||||||
Reference in new issue
Block a user