feat: 优化搜索广度、深度、效率及可靠性

- 新增百度千帆 AI 搜索 API 引擎 (scripts/engines/baidu_qianfan.py)
- CLI 新增 baidu 子命令,支持 --site 站点限定和 --recency 时间过滤
- UnifiedSearch: 引擎并发控制 (Semaphore) + 超时熔断 (wait_for)
- UnifiedSearch: 搜索结果按 URL 去重
- CLI 新增 health 健康检查命令
- sources 命令新增 API 引擎类别展示
- 支持 .env 文件读取 BAIDU_API_KEY
This commit is contained in:
vex1023 committed 2026-05-14 17:20:44 +08:00
1 parent 87935b59ce
commit 7eb1467e07
5 files changed
+322 -17

No files matched your search

+3
View File
@@ -15,6 +15,9 @@
{"name": "Ecosia", "url": "https://www.ecosia.org/search?q={keyword}", "region": "global", "type": "url"}, {"name": "Ecosia", "url": "https://www.ecosia.org/search?q={keyword}", "region": "global", "type": "url"},
{"name": "Qwant", "url": "https://www.qwant.com/?q={keyword}", "region": "global", "type": "url"} {"name": "Qwant", "url": "https://www.qwant.com/?q={keyword}", "region": "global", "type": "url"}
], ],
"api_engines": [
{"name": "Baidu Qianfan", "type": "baidu_qianfan", "region": "cn"}
],
"special_engines": [ "special_engines": [
{"name": "WolframAlpha", "url": "https://www.wolframalpha.com/input?i={keyword}", "region": "global", "type": "url"} {"name": "WolframAlpha", "url": "https://www.wolframalpha.com/input?i={keyword}", "region": "global", "type": "url"}
], ],
+65 -5
View File
@@ -8,11 +8,13 @@ Usage:
Commands: Commands:
search Search all sources (default) search Search all sources (default)
web Search web search engines (Google, Baidu, DuckDuckGo, etc.) web Search web search engines (Google, Baidu, DuckDuckGo, etc.)
baidu Search via Baidu Qianfan AI Search API (需配置 BAIDU_API_KEY)
news Search news (财联社, 华尔街见闻, RSS) news Search news (财联社, 华尔街见闻, RSS)
wechat Search WeChat public account articles wechat Search WeChat public account articles (via Sougou)
academic Search academic papers (ArXiv) academic Search academic papers (ArXiv)
social Search social media (Twitter/X) social Search social media (Twitter/X)
rss Search RSS news feeds rss Search RSS news feeds
health Test engine connectivity
sources List all available sources sources List all available sources
urls Get search URLs without executing urls Get search URLs without executing
@@ -20,16 +22,21 @@ Options:
-n, --max-results N Max results per source (default: 10) -n, --max-results N Max results per source (default: 10)
-j, --json Output as JSON -j, --json Output as JSON
-o, --output FILE Save results to file -o, --output FILE Save results to file
-s, --source NAME Specific source name -s, --source NAME Specific source name(s)
-r, --region REGION Region filter (cn/global) -r, --region REGION Region filter (cn/global)
--site SITE Restrict to specific site (baidu command only, e.g. mp.weixin.qq.com)
--recency RECENCY Time filter: day, week, month, semiyear, year (baidu command only)
-v, --verbose Show debug logs -v, --verbose Show debug logs
-h, --help Show this help -h, --help Show this help
Examples: Examples:
web-search wechat 蒙面财经 web-search wechat 蒙面财经
web-search baidu 今日新闻 --recency week
web-search baidu "央视新闻 中美元首会晤" --site mp.weixin.qq.com --recency week
web-search news "人工智能" -n 5 web-search news "人工智能" -n 5
web-search web "Python async" -r global -j web-search web "Python async" -r global -j
web-search search "量子计算" -o results.json web-search search "量子计算" -o results.json
web-search health
web-search sources web-search sources
web-search urls "机器学习" web-search urls "机器学习"
""" """
@@ -226,6 +233,15 @@ async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]:
await us.close() await us.close()
async def _search_baidu(query: str, max_results: int, **kwargs) -> list[Any]:
from scripts.engines.baidu_qianfan import BaiduQianFanEngine
engine = BaiduQianFanEngine({"api_key": os.getenv("BAIDU_API_KEY", "")})
try:
return await engine.search(query, max_results=max_results, **kwargs)
finally:
await engine.close()
async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]: async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()() us = _get_unified()()
try: try:
@@ -258,6 +274,7 @@ def cmd_sources(args):
("academic_sources", "学术源"), ("academic_sources", "学术源"),
("wechat_sources", "微信源"), ("wechat_sources", "微信源"),
("social_sources", "社交源"), ("social_sources", "社交源"),
("api_engines", "API 引擎"),
("special_engines", "特殊源"), ("special_engines", "特殊源"),
]: ]:
count = categories.get(cat_key, 0) count = categories.get(cat_key, 0)
@@ -280,6 +297,26 @@ def cmd_urls(args):
print() print()
async def cmd_health(args):
"""Test all engine connectivity."""
us = _get_unified()()
try:
print("正在检测各引擎连通性...\n")
statuses = await us.search_health()
ok_count = sum(1 for s in statuses if s["status"] == "ok")
for s in statuses:
if s["status"] == "ok":
print(f" ✅ {s['name']}")
elif s["status"] == "timeout":
print(f" ⏱️ {s['name']} 超时")
else:
detail = s.get("detail", "")
print(f" ❌ {s['name']} {detail}")
print(f"\n总计: {len(statuses)} 个引擎, {ok_count} 个可用")
finally:
await us.close()
async def _run_search(args): async def _run_search(args):
"""Route search to the right engine(s).""" """Route search to the right engine(s)."""
if args.verbose: if args.verbose:
@@ -288,6 +325,7 @@ async def _run_search(args):
method_map = { method_map = {
"search": _search_all, "search": _search_all,
"web": _search_web, "web": _search_web,
"baidu": _search_baidu,
"news": _search_news, "news": _search_news,
"wechat": _search_wechat, "wechat": _search_wechat,
"academic": _search_academic, "academic": _search_academic,
@@ -300,6 +338,11 @@ async def _run_search(args):
kwargs = {} kwargs = {}
if args.command == "web" and args.region: if args.command == "web" and args.region:
kwargs["region"] = args.region kwargs["region"] = args.region
if args.command == "baidu":
if args.site:
kwargs["site"] = args.site
if args.recency:
kwargs["recency"] = args.recency
try: try:
results = await search_fn( results = await search_fn(
@@ -318,6 +361,7 @@ async def _run_search(args):
cmd_names = { cmd_names = {
"search": "全源搜索", "search": "全源搜索",
"web": "网页搜索", "web": "网页搜索",
"baidu": "百度千帆 AI 搜索",
"news": "新闻搜索", "news": "新闻搜索",
"wechat": "微信公众号搜索", "wechat": "微信公众号搜索",
"academic": "学术搜索", "academic": "学术搜索",
@@ -376,8 +420,8 @@ def build_parser() -> argparse.ArgumentParser:
"command", "command",
nargs="?", nargs="?",
default="search", default="search",
choices=["search", "web", "news", "wechat", "academic", "social", choices=["search", "web", "baidu", "news", "wechat", "academic",
"rss", "sources", "urls"], "social", "rss", "health", "sources", "urls"],
help="Search type (default: search)", help="Search type (default: search)",
) )
parser.add_argument( parser.add_argument(
@@ -414,6 +458,18 @@ def build_parser() -> argparse.ArgumentParser:
choices=["cn", "global"], choices=["cn", "global"],
help="Region filter (web search only)", help="Region filter (web search only)",
) )
parser.add_argument(
"--site",
type=str,
help="Restrict to specific site (baidu command only, e.g. mp.weixin.qq.com)",
)
parser.add_argument(
"--recency",
type=str,
choices=["day", "week", "month", "semiyear", "year"],
default=None,
help="Time filter (baidu command only, default: year)",
)
parser.add_argument( parser.add_argument(
"-v", "--verbose", "-v", "--verbose",
action="store_true", action="store_true",
@@ -434,11 +490,15 @@ def main():
if args.verbose: if args.verbose:
logging.getLogger().setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG)
# sources and urls commands don't need a query # sources, urls, health don't need a query
if args.command == "sources": if args.command == "sources":
cmd_sources(args) cmd_sources(args)
return return
if args.command == "health":
asyncio.run(cmd_health(args))
return
if args.command == "urls": if args.command == "urls":
if not args.query: if not args.query:
print("错误: urls 命令需要提供搜索关键词", file=sys.stderr) print("错误: urls 命令需要提供搜索关键词", file=sys.stderr)
+11 -3
View File
@@ -38,6 +38,11 @@ def get_social_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
return _load_config(path).get("social_sources", []) return _load_config(path).get("social_sources", [])
def get_api_engines(path: Optional[str] = None) -> list[dict[str, Any]]:
"""Get API-based search engines (Baidu Qianfan, etc.)."""
return _load_config(path).get("api_engines", [])
def get_special_engines(path: Optional[str] = None) -> list[dict[str, Any]]: def get_special_engines(path: Optional[str] = None) -> list[dict[str, Any]]:
return _load_config(path).get("special_engines", []) return _load_config(path).get("special_engines", [])
@@ -45,7 +50,8 @@ def get_special_engines(path: Optional[str] = None) -> list[dict[str, Any]]:
def get_engine(name: str, path: Optional[str] = None): def get_engine(name: str, path: Optional[str] = None):
config = _load_config(path) config = _load_config(path)
for category in ["search_engines", "news_sources", "academic_sources", for category in ["search_engines", "news_sources", "academic_sources",
"wechat_sources", "social_sources", "special_engines"]: "wechat_sources", "social_sources", "api_engines",
"special_engines"]:
for engine in config.get(category, []): for engine in config.get(category, []):
if engine["name"].lower() == name.lower(): if engine["name"].lower() == name.lower():
return engine return engine
@@ -56,7 +62,8 @@ def get_all_sources(path: Optional[str] = None) -> list[dict[str, Any]]:
config = _load_config(path) config = _load_config(path)
all_sources = [] all_sources = []
for category in ["search_engines", "news_sources", "academic_sources", for category in ["search_engines", "news_sources", "academic_sources",
"wechat_sources", "social_sources", "special_engines"]: "wechat_sources", "social_sources", "api_engines",
"special_engines"]:
all_sources.extend(config.get(category, [])) all_sources.extend(config.get(category, []))
return all_sources return all_sources
@@ -65,6 +72,7 @@ def list_source_categories(path: Optional[str] = None) -> dict[str, int]:
config = _load_config(path) config = _load_config(path)
categories = [ categories = [
"search_engines", "news_sources", "academic_sources", "search_engines", "news_sources", "academic_sources",
"wechat_sources", "social_sources", "special_engines", "wechat_sources", "social_sources", "api_engines",
"special_engines",
] ]
return {cat: len(config.get(cat, [])) for cat in categories} return {cat: len(config.get(cat, [])) for cat in categories}
+172
View File
@@ -0,0 +1,172 @@
"""
Baidu Qianfan AI Search API engine.
Uses the official Baidu Qianfan (千帆) AI Search API to get
structured search results. Requires BAIDU_API_KEY environment variable
or .env file in the project root.
API docs: https://console.bce.baidu.com/qianfan/ais/console/apiKey
"""
from __future__ import annotations
import json
import logging
import os
from pathlib import Path
from typing import Any, Optional
import httpx
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
API_URL = "https://qianfan.baidubce.com/v2/ai_search/web_search"
ENV_KEY = "BAIDU_API_KEY"
def _load_env_file() -> dict[str, str]:
"""Load .env file from project root (simple parser, no external deps)."""
env_path = Path(__file__).resolve().parent.parent.parent / ".env"
if not env_path.exists():
return {}
env_vars = {}
with open(env_path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, val = line.partition("=")
env_vars[key.strip()] = val.strip().strip("\"'")
return env_vars
def _get_api_key() -> str:
"""Get API key: env var > .env file > config."""
key = os.getenv(ENV_KEY)
if key:
return key
env_from_file = _load_env_file()
return env_from_file.get(ENV_KEY, "")
class BaiduQianFanEngine(SearchEngine):
"""百度千帆 AI 搜索 API 引擎."""
def __init__(self, config: Optional[dict] = None):
super().__init__("Baidu Qianfan", config)
self.api_key = (config or {}).get("api_key", "") or _get_api_key()
if not self.api_key:
logger.warning(
f"BAIDU_API_KEY not set! Set it in .env file or as environment variable."
)
timeout = (config or {}).get("timeout", 30)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"Authorization": f"Bearer {self.api_key}",
"X-Appbuilder-From": "openclaw",
"Content-Type": "application/json",
},
follow_redirects=True,
)
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search via Baidu Qianfan AI Search API.
Args:
query: Search keywords.
max_results: Max results to return.
**kwargs:
site: Restrict search to a specific site (e.g. "mp.weixin.qq.com").
recency: Time filter: "week", "month", "semiyear", "year" (default: "year").
edition: "standard" (default) or "lite".
safe_search: bool, default False.
"""
if not self.api_key:
return [
SearchResult(
title="错误: BAIDU_API_KEY 未配置",
url="https://console.bce.baidu.com/qianfan/ais/console/apiKey",
snippet="请先设置 BAIDU_API_KEY 环境变量",
source=self.name,
rank=1,
category="web",
extra={"error": "missing_api_key"},
)
]
# Build resource_type_filter based on max_results
top_k = min(max_results, 50)
resource_type_filter = [
{"type": "web", "top_k": top_k},
]
# Build search_filter if site is specified
search_filter = {}
site = kwargs.get("site")
if site:
search_filter["match"] = {"site": [site]}
request_body = {
"messages": [{"content": query, "role": "user"}],
"edition": kwargs.get("edition", "standard"),
"search_source": "baidu_search_v2",
"resource_type_filter": resource_type_filter,
"search_recency_filter": kwargs.get("recency", "year"),
"safe_search": kwargs.get("safe_search", False),
}
if search_filter:
request_body["search_filter"] = search_filter
if kwargs.get("block_websites"):
request_body["block_websites"] = kwargs["block_websites"]
try:
response = await self._client.post(API_URL, json=request_body)
response.raise_for_status()
data = response.json()
if "code" in data:
logger.error(f"Baidu Qianfan API error: {data.get('message', '')}")
return []
references = data.get("references", [])
results = []
for i, ref in enumerate(references[:max_results]):
results.append(
SearchResult(
title=ref.get("title", ""),
url=ref.get("url", ""),
snippet=ref.get("snippet", ""),
source=self.name,
rank=i + 1,
category="web",
extra={
k: ref[k]
for k in ("site", "date", "type", "source")
if k in ref
},
)
)
return results
except httpx.HTTPStatusError as e:
logger.error(f"Baidu Qianfan API HTTP error: {e.response.status_code}")
if e.response.status_code == 401:
logger.error("API Key 无效或未授权,请检查 BAIDU_API_KEY")
elif e.response.status_code == 429:
logger.error("API 配额不足")
return []
except httpx.RequestError as e:
logger.error(f"Baidu Qianfan API request failed: {e}")
return []
except json.JSONDecodeError as e:
logger.error(f"Baidu Qianfan API response parse failed: {e}")
return []
async def close(self):
await self._client.aclose()
+71 -9
View File
@@ -13,8 +13,8 @@ from typing import Any, Optional
from scripts.core.base import SearchResult from scripts.core.base import SearchResult
from scripts.core.config_loader import ( from scripts.core.config_loader import (
get_search_engines, get_news_sources, get_academic_sources, get_search_engines, get_news_sources, get_academic_sources,
get_wechat_sources, get_special_engines, get_social_sources, get_wechat_sources, get_api_engines, get_special_engines,
get_all_sources, get_engine, get_social_sources, get_all_sources, get_engine,
) )
from scripts.engines.url_engine import UrlSearchEngine from scripts.engines.url_engine import UrlSearchEngine
from scripts.engines.parser_engines import DuckDuckGoEngine from scripts.engines.parser_engines import DuckDuckGoEngine
@@ -24,15 +24,22 @@ from scripts.news.rss import RSSNewsEngine
from scripts.academic.arxiv import ArXivEngine from scripts.academic.arxiv import ArXivEngine
from scripts.wechat.sogou_weixin import SogouWeChatEngine from scripts.wechat.sogou_weixin import SogouWeChatEngine
from scripts.social.twitter import TwitterSearchEngine from scripts.social.twitter import TwitterSearchEngine
from scripts.engines.baidu_qianfan import BaiduQianFanEngine
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Default timeout per engine (seconds)
_DEFAULT_ENGINE_TIMEOUT = 15
# Max concurrent engine calls
_DEFAULT_MAX_CONCURRENT = 5
SOURCE_CATEGORIES = { SOURCE_CATEGORIES = {
"web": get_search_engines, "web": get_search_engines,
"news": get_news_sources, "news": get_news_sources,
"academic": get_academic_sources, "academic": get_academic_sources,
"wechat": get_wechat_sources, "wechat": get_wechat_sources,
"social": get_social_sources, "social": get_social_sources,
"api": get_api_engines,
"special": get_special_engines, "special": get_special_engines,
} }
@@ -44,6 +51,10 @@ class UnifiedSearch:
self.config = config or {} self.config = config or {}
self._cache: dict[str, list[SearchResult]] = {} self._cache: dict[str, list[SearchResult]] = {}
self._engines: dict[str, Any] = {} self._engines: dict[str, Any] = {}
self._timeout = self.config.get("engine_timeout", _DEFAULT_ENGINE_TIMEOUT)
self._semaphore = asyncio.Semaphore(
self.config.get("max_concurrent", _DEFAULT_MAX_CONCURRENT)
)
async def __aenter__(self): async def __aenter__(self):
return self return self
@@ -80,6 +91,7 @@ class UnifiedSearch:
if not tasks: if not tasks:
return [] return []
# Run all engines with semaphore-based concurrency control
results_lists = await asyncio.gather(*tasks, return_exceptions=True) results_lists = await asyncio.gather(*tasks, return_exceptions=True)
all_results: list[SearchResult] = [] all_results: list[SearchResult] = []
for results in results_lists: for results in results_lists:
@@ -88,9 +100,64 @@ class UnifiedSearch:
continue continue
all_results.extend(results) all_results.extend(results)
# Dedup by URL, keep first occurrence
all_results = self._dedup_results(all_results)
self._cache[cache_key] = all_results self._cache[cache_key] = all_results
return all_results return all_results
def _dedup_results(self, results: list[SearchResult]) -> list[SearchResult]:
"""Remove duplicates by URL, keeping first occurrence."""
seen: set[str] = set()
deduped = []
for r in results:
key = r.url.strip().rstrip("/")
if key and key not in seen:
seen.add(key)
deduped.append(r)
return deduped
async def _safe_search(self, engine, query, max_results, **kwargs):
async with self._semaphore:
try:
return await asyncio.wait_for(
engine.search(query, max_results=max_results, **kwargs),
timeout=self._timeout,
)
except asyncio.TimeoutError:
logger.warning(f"Engine {engine.name} timed out after {self._timeout}s")
return []
except Exception as e:
logger.error(f"Engine {engine.name} failed: {e}")
return []
async def search_health(self) -> list[dict[str, Any]]:
"""Check health of all configured engines."""
statuses = []
seen_names = set()
for cat_key, cat_fn in SOURCE_CATEGORIES.items():
for engine_def in cat_fn():
name = engine_def.get("name", "?")
if name in seen_names:
continue
seen_names.add(name)
engine = self._get_engine(engine_def)
if engine is None:
statuses.append({"name": name, "status": "error", "detail": "engine_init_failed"})
continue
try:
async with self._semaphore:
await asyncio.wait_for(
engine.search("health check", max_results=1),
timeout=8,
)
statuses.append({"name": name, "status": "ok"})
except asyncio.TimeoutError:
statuses.append({"name": name, "status": "timeout"})
except Exception as e:
statuses.append({"name": name, "status": "error", "detail": str(e)[:80]})
return statuses
def _get_engine(self, engine_def: dict[str, Any]): def _get_engine(self, engine_def: dict[str, Any]):
"""Get or create an engine instance by definition.""" """Get or create an engine instance by definition."""
name = engine_def.get("name", "") name = engine_def.get("name", "")
@@ -101,6 +168,8 @@ class UnifiedSearch:
if name == "DuckDuckGo": if name == "DuckDuckGo":
engine = DuckDuckGoEngine(self.config) engine = DuckDuckGoEngine(self.config)
elif engine_type == "baidu_qianfan" or "baidu qianfan" in name.lower():
engine = BaiduQianFanEngine(self.config)
elif "cls" in name.lower(): elif "cls" in name.lower():
engine = CLSNewsEngine(self.config) engine = CLSNewsEngine(self.config)
elif "wallstreet" in name.lower() or "华尔街" in name: elif "wallstreet" in name.lower() or "华尔街" in name:
@@ -121,13 +190,6 @@ class UnifiedSearch:
self._engines[name] = engine self._engines[name] = engine
return engine return engine
async def _safe_search(self, engine, query, max_results, **kwargs):
try:
return await engine.search(query, max_results=max_results, **kwargs)
except Exception as e:
logger.error(f"Engine {engine.name} failed: {e}")
return []
async def search_engines(self, query, region=None, max_results=10): async def search_engines(self, query, region=None, max_results=10):
"""Quick search across web search engines.""" """Quick search across web search engines."""
engines = get_search_engines() engines = get_search_engines()