diff --git a/config/engines.json b/config/engines.json index ab36cb3..14c91d5 100644 --- a/config/engines.json +++ b/config/engines.json @@ -15,6 +15,9 @@ {"name": "Ecosia", "url": "https://www.ecosia.org/search?q={keyword}", "region": "global", "type": "url"}, {"name": "Qwant", "url": "https://www.qwant.com/?q={keyword}", "region": "global", "type": "url"} ], + "api_engines": [ + {"name": "Baidu Qianfan", "type": "baidu_qianfan", "region": "cn"} + ], "special_engines": [ {"name": "WolframAlpha", "url": "https://www.wolframalpha.com/input?i={keyword}", "region": "global", "type": "url"} ], diff --git a/scripts/cli.py b/scripts/cli.py index 6550bd0..59ed0b8 100644 --- a/scripts/cli.py +++ b/scripts/cli.py @@ -8,11 +8,13 @@ Usage: Commands: search Search all sources (default) web Search web search engines (Google, Baidu, DuckDuckGo, etc.) + baidu Search via Baidu Qianfan AI Search API (需配置 BAIDU_API_KEY) news Search news (财联社, 华尔街见闻, RSS) - wechat Search WeChat public account articles + wechat Search WeChat public account articles (via Sougou) academic Search academic papers (ArXiv) social Search social media (Twitter/X) rss Search RSS news feeds + health Test engine connectivity sources List all available sources urls Get search URLs without executing @@ -20,16 +22,21 @@ Options: -n, --max-results N Max results per source (default: 10) -j, --json Output as JSON -o, --output FILE Save results to file - -s, --source NAME Specific source name + -s, --source NAME Specific source name(s) -r, --region REGION Region filter (cn/global) + --site SITE Restrict to specific site (baidu command only, e.g. mp.weixin.qq.com) + --recency RECENCY Time filter: day, week, month, semiyear, year (baidu command only) -v, --verbose Show debug logs -h, --help Show this help Examples: web-search wechat 蒙面财经 + web-search baidu 今日新闻 --recency week + web-search baidu "央视新闻 中美元首会晤" --site mp.weixin.qq.com --recency week web-search news "人工智能" -n 5 web-search web "Python async" -r global -j web-search search "量子计算" -o results.json + web-search health web-search sources web-search urls "机器学习" """ @@ -226,6 +233,15 @@ async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]: await us.close() +async def _search_baidu(query: str, max_results: int, **kwargs) -> list[Any]: + from scripts.engines.baidu_qianfan import BaiduQianFanEngine + engine = BaiduQianFanEngine({"api_key": os.getenv("BAIDU_API_KEY", "")}) + try: + return await engine.search(query, max_results=max_results, **kwargs) + finally: + await engine.close() + + async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]: us = _get_unified()() try: @@ -258,6 +274,7 @@ def cmd_sources(args): ("academic_sources", "学术源"), ("wechat_sources", "微信源"), ("social_sources", "社交源"), + ("api_engines", "API 引擎"), ("special_engines", "特殊源"), ]: count = categories.get(cat_key, 0) @@ -280,6 +297,26 @@ def cmd_urls(args): print() +async def cmd_health(args): + """Test all engine connectivity.""" + us = _get_unified()() + try: + print("正在检测各引擎连通性...\n") + statuses = await us.search_health() + ok_count = sum(1 for s in statuses if s["status"] == "ok") + for s in statuses: + if s["status"] == "ok": + print(f" ✅ {s['name']}") + elif s["status"] == "timeout": + print(f" ⏱️ {s['name']} 超时") + else: + detail = s.get("detail", "") + print(f" ❌ {s['name']} {detail}") + print(f"\n总计: {len(statuses)} 个引擎, {ok_count} 个可用") + finally: + await us.close() + + async def _run_search(args): """Route search to the right engine(s).""" if args.verbose: @@ -288,6 +325,7 @@ async def _run_search(args): method_map = { "search": _search_all, "web": _search_web, + "baidu": _search_baidu, "news": _search_news, "wechat": _search_wechat, "academic": _search_academic, @@ -300,6 +338,11 @@ async def _run_search(args): kwargs = {} if args.command == "web" and args.region: kwargs["region"] = args.region + if args.command == "baidu": + if args.site: + kwargs["site"] = args.site + if args.recency: + kwargs["recency"] = args.recency try: results = await search_fn( @@ -318,6 +361,7 @@ async def _run_search(args): cmd_names = { "search": "全源搜索", "web": "网页搜索", + "baidu": "百度千帆 AI 搜索", "news": "新闻搜索", "wechat": "微信公众号搜索", "academic": "学术搜索", @@ -376,8 +420,8 @@ def build_parser() -> argparse.ArgumentParser: "command", nargs="?", default="search", - choices=["search", "web", "news", "wechat", "academic", "social", - "rss", "sources", "urls"], + choices=["search", "web", "baidu", "news", "wechat", "academic", + "social", "rss", "health", "sources", "urls"], help="Search type (default: search)", ) parser.add_argument( @@ -414,6 +458,18 @@ def build_parser() -> argparse.ArgumentParser: choices=["cn", "global"], help="Region filter (web search only)", ) + parser.add_argument( + "--site", + type=str, + help="Restrict to specific site (baidu command only, e.g. mp.weixin.qq.com)", + ) + parser.add_argument( + "--recency", + type=str, + choices=["day", "week", "month", "semiyear", "year"], + default=None, + help="Time filter (baidu command only, default: year)", + ) parser.add_argument( "-v", "--verbose", action="store_true", @@ -434,11 +490,15 @@ def main(): if args.verbose: logging.getLogger().setLevel(logging.DEBUG) - # sources and urls commands don't need a query + # sources, urls, health don't need a query if args.command == "sources": cmd_sources(args) return + if args.command == "health": + asyncio.run(cmd_health(args)) + return + if args.command == "urls": if not args.query: print("错误: urls 命令需要提供搜索关键词", file=sys.stderr) diff --git a/scripts/core/config_loader.py b/scripts/core/config_loader.py index c17928e..97da0b8 100644 --- a/scripts/core/config_loader.py +++ b/scripts/core/config_loader.py @@ -38,6 +38,11 @@ def get_social_sources(path: Optional[str] = None) -> list[dict[str, Any]]: return _load_config(path).get("social_sources", []) +def get_api_engines(path: Optional[str] = None) -> list[dict[str, Any]]: + """Get API-based search engines (Baidu Qianfan, etc.).""" + return _load_config(path).get("api_engines", []) + + def get_special_engines(path: Optional[str] = None) -> list[dict[str, Any]]: return _load_config(path).get("special_engines", []) @@ -45,7 +50,8 @@ def get_special_engines(path: Optional[str] = None) -> list[dict[str, Any]]: def get_engine(name: str, path: Optional[str] = None): config = _load_config(path) for category in ["search_engines", "news_sources", "academic_sources", - "wechat_sources", "social_sources", "special_engines"]: + "wechat_sources", "social_sources", "api_engines", + "special_engines"]: for engine in config.get(category, []): if engine["name"].lower() == name.lower(): return engine @@ -56,7 +62,8 @@ def get_all_sources(path: Optional[str] = None) -> list[dict[str, Any]]: config = _load_config(path) all_sources = [] for category in ["search_engines", "news_sources", "academic_sources", - "wechat_sources", "social_sources", "special_engines"]: + "wechat_sources", "social_sources", "api_engines", + "special_engines"]: all_sources.extend(config.get(category, [])) return all_sources @@ -65,6 +72,7 @@ def list_source_categories(path: Optional[str] = None) -> dict[str, int]: config = _load_config(path) categories = [ "search_engines", "news_sources", "academic_sources", - "wechat_sources", "social_sources", "special_engines", + "wechat_sources", "social_sources", "api_engines", + "special_engines", ] return {cat: len(config.get(cat, [])) for cat in categories} diff --git a/scripts/engines/baidu_qianfan.py b/scripts/engines/baidu_qianfan.py new file mode 100644 index 0000000..3ef4272 --- /dev/null +++ b/scripts/engines/baidu_qianfan.py @@ -0,0 +1,172 @@ +""" +Baidu Qianfan AI Search API engine. + +Uses the official Baidu Qianfan (千帆) AI Search API to get +structured search results. Requires BAIDU_API_KEY environment variable +or .env file in the project root. + +API docs: https://console.bce.baidu.com/qianfan/ais/console/apiKey +""" + +from __future__ import annotations + +import json +import logging +import os +from pathlib import Path +from typing import Any, Optional + +import httpx + +from scripts.core.base import SearchEngine, SearchResult + +logger = logging.getLogger(__name__) + +API_URL = "https://qianfan.baidubce.com/v2/ai_search/web_search" +ENV_KEY = "BAIDU_API_KEY" + + +def _load_env_file() -> dict[str, str]: + """Load .env file from project root (simple parser, no external deps).""" + env_path = Path(__file__).resolve().parent.parent.parent / ".env" + if not env_path.exists(): + return {} + env_vars = {} + with open(env_path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, val = line.partition("=") + env_vars[key.strip()] = val.strip().strip("\"'") + return env_vars + + +def _get_api_key() -> str: + """Get API key: env var > .env file > config.""" + key = os.getenv(ENV_KEY) + if key: + return key + env_from_file = _load_env_file() + return env_from_file.get(ENV_KEY, "") + + +class BaiduQianFanEngine(SearchEngine): + """百度千帆 AI 搜索 API 引擎.""" + + def __init__(self, config: Optional[dict] = None): + super().__init__("Baidu Qianfan", config) + self.api_key = (config or {}).get("api_key", "") or _get_api_key() + if not self.api_key: + logger.warning( + f"BAIDU_API_KEY not set! Set it in .env file or as environment variable." + ) + + timeout = (config or {}).get("timeout", 30) + self._client = httpx.AsyncClient( + timeout=timeout, + headers={ + "Authorization": f"Bearer {self.api_key}", + "X-Appbuilder-From": "openclaw", + "Content-Type": "application/json", + }, + follow_redirects=True, + ) + + async def search( + self, query: str, max_results: int = 10, **kwargs + ) -> list[SearchResult]: + """Search via Baidu Qianfan AI Search API. + + Args: + query: Search keywords. + max_results: Max results to return. + **kwargs: + site: Restrict search to a specific site (e.g. "mp.weixin.qq.com"). + recency: Time filter: "week", "month", "semiyear", "year" (default: "year"). + edition: "standard" (default) or "lite". + safe_search: bool, default False. + """ + if not self.api_key: + return [ + SearchResult( + title="错误: BAIDU_API_KEY 未配置", + url="https://console.bce.baidu.com/qianfan/ais/console/apiKey", + snippet="请先设置 BAIDU_API_KEY 环境变量", + source=self.name, + rank=1, + category="web", + extra={"error": "missing_api_key"}, + ) + ] + + # Build resource_type_filter based on max_results + top_k = min(max_results, 50) + resource_type_filter = [ + {"type": "web", "top_k": top_k}, + ] + + # Build search_filter if site is specified + search_filter = {} + site = kwargs.get("site") + if site: + search_filter["match"] = {"site": [site]} + + request_body = { + "messages": [{"content": query, "role": "user"}], + "edition": kwargs.get("edition", "standard"), + "search_source": "baidu_search_v2", + "resource_type_filter": resource_type_filter, + "search_recency_filter": kwargs.get("recency", "year"), + "safe_search": kwargs.get("safe_search", False), + } + if search_filter: + request_body["search_filter"] = search_filter + if kwargs.get("block_websites"): + request_body["block_websites"] = kwargs["block_websites"] + + try: + response = await self._client.post(API_URL, json=request_body) + response.raise_for_status() + data = response.json() + + if "code" in data: + logger.error(f"Baidu Qianfan API error: {data.get('message', '')}") + return [] + + references = data.get("references", []) + results = [] + for i, ref in enumerate(references[:max_results]): + results.append( + SearchResult( + title=ref.get("title", ""), + url=ref.get("url", ""), + snippet=ref.get("snippet", ""), + source=self.name, + rank=i + 1, + category="web", + extra={ + k: ref[k] + for k in ("site", "date", "type", "source") + if k in ref + }, + ) + ) + return results + + except httpx.HTTPStatusError as e: + logger.error(f"Baidu Qianfan API HTTP error: {e.response.status_code}") + if e.response.status_code == 401: + logger.error("API Key 无效或未授权,请检查 BAIDU_API_KEY") + elif e.response.status_code == 429: + logger.error("API 配额不足") + return [] + except httpx.RequestError as e: + logger.error(f"Baidu Qianfan API request failed: {e}") + return [] + except json.JSONDecodeError as e: + logger.error(f"Baidu Qianfan API response parse failed: {e}") + return [] + + async def close(self): + await self._client.aclose() diff --git a/scripts/unified_search.py b/scripts/unified_search.py index 91b5b2d..2cdc40e 100644 --- a/scripts/unified_search.py +++ b/scripts/unified_search.py @@ -13,8 +13,8 @@ from typing import Any, Optional from scripts.core.base import SearchResult from scripts.core.config_loader import ( get_search_engines, get_news_sources, get_academic_sources, - get_wechat_sources, get_special_engines, get_social_sources, - get_all_sources, get_engine, + get_wechat_sources, get_api_engines, get_special_engines, + get_social_sources, get_all_sources, get_engine, ) from scripts.engines.url_engine import UrlSearchEngine from scripts.engines.parser_engines import DuckDuckGoEngine @@ -24,15 +24,22 @@ from scripts.news.rss import RSSNewsEngine from scripts.academic.arxiv import ArXivEngine from scripts.wechat.sogou_weixin import SogouWeChatEngine from scripts.social.twitter import TwitterSearchEngine +from scripts.engines.baidu_qianfan import BaiduQianFanEngine logger = logging.getLogger(__name__) +# Default timeout per engine (seconds) +_DEFAULT_ENGINE_TIMEOUT = 15 +# Max concurrent engine calls +_DEFAULT_MAX_CONCURRENT = 5 + SOURCE_CATEGORIES = { "web": get_search_engines, "news": get_news_sources, "academic": get_academic_sources, "wechat": get_wechat_sources, "social": get_social_sources, + "api": get_api_engines, "special": get_special_engines, } @@ -44,6 +51,10 @@ class UnifiedSearch: self.config = config or {} self._cache: dict[str, list[SearchResult]] = {} self._engines: dict[str, Any] = {} + self._timeout = self.config.get("engine_timeout", _DEFAULT_ENGINE_TIMEOUT) + self._semaphore = asyncio.Semaphore( + self.config.get("max_concurrent", _DEFAULT_MAX_CONCURRENT) + ) async def __aenter__(self): return self @@ -80,6 +91,7 @@ class UnifiedSearch: if not tasks: return [] + # Run all engines with semaphore-based concurrency control results_lists = await asyncio.gather(*tasks, return_exceptions=True) all_results: list[SearchResult] = [] for results in results_lists: @@ -88,9 +100,64 @@ class UnifiedSearch: continue all_results.extend(results) + # Dedup by URL, keep first occurrence + all_results = self._dedup_results(all_results) + self._cache[cache_key] = all_results return all_results + def _dedup_results(self, results: list[SearchResult]) -> list[SearchResult]: + """Remove duplicates by URL, keeping first occurrence.""" + seen: set[str] = set() + deduped = [] + for r in results: + key = r.url.strip().rstrip("/") + if key and key not in seen: + seen.add(key) + deduped.append(r) + return deduped + + async def _safe_search(self, engine, query, max_results, **kwargs): + async with self._semaphore: + try: + return await asyncio.wait_for( + engine.search(query, max_results=max_results, **kwargs), + timeout=self._timeout, + ) + except asyncio.TimeoutError: + logger.warning(f"Engine {engine.name} timed out after {self._timeout}s") + return [] + except Exception as e: + logger.error(f"Engine {engine.name} failed: {e}") + return [] + + async def search_health(self) -> list[dict[str, Any]]: + """Check health of all configured engines.""" + statuses = [] + seen_names = set() + for cat_key, cat_fn in SOURCE_CATEGORIES.items(): + for engine_def in cat_fn(): + name = engine_def.get("name", "?") + if name in seen_names: + continue + seen_names.add(name) + engine = self._get_engine(engine_def) + if engine is None: + statuses.append({"name": name, "status": "error", "detail": "engine_init_failed"}) + continue + try: + async with self._semaphore: + await asyncio.wait_for( + engine.search("health check", max_results=1), + timeout=8, + ) + statuses.append({"name": name, "status": "ok"}) + except asyncio.TimeoutError: + statuses.append({"name": name, "status": "timeout"}) + except Exception as e: + statuses.append({"name": name, "status": "error", "detail": str(e)[:80]}) + return statuses + def _get_engine(self, engine_def: dict[str, Any]): """Get or create an engine instance by definition.""" name = engine_def.get("name", "") @@ -101,6 +168,8 @@ class UnifiedSearch: if name == "DuckDuckGo": engine = DuckDuckGoEngine(self.config) + elif engine_type == "baidu_qianfan" or "baidu qianfan" in name.lower(): + engine = BaiduQianFanEngine(self.config) elif "cls" in name.lower(): engine = CLSNewsEngine(self.config) elif "wallstreet" in name.lower() or "华尔街" in name: @@ -121,13 +190,6 @@ class UnifiedSearch: self._engines[name] = engine return engine - async def _safe_search(self, engine, query, max_results, **kwargs): - try: - return await engine.search(query, max_results=max_results, **kwargs) - except Exception as e: - logger.error(f"Engine {engine.name} failed: {e}") - return [] - async def search_engines(self, query, region=None, max_results=10): """Quick search across web search engines.""" engines = get_search_engines()