refactor: cleanup dead code, fix model confusion, streamline CLI

- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/)
- Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py)
- scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py)
- scripts/engines/base.py: re-exports from core.base for compatibility
- scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py)
- scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py)
- scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources)
- scripts/unified_search.py: clean async Python API (was a stub)
- ws.py: lightweight launcher
- README.md + SKILL.md: updated to reflect actual 15+ engines
- All EngineResult → SearchResult
- All engines return list[SearchResult]
This commit is contained in:
openbot-coder committed 2026-05-30 16:25:41 +08:00
1 parent e20478fcbd
commit c077a7ae40
31 files changed
+1188 -3148

No files matched your search

+25 -54
View File
@@ -18,7 +18,7 @@ from typing import Any, Optional
import httpx
from scripts.core.base import SearchEngine, SearchResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
@@ -51,18 +51,18 @@ def _get_api_key() -> str:
return env_from_file.get(ENV_KEY, "")
class BaiduQianFanEngine(SearchEngine):
class BaiduQianFanEngine(BaseEngine):
"""百度千帆 AI 搜索 API 引擎."""
def __init__(self, config: Optional[dict] = None):
super().__init__("Baidu Qianfan", config)
name = "baidu_qianfan"
def __init__(self, config: Optional[dict] = None, timeout: float = 30.0):
super().__init__(timeout=timeout)
self.api_key = (config or {}).get("api_key", "") or _get_api_key()
if not self.api_key:
logger.warning(
f"BAIDU_API_KEY not set! Set it in .env file or as environment variable."
"BAIDU_API_KEY not set. Set it in .env or as environment variable."
)
timeout = (config or {}).get("timeout", 30)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
@@ -76,17 +76,7 @@ class BaiduQianFanEngine(SearchEngine):
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search via Baidu Qianfan AI Search API.
Args:
query: Search keywords.
max_results: Max results to return.
**kwargs:
site: Restrict search to a specific site (e.g. "mp.weixin.qq.com").
recency: Time filter: "week", "month", "semiyear", "year" (default: "year").
edition: "standard" (default) or "lite".
safe_search: bool, default False.
"""
"""Search via Baidu Qianfan AI Search API."""
if not self.api_key:
return [
SearchResult(
@@ -100,19 +90,14 @@ class BaiduQianFanEngine(SearchEngine):
)
]
# Build resource_type_filter based on max_results
top_k = min(max_results, 50)
resource_type_filter = [
{"type": "web", "top_k": top_k},
]
resource_type_filter = [{"type": "web", "top_k": top_k}]
# Build search_filter if site is specified
search_filter = {}
site = kwargs.get("site")
if site:
search_filter["match"] = {"site": [site]}
search_filter: dict[str, Any] = {}
if kwargs.get("site"):
search_filter["match"] = {"site": [kwargs["site"]]}
request_body = {
request_body: dict[str, Any] = {
"messages": [{"content": query, "role": "user"}],
"edition": kwargs.get("edition", "standard"),
"search_source": "baidu_search_v2",
@@ -134,38 +119,24 @@ class BaiduQianFanEngine(SearchEngine):
logger.error(f"Baidu Qianfan API error: {data.get('message', '')}")
return []
references = data.get("references", [])
results = []
for i, ref in enumerate(references[:max_results]):
results.append(
SearchResult(
title=ref.get("title", ""),
url=ref.get("url", ""),
snippet=ref.get("snippet", ""),
source=self.name,
rank=i + 1,
category="web",
extra={
k: ref[k]
for k in ("site", "date", "type", "source")
if k in ref
},
)
)
for i, ref in enumerate(data.get("references", [])[:max_results]):
results.append(SearchResult(
title=ref.get("title", ""),
url=ref.get("url", ""),
snippet=ref.get("snippet", ""),
source=self.name,
rank=i + 1,
category="web",
extra={k: ref[k] for k in ("site", "date", "type", "source") if k in ref},
))
return results
except httpx.HTTPStatusError as e:
logger.error(f"Baidu Qianfan API HTTP error: {e.response.status_code}")
if e.response.status_code == 401:
logger.error("API Key 无效或未授权,请检查 BAIDU_API_KEY")
elif e.response.status_code == 429:
logger.error("API 配额不足")
logger.error(f"Baidu Qianfan HTTP {e.response.status_code}: {e}")
return []
except httpx.RequestError as e:
logger.error(f"Baidu Qianfan API request failed: {e}")
return []
except json.JSONDecodeError as e:
logger.error(f"Baidu Qianfan API response parse failed: {e}")
logger.error(f"Baidu Qianfan request failed: {e}")
return []
async def close(self):