docs: 重构 SKILL.md 为 Agent 决策驱动,更新文档版本号及 CLI docstring

This commit is contained in:
vex1023 committed 2026-05-14 18:30:42 +08:00
1 parent 39c01ebf69
commit a81661d828
9 files changed
+932 -183

No files matched your search

+27 -20
View File
@@ -1,41 +1,45 @@
#!/usr/bin/env python3
"""
Web Search Skills CLI — search across 20+ sources from the command line.
Web Search Skills CLI — search across 28 sources from the command line.
Usage:
web-search <command> <query> [options]
Commands:
search Search all sources (default)
web Search web search engines (Google, Baidu, DuckDuckGo, etc.)
baidu Search via Baidu Qianfan AI Search API (需配置 BAIDU_API_KEY)
news Search news (财联社, 华尔街见闻, RSS)
wechat Search WeChat public account articles (via Sougou)
academic Search academic papers (ArXiv)
social Search social media (Twitter/X)
rss Search RSS news feeds
health Test engine connectivity
search Search all 28 sources in parallel (default)
web Web search engines (Google, Baidu, DuckDuckGo, etc.)
baidu Baidu Qianfan AI Search API (需配置 BAIDU_API_KEY)
news News (Hacker News, GitHub Trending, 36氪, RSS, etc.)
wechat WeChat public account articles (via Sogou)
academic Academic papers (ArXiv)
social Social media (Twitter/X)
rss RSS feeds (AI newsletters, essays, podcasts)
health Test connectivity of all 28 engines (concurrent)
sources List all available sources
urls Get search URLs without executing
urls Generate search URLs without executing
Options:
-n, --max-results N Max results per source (default: 10)
-j, --json Output as JSON
-o, --output FILE Save results to file
-s, --source NAME Specific source name(s)
-r, --region REGION Region filter (cn/global)
--site SITE Restrict to specific site (baidu command only, e.g. mp.weixin.qq.com)
--recency RECENCY Time filter: day, week, month, semiyear, year (baidu command only)
-s, --source NAME Specific source name(s), support multiple
-r, --region REGION Region filter: cn / global (web only)
--site SITE Restrict to site (baidu only, e.g. mp.weixin.qq.com)
--recency RECENCY Time filter: day/week/month/semiyear/year (baidu only)
-v, --verbose Show debug logs
-h, --help Show this help
Examples:
web-search wechat 蒙面财经
web-search baidu 今日新闻 --recency week
web-search baidu "央视新闻 中美元首会晤" --site mp.weixin.qq.com --recency week
web-search news "人工智能" -n 5
web-search web "Python async" -r global -j
web-search search "量子计算" -o results.json
web-search search "AI" -s web news -n 5
web-search search "technology" -s "Hacker News" "V2EX" -n 3
web-search web "Python async" -r global -j
web-search baidu "今日新闻" --recency week
web-search baidu "中美元首会晤" --site mp.weixin.qq.com
web-search news "人工智能" -n 5
web-search wechat 蒙面财经
web-search academic "quantum computing"
web-search social "Iran nuclear" -n 5
web-search health
web-search sources
web-search urls "机器学习"
@@ -377,6 +381,9 @@ async def _run_search(args):
kwargs["site"] = args.site
if args.recency:
kwargs["recency"] = args.recency
# --source / -s: pass specific source names/categories to search
if args.source:
kwargs["sources"] = args.source
try:
results = await search_fn(
+52
View File
@@ -0,0 +1,52 @@
"""
Thread pool utilities for CPU-bound operations.
BeautifulSoup/HTML parsing is CPU-bound and can block the async event loop.
Use this shared thread pool to offload parsing work to background threads.
"""
from __future__ import annotations
import logging
from concurrent.futures import ThreadPoolExecutor
from typing import Callable, TypeVar
logger = logging.getLogger(__name__)
T = TypeVar("T")
# Shared thread pool - max_workers=4 balances parallelism with overhead
_PARSE_POOL: ThreadPoolExecutor | None = None
_MAX_WORKERS = 4
def get_parse_pool() -> ThreadPoolExecutor:
"""Get the shared thread pool for parsing operations."""
global _PARSE_POOL
if _PARSE_POOL is None:
_PARSE_POOL = ThreadPoolExecutor(
max_workers=_MAX_WORKERS,
thread_name_prefix="parse",
)
return _PARSE_POOL
async def run_in_thread(func: Callable[..., T], *args, **kwargs) -> T:
"""
Run a CPU-bound function in a thread pool to avoid blocking the event loop.
Usage:
soup = await run_in_thread(BeautifulSoup, html, "lxml")
results = await run_in_thread(parse_func, html, max_results)
"""
import asyncio
loop = asyncio.get_running_loop()
return await loop.run_in_executor(get_parse_pool(), func, *args, **kwargs)
def shutdown_pool():
"""Clean up the thread pool (call during application shutdown)."""
global _PARSE_POOL
if _PARSE_POOL:
_PARSE_POOL.shutdown(wait=True)
_PARSE_POOL = None
+80 -1
View File
@@ -27,7 +27,8 @@ from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
# RSS feed definitions (from news-summary skill + additional sources)
# RSS feed definitions
# Core sources (general news) + AI newsletters + essays + podcasts
RSS_FEEDS = {
"BBC World": {
"url": "https://feeds.bbci.co.uk/news/world/rss.xml",
@@ -64,6 +65,84 @@ RSS_FEEDS = {
"category": "news",
"lang": "en",
},
# -----------------------------------------------------------------------
# AI Newsletters (inspired by news-aggregator-skill)
# -----------------------------------------------------------------------
"Latent Space": {
"url": "https://feeds.transistor.fm/latent-space-podcast",
"category": "technology",
"lang": "en",
"subcategory": "ai_newsletter",
},
"ChinAI": {
"url": "https://chinai.substack.com/feed",
"category": "technology",
"lang": "en",
"subcategory": "ai_newsletter",
},
"Memia": {
"url": "https://memia.substack.com/feed",
"category": "technology",
"lang": "en",
"subcategory": "ai_newsletter",
},
"Interconnects": {
"url": "https://interconnects.substack.com/feed",
"category": "technology",
"lang": "en",
"subcategory": "ai_newsletter",
},
"KDnuggets": {
"url": "https://www.kdnuggets.com/feed",
"category": "technology",
"lang": "en",
"subcategory": "ai_newsletter",
},
# -----------------------------------------------------------------------
# Deep Essays & Thinkers (inspired by news-aggregator-skill)
# -----------------------------------------------------------------------
"Paul Graham": {
"url": "https://paulgraham.com/rss.xml",
"category": "technology",
"lang": "en",
"subcategory": "essay",
},
"James Clear": {
"url": "https://jamesclear.com/feed",
"category": "news",
"lang": "en",
"subcategory": "essay",
},
"Wait But Why": {
"url": "https://waitbutwhy.com/feed",
"category": "news",
"lang": "en",
"subcategory": "essay",
},
"Farnam Street": {
"url": "https://fs.blog/feed/",
"category": "news",
"lang": "en",
"subcategory": "essay",
},
# -----------------------------------------------------------------------
# Podcasts (inspired by news-aggregator-skill)
# -----------------------------------------------------------------------
"Lex Fridman Podcast": {
"url": "https://lexfridman.com/feed/podcast/",
"category": "technology",
"lang": "en",
"subcategory": "podcast",
},
"80,000 Hours": {
"url": "https://80000hours.org/feed/podcast/",
"category": "news",
"lang": "en",
"subcategory": "podcast",
},
}
+461
View File
@@ -0,0 +1,461 @@
"""
Tech & social news aggregation engines.
Implements non-Playwright news sources inspired by news-aggregator-skill:
- Hacker News (via Algolia API)
- GitHub Trending (HTML scrape)
- 36氪 (HTML scrape)
- 微博热搜 (Ajax API)
- V2EX (Public API)
- 腾讯新闻 (API)
"""
from __future__ import annotations
import logging
import re
from datetime import datetime, timezone
from typing import Any, Optional
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.core.base import SearchEngine, SearchResult
from scripts.core.thread_utils import run_in_thread
logger = logging.getLogger(__name__)
# Shared HTTP client settings
_DEFAULT_TIMEOUT = 15
_COMMON_HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
# ---------------------------------------------------------------------------
# Hacker News (via Algolia API)
# ---------------------------------------------------------------------------
class HackerNewsEngine(SearchEngine):
"""Hacker News search via Algolia public API."""
SEARCH_URL = "https://hn.algolia.com/api/v1/search"
def __init__(self, config: Optional[dict] = None):
super().__init__("Hacker News", config)
self._client = httpx.AsyncClient(
timeout=_DEFAULT_TIMEOUT,
headers={"User-Agent": _COMMON_HEADERS["User-Agent"]},
follow_redirects=True,
)
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
try:
params = {
"query": query,
"hitsPerPage": max_results,
"tags": "story",
}
response = await self._client.get(self.SEARCH_URL, params=params)
response.raise_for_status()
data = response.json()
return self._parse_results(data, max_results)
except Exception as e:
logger.error(f"Hacker News search failed: {e}")
return []
def _parse_results(self, data: dict, max_results: int) -> list[SearchResult]:
results = []
hits = data.get("hits", [])[:max_results]
for i, hit in enumerate(hits):
title = hit.get("title", "")
url = hit.get("url") or f"https://news.ycombinator.com/item?id={hit.get('objectID', '')}"
snippet = hit.get("story_text", "") or ""
# Clean HTML from snippet
snippet = re.sub(r"<[^>]+>", "", snippet)[:200]
points = hit.get("points", 0)
author = hit.get("author", "")
created_at = hit.get("created_at", "")
results.append(SearchResult(
title=title,
url=url,
snippet=snippet or f"[{points} points] by {author}",
source=self.name,
rank=i + 1,
category="news",
extra={
"points": points,
"author": author,
"time": created_at,
"heat": f"{points} points",
},
))
return results
async def close(self):
await self._client.aclose()
# ---------------------------------------------------------------------------
# GitHub Trending
# ---------------------------------------------------------------------------
class GitHubTrendingEngine(SearchEngine):
"""GitHub Trending repositories."""
TRENDING_URL = "https://github.com/trending"
def __init__(self, config: Optional[dict] = None):
super().__init__("GitHub Trending", config)
self._client = httpx.AsyncClient(
timeout=_DEFAULT_TIMEOUT,
headers=_COMMON_HEADERS,
follow_redirects=True,
)
async def search(self, query: str = "", max_results: int = 10, **kwargs) -> list[SearchResult]:
"""GitHub Trending doesn't support keyword search; returns top trending repos."""
url = self.TRENDING_URL
if query:
url += f"?spoken_language_code=&q={quote_plus(query)}"
try:
response = await self._client.get(url)
response.raise_for_status()
return await run_in_thread(self._parse_html, response.text, max_results)
except Exception as e:
logger.error(f"GitHub Trending failed: {e}")
return []
def _parse_html(self, html: str, max_results: int) -> list[SearchResult]:
soup = BeautifulSoup(html, "lxml")
results = []
articles = soup.select("article.Box-row")[:max_results]
for i, article in enumerate(articles):
h2 = article.select_one("h2")
if not h2:
continue
repo_path = h2.get_text(strip=True).replace(" ", "")
repo_url = f"https://github.com/{repo_path}"
desc_el = article.select_one("p")
description = desc_el.get_text(strip=True) if desc_el else ""
# Language
lang_el = article.select_one("[itemprop='programmingLanguage']")
language = lang_el.get_text(strip=True) if lang_el else ""
# Stars today
stars_el = article.select_one(".float-sm-right")
stars_today = stars_el.get_text(strip=True) if stars_el else ""
results.append(SearchResult(
title=repo_path,
url=repo_url,
snippet=description[:200] if description else language,
source=self.name,
rank=i + 1,
category="news",
extra={
"language": language,
"heat": stars_today,
"time": "",
},
))
return results
async def close(self):
await self._client.aclose()
# ---------------------------------------------------------------------------
# 36氪 (36kr.com)
# ---------------------------------------------------------------------------
class Kr36Engine(SearchEngine):
"""36氪 tech news search."""
SEARCH_URL = "https://36kr.com/search/articles/{keyword}"
def __init__(self, config: Optional[dict] = None):
super().__init__("36氪", config)
self._client = httpx.AsyncClient(
timeout=_DEFAULT_TIMEOUT,
headers={**_COMMON_HEADERS, "Referer": "https://36kr.com/"},
follow_redirects=True,
)
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
url = self.SEARCH_URL.format(keyword=quote_plus(query))
try:
response = await self._client.get(url)
response.raise_for_status()
return await run_in_thread(self._parse_html, response.text, query, max_results)
except Exception as e:
logger.error(f"36氪 search failed: {e}")
return []
def _parse_html(self, html: str, query: str, max_results: int) -> list[SearchResult]:
soup = BeautifulSoup(html, "lxml")
results = []
# Look for article cards
for i, item in enumerate(soup.select("a[href*='/p/'], .article-item, .search-result-item, [class*='article']")[:max_results]):
title_el = item.select_one("h3, h4, .title, [class*='title']")
link = item.get("href", "")
snippet_el = item.select_one(".abstract, .summary, .desc, p, [class*='desc']")
title = title_el.get_text(strip=True) if title_el else ""
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if not title and not link:
continue
full_url = link if link.startswith("http") else f"https://36kr.com{link}"
results.append(SearchResult(
title=title or f"36氪: {query}",
url=full_url,
snippet=snippet[:200] if snippet else "",
source=self.name,
rank=i + 1,
category="news",
extra={"region": "cn"},
))
if not results:
search_url = f"https://36kr.com/search/articles/{quote_plus(query)}"
results.append(SearchResult(
title=f"36氪: {query}",
url=search_url,
snippet="Search 36氪 (site may require JS)",
source=self.name,
rank=1,
category="news",
extra={"search_url": search_url, "note": "JS-rendered page"},
))
return results
async def close(self):
await self._client.aclose()
# ---------------------------------------------------------------------------
# 微博热搜 (Weibo Trending)
# ---------------------------------------------------------------------------
class WeiboEngine(SearchEngine):
"""微博热搜 trending topics."""
HOT_URL = "https://weibo.com/ajax/side/hotSearch"
def __init__(self, config: Optional[dict] = None):
super().__init__("微博热搜", config)
self._client = httpx.AsyncClient(
timeout=_DEFAULT_TIMEOUT,
headers={**_COMMON_HEADERS, "Referer": "https://weibo.com/"},
follow_redirects=True,
)
async def search(self, query: str = "", max_results: int = 10, **kwargs) -> list[SearchResult]:
try:
response = await self._client.get(self.HOT_URL)
response.raise_for_status()
data = response.json()
return self._parse_results(data, query, max_results)
except Exception as e:
logger.error(f"微博热搜 failed: {e}")
return []
def _parse_results(self, data: dict, query: str, max_results: int) -> list[SearchResult]:
results = []
realtime = data.get("data", {}).get("realtime", [])[:max_results]
for i, item in enumerate(realtime):
word = item.get("word", "")
raw_hot = item.get("raw_hot", 0)
url = f"https://s.weibo.com/weibo?q={quote_plus(word)}"
# Filter by keyword if query provided
if query and query.lower() not in word.lower():
continue
results.append(SearchResult(
title=f"#{word}#",
url=url,
snippet=f"热度: {raw_hot:,}" if raw_hot else "",
source=self.name,
rank=i + 1,
category="news",
extra={
"heat": f"{raw_hot:,}" if raw_hot else "",
"region": "cn",
},
))
return results
async def close(self):
await self._client.aclose()
# ---------------------------------------------------------------------------
# V2EX (via public API)
# ---------------------------------------------------------------------------
class V2EXEngine(SearchEngine):
"""V2EX tech community hot topics."""
HOT_API = "https://www.v2ex.com/api/v2/topics/hot"
# Fallback to older API if v2 needs auth
HOT_API_FALLBACK = "https://www.v2ex.com/api/topics/hot.json"
def __init__(self, config: Optional[dict] = None):
super().__init__("V2EX", config)
self._client = httpx.AsyncClient(
timeout=_DEFAULT_TIMEOUT,
headers=_COMMON_HEADERS,
follow_redirects=True,
)
async def search(self, query: str = "", max_results: int = 10, **kwargs) -> list[SearchResult]:
# Try v2 API first, fallback to old API
for api_url in [self.HOT_API, self.HOT_API_FALLBACK]:
try:
response = await self._client.get(api_url)
if response.status_code == 200:
data = response.json()
# v2 API returns list directly; old API also returns list
topics = data if isinstance(data, list) else data.get("data", [])
return self._parse_results(topics, query, max_results)
except Exception:
continue
logger.error("V2EX API failed (both v2 and fallback)")
return []
def _parse_results(self, topics: list, query: str, max_results: int) -> list[SearchResult]:
results = []
for i, topic in enumerate(topics[:max_results]):
title = topic.get("title", "")
node_name = ""
if "node" in topic and isinstance(topic["node"], dict):
node_name = topic["node"].get("title", "")
topic_id = topic.get("id", "")
url = f"https://www.v2ex.com/t/{topic_id}"
replies = topic.get("replies", 0)
created = topic.get("created", "")
member = ""
if "member" in topic and isinstance(topic["member"], dict):
member = topic["member"].get("username", "")
# Filter by keyword
if query and query.lower() not in title.lower():
continue
# Format time
time_str = ""
if created:
try:
dt = datetime.fromtimestamp(int(created), tz=timezone.utc)
time_str = dt.strftime("%Y-%m-%d %H:%M")
except (ValueError, OSError):
time_str = str(created)
results.append(SearchResult(
title=title,
url=url,
snippet=f"[{node_name}] by {member}" if node_name and member else f"by {member}" if member else "",
source=self.name,
rank=i + 1,
category="news",
extra={
"heat": f"{replies} replies",
"author": member,
"time": time_str,
"region": "cn",
},
))
return results
async def close(self):
await self._client.aclose()
# ---------------------------------------------------------------------------
# 腾讯新闻 (news.qq.com)
# ---------------------------------------------------------------------------
class TencentNewsEngine(SearchEngine):
"""腾讯新闻 search."""
SEARCH_URL = "https://news.qq.com/search"
def __init__(self, config: Optional[dict] = None):
super().__init__("腾讯新闻", config)
self._client = httpx.AsyncClient(
timeout=_DEFAULT_TIMEOUT,
headers={**_COMMON_HEADERS, "Referer": "https://news.qq.com/"},
follow_redirects=True,
)
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
url = f"{self.SEARCH_URL}?q={quote_plus(query)}"
try:
response = await self._client.get(url)
response.raise_for_status()
return await run_in_thread(self._parse_html, response.text, query, max_results)
except Exception as e:
logger.error(f"腾讯新闻 search failed: {e}")
return []
def _parse_html(self, html: str, query: str, max_results: int) -> list[SearchResult]:
soup = BeautifulSoup(html, "lxml")
results = []
for i, item in enumerate(soup.select("a[href*='news.qq.com'], .list-item, [class*='result'], article")[:max_results]):
title_el = item.select_one("h3, h4, .title, a")
link = item.get("href", "") or (title_el.get("href", "") if title_el else "")
snippet_el = item.select_one(".summary, .desc, .abstract, p, .text")
title = title_el.get_text(strip=True) if title_el else ""
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if not title:
continue
full_url = link if link.startswith("http") else f"https:{link}" if link.startswith("//") else link
results.append(SearchResult(
title=title,
url=full_url,
snippet=snippet[:200] if snippet else "",
source=self.name,
rank=i + 1,
category="news",
extra={"region": "cn"},
))
if not results:
search_url = f"{self.SEARCH_URL}?q={quote_plus(query)}"
results.append(SearchResult(
title=f"腾讯新闻: {query}",
url=search_url,
snippet=f"Search Tencent News for '{query}'",
source=self.name,
rank=1,
category="news",
extra={"search_url": search_url, "note": "JS-rendered page"},
))
return results
async def close(self):
await self._client.aclose()
+49 -20
View File
@@ -21,6 +21,10 @@ from scripts.engines.parser_engines import DuckDuckGoEngine
from scripts.news.cls import CLSNewsEngine
from scripts.news.wallstreetcn import WallStreetCNEngine
from scripts.news.rss import RSSNewsEngine
from scripts.news.tech_news import (
HackerNewsEngine, GitHubTrendingEngine, Kr36Engine,
WeiboEngine, V2EXEngine, TencentNewsEngine,
)
from scripts.academic.arxiv import ArXivEngine
from scripts.wechat.sogou_weixin import SogouWeChatEngine
from scripts.social.twitter import TwitterSearchEngine
@@ -132,32 +136,45 @@ class UnifiedSearch:
return []
async def search_health(self) -> list[dict[str, Any]]:
"""Check health of all configured engines."""
statuses = []
seen_names = set()
for cat_key, cat_fn in SOURCE_CATEGORIES.items():
"""Check health of all configured engines (concurrent)."""
seen_names: set[str] = set()
tasks: list[asyncio.Task] = []
for cat_fn in SOURCE_CATEGORIES.values():
for engine_def in cat_fn():
name = engine_def.get("name", "?")
if name in seen_names:
continue
seen_names.add(name)
engine = self._get_engine(engine_def)
if engine is None:
statuses.append({"name": name, "status": "error", "detail": "engine_init_failed"})
continue
try:
async with self._semaphore:
await asyncio.wait_for(
engine.search("health check", max_results=1),
timeout=8,
)
statuses.append({"name": name, "status": "ok"})
except asyncio.TimeoutError:
statuses.append({"name": name, "status": "timeout"})
except Exception as e:
statuses.append({"name": name, "status": "error", "detail": str(e)[:80]})
tasks.append(asyncio.create_task(self._check_one_engine(engine_def, name)))
results = await asyncio.gather(*tasks, return_exceptions=True)
statuses: list[dict[str, Any]] = []
for r in results:
if isinstance(r, Exception):
statuses.append({"name": "?", "status": "error", "detail": str(r)[:80]})
else:
statuses.append(r)
return statuses
async def _check_one_engine(
self, engine_def: dict[str, Any], name: str,
) -> dict[str, Any]:
"""Check health of a single engine."""
engine = self._get_engine(engine_def)
if engine is None:
return {"name": name, "status": "error", "detail": "engine_init_failed"}
try:
async with self._semaphore:
await asyncio.wait_for(
engine.search("health check", max_results=1),
timeout=8,
)
return {"name": name, "status": "ok"}
except asyncio.TimeoutError:
return {"name": name, "status": "timeout"}
except Exception as e:
return {"name": name, "status": "error", "detail": str(e)[:80]}
def _get_engine(self, engine_def: dict[str, Any]):
"""Get or create an engine instance by definition."""
name = engine_def.get("name", "")
@@ -180,7 +197,19 @@ class UnifiedSearch:
engine = SogouWeChatEngine(self.config)
elif "twitter" in name.lower():
engine = TwitterSearchEngine(self.config)
elif engine_type == "news" and name not in ("cls", "wallstreetcn"):
elif name == "Hacker News":
engine = HackerNewsEngine(self.config)
elif name == "GitHub Trending":
engine = GitHubTrendingEngine(self.config)
elif name == "36氪":
engine = Kr36Engine(self.config)
elif name == "微博热搜":
engine = WeiboEngine(self.config)
elif name == "V2EX":
engine = V2EXEngine(self.config)
elif name == "腾讯新闻":
engine = TencentNewsEngine(self.config)
elif engine_type == "news" and name not in ("cls", "wallstreetcn", "Hacker News", "GitHub Trending", "36氪", "微博热搜", "V2EX", "腾讯新闻"):
engine = RSSNewsEngine(self.config)
elif engine_type == "social":
engine = TwitterSearchEngine(self.config) if "twitter" in name.lower() else UrlSearchEngine(name, engine_def)