docs: 重构 SKILL.md 为 Agent 决策驱动,更新文档版本号及 CLI docstring
This commit is contained in:
1 parent
39c01ebf69
commit
a81661d828
9 files changed
+932
-183
No files matched your search
+27
-20
@@ -1,41 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Web Search Skills CLI — search across 20+ sources from the command line.
|
||||
Web Search Skills CLI — search across 28 sources from the command line.
|
||||
|
||||
Usage:
|
||||
web-search <command> <query> [options]
|
||||
|
||||
Commands:
|
||||
search Search all sources (default)
|
||||
web Search web search engines (Google, Baidu, DuckDuckGo, etc.)
|
||||
baidu Search via Baidu Qianfan AI Search API (需配置 BAIDU_API_KEY)
|
||||
news Search news (财联社, 华尔街见闻, RSS)
|
||||
wechat Search WeChat public account articles (via Sougou)
|
||||
academic Search academic papers (ArXiv)
|
||||
social Search social media (Twitter/X)
|
||||
rss Search RSS news feeds
|
||||
health Test engine connectivity
|
||||
search Search all 28 sources in parallel (default)
|
||||
web Web search engines (Google, Baidu, DuckDuckGo, etc.)
|
||||
baidu Baidu Qianfan AI Search API (需配置 BAIDU_API_KEY)
|
||||
news News (Hacker News, GitHub Trending, 36氪, RSS, etc.)
|
||||
wechat WeChat public account articles (via Sogou)
|
||||
academic Academic papers (ArXiv)
|
||||
social Social media (Twitter/X)
|
||||
rss RSS feeds (AI newsletters, essays, podcasts)
|
||||
health Test connectivity of all 28 engines (concurrent)
|
||||
sources List all available sources
|
||||
urls Get search URLs without executing
|
||||
urls Generate search URLs without executing
|
||||
|
||||
Options:
|
||||
-n, --max-results N Max results per source (default: 10)
|
||||
-j, --json Output as JSON
|
||||
-o, --output FILE Save results to file
|
||||
-s, --source NAME Specific source name(s)
|
||||
-r, --region REGION Region filter (cn/global)
|
||||
--site SITE Restrict to specific site (baidu command only, e.g. mp.weixin.qq.com)
|
||||
--recency RECENCY Time filter: day, week, month, semiyear, year (baidu command only)
|
||||
-s, --source NAME Specific source name(s), support multiple
|
||||
-r, --region REGION Region filter: cn / global (web only)
|
||||
--site SITE Restrict to site (baidu only, e.g. mp.weixin.qq.com)
|
||||
--recency RECENCY Time filter: day/week/month/semiyear/year (baidu only)
|
||||
-v, --verbose Show debug logs
|
||||
-h, --help Show this help
|
||||
|
||||
Examples:
|
||||
web-search wechat 蒙面财经
|
||||
web-search baidu 今日新闻 --recency week
|
||||
web-search baidu "央视新闻 中美元首会晤" --site mp.weixin.qq.com --recency week
|
||||
web-search news "人工智能" -n 5
|
||||
web-search web "Python async" -r global -j
|
||||
web-search search "量子计算" -o results.json
|
||||
web-search search "AI" -s web news -n 5
|
||||
web-search search "technology" -s "Hacker News" "V2EX" -n 3
|
||||
web-search web "Python async" -r global -j
|
||||
web-search baidu "今日新闻" --recency week
|
||||
web-search baidu "中美元首会晤" --site mp.weixin.qq.com
|
||||
web-search news "人工智能" -n 5
|
||||
web-search wechat 蒙面财经
|
||||
web-search academic "quantum computing"
|
||||
web-search social "Iran nuclear" -n 5
|
||||
web-search health
|
||||
web-search sources
|
||||
web-search urls "机器学习"
|
||||
@@ -377,6 +381,9 @@ async def _run_search(args):
|
||||
kwargs["site"] = args.site
|
||||
if args.recency:
|
||||
kwargs["recency"] = args.recency
|
||||
# --source / -s: pass specific source names/categories to search
|
||||
if args.source:
|
||||
kwargs["sources"] = args.source
|
||||
|
||||
try:
|
||||
results = await search_fn(
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""
|
||||
Thread pool utilities for CPU-bound operations.
|
||||
|
||||
BeautifulSoup/HTML parsing is CPU-bound and can block the async event loop.
|
||||
Use this shared thread pool to offload parsing work to background threads.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Callable, TypeVar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
# Shared thread pool - max_workers=4 balances parallelism with overhead
|
||||
_PARSE_POOL: ThreadPoolExecutor | None = None
|
||||
_MAX_WORKERS = 4
|
||||
|
||||
|
||||
def get_parse_pool() -> ThreadPoolExecutor:
|
||||
"""Get the shared thread pool for parsing operations."""
|
||||
global _PARSE_POOL
|
||||
if _PARSE_POOL is None:
|
||||
_PARSE_POOL = ThreadPoolExecutor(
|
||||
max_workers=_MAX_WORKERS,
|
||||
thread_name_prefix="parse",
|
||||
)
|
||||
return _PARSE_POOL
|
||||
|
||||
|
||||
async def run_in_thread(func: Callable[..., T], *args, **kwargs) -> T:
|
||||
"""
|
||||
Run a CPU-bound function in a thread pool to avoid blocking the event loop.
|
||||
|
||||
Usage:
|
||||
soup = await run_in_thread(BeautifulSoup, html, "lxml")
|
||||
results = await run_in_thread(parse_func, html, max_results)
|
||||
"""
|
||||
import asyncio
|
||||
loop = asyncio.get_running_loop()
|
||||
return await loop.run_in_executor(get_parse_pool(), func, *args, **kwargs)
|
||||
|
||||
|
||||
def shutdown_pool():
|
||||
"""Clean up the thread pool (call during application shutdown)."""
|
||||
global _PARSE_POOL
|
||||
if _PARSE_POOL:
|
||||
_PARSE_POOL.shutdown(wait=True)
|
||||
_PARSE_POOL = None
|
||||
+80
-1
@@ -27,7 +27,8 @@ from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# RSS feed definitions (from news-summary skill + additional sources)
|
||||
# RSS feed definitions
|
||||
# Core sources (general news) + AI newsletters + essays + podcasts
|
||||
RSS_FEEDS = {
|
||||
"BBC World": {
|
||||
"url": "https://feeds.bbci.co.uk/news/world/rss.xml",
|
||||
@@ -64,6 +65,84 @@ RSS_FEEDS = {
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
},
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# AI Newsletters (inspired by news-aggregator-skill)
|
||||
# -----------------------------------------------------------------------
|
||||
"Latent Space": {
|
||||
"url": "https://feeds.transistor.fm/latent-space-podcast",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "ai_newsletter",
|
||||
},
|
||||
"ChinAI": {
|
||||
"url": "https://chinai.substack.com/feed",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "ai_newsletter",
|
||||
},
|
||||
"Memia": {
|
||||
"url": "https://memia.substack.com/feed",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "ai_newsletter",
|
||||
},
|
||||
"Interconnects": {
|
||||
"url": "https://interconnects.substack.com/feed",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "ai_newsletter",
|
||||
},
|
||||
"KDnuggets": {
|
||||
"url": "https://www.kdnuggets.com/feed",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "ai_newsletter",
|
||||
},
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Deep Essays & Thinkers (inspired by news-aggregator-skill)
|
||||
# -----------------------------------------------------------------------
|
||||
"Paul Graham": {
|
||||
"url": "https://paulgraham.com/rss.xml",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "essay",
|
||||
},
|
||||
"James Clear": {
|
||||
"url": "https://jamesclear.com/feed",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
"subcategory": "essay",
|
||||
},
|
||||
"Wait But Why": {
|
||||
"url": "https://waitbutwhy.com/feed",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
"subcategory": "essay",
|
||||
},
|
||||
"Farnam Street": {
|
||||
"url": "https://fs.blog/feed/",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
"subcategory": "essay",
|
||||
},
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Podcasts (inspired by news-aggregator-skill)
|
||||
# -----------------------------------------------------------------------
|
||||
"Lex Fridman Podcast": {
|
||||
"url": "https://lexfridman.com/feed/podcast/",
|
||||
"category": "technology",
|
||||
"lang": "en",
|
||||
"subcategory": "podcast",
|
||||
},
|
||||
"80,000 Hours": {
|
||||
"url": "https://80000hours.org/feed/podcast/",
|
||||
"category": "news",
|
||||
"lang": "en",
|
||||
"subcategory": "podcast",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,461 @@
|
||||
"""
|
||||
Tech & social news aggregation engines.
|
||||
|
||||
Implements non-Playwright news sources inspired by news-aggregator-skill:
|
||||
- Hacker News (via Algolia API)
|
||||
- GitHub Trending (HTML scrape)
|
||||
- 36氪 (HTML scrape)
|
||||
- 微博热搜 (Ajax API)
|
||||
- V2EX (Public API)
|
||||
- 腾讯新闻 (API)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
from scripts.core.thread_utils import run_in_thread
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Shared HTTP client settings
|
||||
_DEFAULT_TIMEOUT = 15
|
||||
_COMMON_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hacker News (via Algolia API)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class HackerNewsEngine(SearchEngine):
|
||||
"""Hacker News search via Algolia public API."""
|
||||
|
||||
SEARCH_URL = "https://hn.algolia.com/api/v1/search"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("Hacker News", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers={"User-Agent": _COMMON_HEADERS["User-Agent"]},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
try:
|
||||
params = {
|
||||
"query": query,
|
||||
"hitsPerPage": max_results,
|
||||
"tags": "story",
|
||||
}
|
||||
response = await self._client.get(self.SEARCH_URL, params=params)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return self._parse_results(data, max_results)
|
||||
except Exception as e:
|
||||
logger.error(f"Hacker News search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_results(self, data: dict, max_results: int) -> list[SearchResult]:
|
||||
results = []
|
||||
hits = data.get("hits", [])[:max_results]
|
||||
for i, hit in enumerate(hits):
|
||||
title = hit.get("title", "")
|
||||
url = hit.get("url") or f"https://news.ycombinator.com/item?id={hit.get('objectID', '')}"
|
||||
snippet = hit.get("story_text", "") or ""
|
||||
# Clean HTML from snippet
|
||||
snippet = re.sub(r"<[^>]+>", "", snippet)[:200]
|
||||
points = hit.get("points", 0)
|
||||
author = hit.get("author", "")
|
||||
created_at = hit.get("created_at", "")
|
||||
|
||||
results.append(SearchResult(
|
||||
title=title,
|
||||
url=url,
|
||||
snippet=snippet or f"[{points} points] by {author}",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={
|
||||
"points": points,
|
||||
"author": author,
|
||||
"time": created_at,
|
||||
"heat": f"{points} points",
|
||||
},
|
||||
))
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GitHub Trending
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class GitHubTrendingEngine(SearchEngine):
|
||||
"""GitHub Trending repositories."""
|
||||
|
||||
TRENDING_URL = "https://github.com/trending"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("GitHub Trending", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers=_COMMON_HEADERS,
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str = "", max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
"""GitHub Trending doesn't support keyword search; returns top trending repos."""
|
||||
url = self.TRENDING_URL
|
||||
if query:
|
||||
url += f"?spoken_language_code=&q={quote_plus(query)}"
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return await run_in_thread(self._parse_html, response.text, max_results)
|
||||
except Exception as e:
|
||||
logger.error(f"GitHub Trending failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_html(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
articles = soup.select("article.Box-row")[:max_results]
|
||||
for i, article in enumerate(articles):
|
||||
h2 = article.select_one("h2")
|
||||
if not h2:
|
||||
continue
|
||||
repo_path = h2.get_text(strip=True).replace(" ", "")
|
||||
repo_url = f"https://github.com/{repo_path}"
|
||||
|
||||
desc_el = article.select_one("p")
|
||||
description = desc_el.get_text(strip=True) if desc_el else ""
|
||||
|
||||
# Language
|
||||
lang_el = article.select_one("[itemprop='programmingLanguage']")
|
||||
language = lang_el.get_text(strip=True) if lang_el else ""
|
||||
|
||||
# Stars today
|
||||
stars_el = article.select_one(".float-sm-right")
|
||||
stars_today = stars_el.get_text(strip=True) if stars_el else ""
|
||||
|
||||
results.append(SearchResult(
|
||||
title=repo_path,
|
||||
url=repo_url,
|
||||
snippet=description[:200] if description else language,
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={
|
||||
"language": language,
|
||||
"heat": stars_today,
|
||||
"time": "",
|
||||
},
|
||||
))
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 36氪 (36kr.com)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class Kr36Engine(SearchEngine):
|
||||
"""36氪 tech news search."""
|
||||
|
||||
SEARCH_URL = "https://36kr.com/search/articles/{keyword}"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("36氪", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers={**_COMMON_HEADERS, "Referer": "https://36kr.com/"},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
url = self.SEARCH_URL.format(keyword=quote_plus(query))
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return await run_in_thread(self._parse_html, response.text, query, max_results)
|
||||
except Exception as e:
|
||||
logger.error(f"36氪 search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_html(self, html: str, query: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
|
||||
# Look for article cards
|
||||
for i, item in enumerate(soup.select("a[href*='/p/'], .article-item, .search-result-item, [class*='article']")[:max_results]):
|
||||
title_el = item.select_one("h3, h4, .title, [class*='title']")
|
||||
link = item.get("href", "")
|
||||
snippet_el = item.select_one(".abstract, .summary, .desc, p, [class*='desc']")
|
||||
|
||||
title = title_el.get_text(strip=True) if title_el else ""
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if not title and not link:
|
||||
continue
|
||||
|
||||
full_url = link if link.startswith("http") else f"https://36kr.com{link}"
|
||||
|
||||
results.append(SearchResult(
|
||||
title=title or f"36氪: {query}",
|
||||
url=full_url,
|
||||
snippet=snippet[:200] if snippet else "",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={"region": "cn"},
|
||||
))
|
||||
|
||||
if not results:
|
||||
search_url = f"https://36kr.com/search/articles/{quote_plus(query)}"
|
||||
results.append(SearchResult(
|
||||
title=f"36氪: {query}",
|
||||
url=search_url,
|
||||
snippet="Search 36氪 (site may require JS)",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="news",
|
||||
extra={"search_url": search_url, "note": "JS-rendered page"},
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 微博热搜 (Weibo Trending)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class WeiboEngine(SearchEngine):
|
||||
"""微博热搜 trending topics."""
|
||||
|
||||
HOT_URL = "https://weibo.com/ajax/side/hotSearch"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("微博热搜", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers={**_COMMON_HEADERS, "Referer": "https://weibo.com/"},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str = "", max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
try:
|
||||
response = await self._client.get(self.HOT_URL)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return self._parse_results(data, query, max_results)
|
||||
except Exception as e:
|
||||
logger.error(f"微博热搜 failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_results(self, data: dict, query: str, max_results: int) -> list[SearchResult]:
|
||||
results = []
|
||||
realtime = data.get("data", {}).get("realtime", [])[:max_results]
|
||||
for i, item in enumerate(realtime):
|
||||
word = item.get("word", "")
|
||||
raw_hot = item.get("raw_hot", 0)
|
||||
url = f"https://s.weibo.com/weibo?q={quote_plus(word)}"
|
||||
|
||||
# Filter by keyword if query provided
|
||||
if query and query.lower() not in word.lower():
|
||||
continue
|
||||
|
||||
results.append(SearchResult(
|
||||
title=f"#{word}#",
|
||||
url=url,
|
||||
snippet=f"热度: {raw_hot:,}" if raw_hot else "",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={
|
||||
"heat": f"{raw_hot:,}" if raw_hot else "",
|
||||
"region": "cn",
|
||||
},
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# V2EX (via public API)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class V2EXEngine(SearchEngine):
|
||||
"""V2EX tech community hot topics."""
|
||||
|
||||
HOT_API = "https://www.v2ex.com/api/v2/topics/hot"
|
||||
# Fallback to older API if v2 needs auth
|
||||
HOT_API_FALLBACK = "https://www.v2ex.com/api/topics/hot.json"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("V2EX", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers=_COMMON_HEADERS,
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str = "", max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
# Try v2 API first, fallback to old API
|
||||
for api_url in [self.HOT_API, self.HOT_API_FALLBACK]:
|
||||
try:
|
||||
response = await self._client.get(api_url)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
# v2 API returns list directly; old API also returns list
|
||||
topics = data if isinstance(data, list) else data.get("data", [])
|
||||
return self._parse_results(topics, query, max_results)
|
||||
except Exception:
|
||||
continue
|
||||
logger.error("V2EX API failed (both v2 and fallback)")
|
||||
return []
|
||||
|
||||
def _parse_results(self, topics: list, query: str, max_results: int) -> list[SearchResult]:
|
||||
results = []
|
||||
for i, topic in enumerate(topics[:max_results]):
|
||||
title = topic.get("title", "")
|
||||
node_name = ""
|
||||
if "node" in topic and isinstance(topic["node"], dict):
|
||||
node_name = topic["node"].get("title", "")
|
||||
topic_id = topic.get("id", "")
|
||||
url = f"https://www.v2ex.com/t/{topic_id}"
|
||||
replies = topic.get("replies", 0)
|
||||
created = topic.get("created", "")
|
||||
member = ""
|
||||
if "member" in topic and isinstance(topic["member"], dict):
|
||||
member = topic["member"].get("username", "")
|
||||
|
||||
# Filter by keyword
|
||||
if query and query.lower() not in title.lower():
|
||||
continue
|
||||
|
||||
# Format time
|
||||
time_str = ""
|
||||
if created:
|
||||
try:
|
||||
dt = datetime.fromtimestamp(int(created), tz=timezone.utc)
|
||||
time_str = dt.strftime("%Y-%m-%d %H:%M")
|
||||
except (ValueError, OSError):
|
||||
time_str = str(created)
|
||||
|
||||
results.append(SearchResult(
|
||||
title=title,
|
||||
url=url,
|
||||
snippet=f"[{node_name}] by {member}" if node_name and member else f"by {member}" if member else "",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={
|
||||
"heat": f"{replies} replies",
|
||||
"author": member,
|
||||
"time": time_str,
|
||||
"region": "cn",
|
||||
},
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 腾讯新闻 (news.qq.com)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TencentNewsEngine(SearchEngine):
|
||||
"""腾讯新闻 search."""
|
||||
|
||||
SEARCH_URL = "https://news.qq.com/search"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("腾讯新闻", config)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=_DEFAULT_TIMEOUT,
|
||||
headers={**_COMMON_HEADERS, "Referer": "https://news.qq.com/"},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
|
||||
url = f"{self.SEARCH_URL}?q={quote_plus(query)}"
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
response.raise_for_status()
|
||||
return await run_in_thread(self._parse_html, response.text, query, max_results)
|
||||
except Exception as e:
|
||||
logger.error(f"腾讯新闻 search failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_html(self, html: str, query: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
|
||||
for i, item in enumerate(soup.select("a[href*='news.qq.com'], .list-item, [class*='result'], article")[:max_results]):
|
||||
title_el = item.select_one("h3, h4, .title, a")
|
||||
link = item.get("href", "") or (title_el.get("href", "") if title_el else "")
|
||||
snippet_el = item.select_one(".summary, .desc, .abstract, p, .text")
|
||||
|
||||
title = title_el.get_text(strip=True) if title_el else ""
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if not title:
|
||||
continue
|
||||
|
||||
full_url = link if link.startswith("http") else f"https:{link}" if link.startswith("//") else link
|
||||
|
||||
results.append(SearchResult(
|
||||
title=title,
|
||||
url=full_url,
|
||||
snippet=snippet[:200] if snippet else "",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="news",
|
||||
extra={"region": "cn"},
|
||||
))
|
||||
|
||||
if not results:
|
||||
search_url = f"{self.SEARCH_URL}?q={quote_plus(query)}"
|
||||
results.append(SearchResult(
|
||||
title=f"腾讯新闻: {query}",
|
||||
url=search_url,
|
||||
snippet=f"Search Tencent News for '{query}'",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="news",
|
||||
extra={"search_url": search_url, "note": "JS-rendered page"},
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
+49
-20
@@ -21,6 +21,10 @@ from scripts.engines.parser_engines import DuckDuckGoEngine
|
||||
from scripts.news.cls import CLSNewsEngine
|
||||
from scripts.news.wallstreetcn import WallStreetCNEngine
|
||||
from scripts.news.rss import RSSNewsEngine
|
||||
from scripts.news.tech_news import (
|
||||
HackerNewsEngine, GitHubTrendingEngine, Kr36Engine,
|
||||
WeiboEngine, V2EXEngine, TencentNewsEngine,
|
||||
)
|
||||
from scripts.academic.arxiv import ArXivEngine
|
||||
from scripts.wechat.sogou_weixin import SogouWeChatEngine
|
||||
from scripts.social.twitter import TwitterSearchEngine
|
||||
@@ -132,32 +136,45 @@ class UnifiedSearch:
|
||||
return []
|
||||
|
||||
async def search_health(self) -> list[dict[str, Any]]:
|
||||
"""Check health of all configured engines."""
|
||||
statuses = []
|
||||
seen_names = set()
|
||||
for cat_key, cat_fn in SOURCE_CATEGORIES.items():
|
||||
"""Check health of all configured engines (concurrent)."""
|
||||
seen_names: set[str] = set()
|
||||
tasks: list[asyncio.Task] = []
|
||||
for cat_fn in SOURCE_CATEGORIES.values():
|
||||
for engine_def in cat_fn():
|
||||
name = engine_def.get("name", "?")
|
||||
if name in seen_names:
|
||||
continue
|
||||
seen_names.add(name)
|
||||
engine = self._get_engine(engine_def)
|
||||
if engine is None:
|
||||
statuses.append({"name": name, "status": "error", "detail": "engine_init_failed"})
|
||||
continue
|
||||
try:
|
||||
async with self._semaphore:
|
||||
await asyncio.wait_for(
|
||||
engine.search("health check", max_results=1),
|
||||
timeout=8,
|
||||
)
|
||||
statuses.append({"name": name, "status": "ok"})
|
||||
except asyncio.TimeoutError:
|
||||
statuses.append({"name": name, "status": "timeout"})
|
||||
except Exception as e:
|
||||
statuses.append({"name": name, "status": "error", "detail": str(e)[:80]})
|
||||
tasks.append(asyncio.create_task(self._check_one_engine(engine_def, name)))
|
||||
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
statuses: list[dict[str, Any]] = []
|
||||
for r in results:
|
||||
if isinstance(r, Exception):
|
||||
statuses.append({"name": "?", "status": "error", "detail": str(r)[:80]})
|
||||
else:
|
||||
statuses.append(r)
|
||||
return statuses
|
||||
|
||||
async def _check_one_engine(
|
||||
self, engine_def: dict[str, Any], name: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Check health of a single engine."""
|
||||
engine = self._get_engine(engine_def)
|
||||
if engine is None:
|
||||
return {"name": name, "status": "error", "detail": "engine_init_failed"}
|
||||
try:
|
||||
async with self._semaphore:
|
||||
await asyncio.wait_for(
|
||||
engine.search("health check", max_results=1),
|
||||
timeout=8,
|
||||
)
|
||||
return {"name": name, "status": "ok"}
|
||||
except asyncio.TimeoutError:
|
||||
return {"name": name, "status": "timeout"}
|
||||
except Exception as e:
|
||||
return {"name": name, "status": "error", "detail": str(e)[:80]}
|
||||
|
||||
def _get_engine(self, engine_def: dict[str, Any]):
|
||||
"""Get or create an engine instance by definition."""
|
||||
name = engine_def.get("name", "")
|
||||
@@ -180,7 +197,19 @@ class UnifiedSearch:
|
||||
engine = SogouWeChatEngine(self.config)
|
||||
elif "twitter" in name.lower():
|
||||
engine = TwitterSearchEngine(self.config)
|
||||
elif engine_type == "news" and name not in ("cls", "wallstreetcn"):
|
||||
elif name == "Hacker News":
|
||||
engine = HackerNewsEngine(self.config)
|
||||
elif name == "GitHub Trending":
|
||||
engine = GitHubTrendingEngine(self.config)
|
||||
elif name == "36氪":
|
||||
engine = Kr36Engine(self.config)
|
||||
elif name == "微博热搜":
|
||||
engine = WeiboEngine(self.config)
|
||||
elif name == "V2EX":
|
||||
engine = V2EXEngine(self.config)
|
||||
elif name == "腾讯新闻":
|
||||
engine = TencentNewsEngine(self.config)
|
||||
elif engine_type == "news" and name not in ("cls", "wallstreetcn", "Hacker News", "GitHub Trending", "36氪", "微博热搜", "V2EX", "腾讯新闻"):
|
||||
engine = RSSNewsEngine(self.config)
|
||||
elif engine_type == "social":
|
||||
engine = TwitterSearchEngine(self.config) if "twitter" in name.lower() else UrlSearchEngine(name, engine_def)
|
||||
|
||||
Reference in new issue
Block a user