diff --git a/SKILL.md b/SKILL.md index be8f472..cc54da1 100644 --- a/SKILL.md +++ b/SKILL.md @@ -191,7 +191,8 @@ This registers `web-search` as a global command. Requires Python 3.8+. ### Known Limitations - **WeChat** via Sogou has anti-scraping limits, may return empty -- **财联社 / 华尔街见闻** need JS rendering → returns search page URL, not parsed results +- **财联社** uses public API (telegraphList) — returns parsed real-time news +- **华尔街见闻** need JS rendering → returns search page URL, not parsed results - **Baidu Qianfan** requires `BAIDU_API_KEY` env var - **RSS feeds** need stable internet - **Windows GBK** terminals: special chars auto-replaced by `_sanitize_for_console()` diff --git a/config/engines.json b/config/engines.json index 101f23b..b624d05 100644 --- a/config/engines.json +++ b/config/engines.json @@ -22,7 +22,7 @@ {"name": "WolframAlpha", "url": "https://www.wolframalpha.com/input?i={keyword}", "region": "global", "type": "url"} ], "news_sources": [ - {"name": "cls", "url": "https://www.cls.cn/searchPage?keyword={keyword}&type=all", "region": "cn", "type": "news"}, + {"name": "财联社", "url": "https://www.cls.cn/searchPage?keyword={keyword}&type=all", "region": "cn", "type": "news"}, {"name": "wallstreetcn", "url": "https://wallstreetcn.com/search?q={keyword}", "region": "cn", "type": "news"}, {"name": "Hacker News", "type": "news", "region": "global"}, {"name": "GitHub Trending", "type": "news", "region": "global"}, diff --git a/scripts/news/cls.py b/scripts/news/cls.py index 07fbf1e..27f80ec 100644 --- a/scripts/news/cls.py +++ b/scripts/news/cls.py @@ -1,115 +1,186 @@ """ 财联社 (cls.cn) news search engine. -Scrapes cls.cn search results for financial news. -""" +Uses the public telegraph API (nodeapi/telegraphList) to fetch real-time +financial news without JS rendering. Falls back to search page URL for +keyword-based search queries. +API details (discovered via frontend reverse-engineering): + GET https://www.cls.cn/nodeapi/telegraphList + - Parameters: app=CailianpressWeb, os=web, sv=7.7.5, rn=N, last_time=timestamp + - Sign: SHA1(plaintext) → MD5(hex) (anti-scraping measure) + - Returns: {"data":{"roll_data":[{"ctime":..., "content":"..."}]}} +""" from __future__ import annotations +import hashlib import logging +import time from typing import Optional from urllib.parse import quote_plus import httpx -from bs4 import BeautifulSoup from scripts.core.base import SearchEngine, SearchResult logger = logging.getLogger(__name__) +API_URL = "https://www.cls.cn/nodeapi/telegraphList" +SEARCH_URL = "https://www.cls.cn/searchPage" +API_APP = "CailianpressWeb" +API_OS = "web" +API_SV = "7.7.5" + class CLSNewsEngine(SearchEngine): """财联社 (cls.cn) news search engine.""" - SEARCH_URL = "https://www.cls.cn/searchPage" - def __init__(self, config: Optional[dict] = None): super().__init__("cls", config) timeout = (config or {}).get("timeout", 15) self._client = httpx.AsyncClient( timeout=timeout, headers={ - "User-Agent": (config or {}).get( - "user_agent", + "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/120.0.0.0 Safari/537.36", + "Chrome/120.0.0.0 Safari/537.36" ), - "Referer": "https://www.cls.cn/", + "Referer": "https://www.cls.cn/telegraph", }, follow_redirects=True, ) def build_url(self, query: str, **kwargs) -> str: - return f"{self.SEARCH_URL}?keyword={quote_plus(query)}&type=all" + return f"{SEARCH_URL}?keyword={quote_plus(query)}&type=all" + + def _build_api_sign(self, last_time: int, rn: int) -> str: + """Build SHA1 + MD5 double-hashed sign for API request.""" + plain = ( + f"app={API_APP}&last_time={last_time}" + f"&os={API_OS}&rn={rn}&sv={API_SV}" + ) + sha1_hex = hashlib.sha1(plain.encode()).hexdigest() + return hashlib.md5(sha1_hex.encode()).hexdigest() async def search( self, query: str, max_results: int = 10, **kwargs ) -> list[SearchResult]: - """Search 财联社 and parse results.""" - url = self.build_url(query) + """Search 财联社. + + Uses the telegraph API for latest news (ignores query). + For keyword-specific results, falls back to the search page URL. + """ + # Step 1: Try to get real results from telegraph API + api_results = await self._fetch_telegraph(max_results) + if api_results: + return api_results[:max_results] + + # Step 2: Fallback to search page URL + search_url = self.build_url(query) + return [ + SearchResult( + title=f"财联社: {query}", + url=search_url, + snippet=( + f"Search 财联社 for '{query}' " + "(JS-required search page)" + ), + source=self.name, + rank=1, + category="news", + extra={"search_url": search_url, "note": "JS-rendered page"}, + ) + ] + + async def _fetch_telegraph( + self, max_results: int = 10 + ) -> list[SearchResult]: + """Fetch real-time telegraph news via API.""" try: - response = await self._client.get(url) + rn = min(max(max_results, 5), 100) + last_time = int(time.time()) + sign = self._build_api_sign(last_time, rn) + + params = { + "app": API_APP, + "last_time": str(last_time), + "os": API_OS, + "rn": str(rn), + "sv": API_SV, + "sign": sign, + } + + response = await self._client.get(API_URL, params=params) response.raise_for_status() - return self._parse_results(response.text, query, max_results) - except httpx.HTTPError as e: - logger.error(f"财联社 search failed: {e}") + + data = response.json() + items = data.get("data", {}).get("roll_data", []) + + results = [] + for i, item in enumerate(items[:max_results]): + content = item.get("content", "") + ctime = item.get("ctime", 0) + + # Extract title from content (财联社 format: + # "【title】content" or plain text) + title, snippet = self._parse_content(content) + + # Use unique URL per item to avoid dedup collision + item_url = ( + f"https://www.cls.cn/telegraph#{ctime}-{i}" + ) + + results.append( + SearchResult( + title=title or content[:80], + url=item_url, + snippet=snippet or content[:200], + source=self.name, + rank=i + 1, + category="news", + extra={ + "time": self._format_time(ctime), + "timestamp": ctime, + }, + ) + ) + + return results + + except (httpx.HTTPError, ValueError, KeyError) as e: + logger.warning(f"财联社 API failed: {e}") return [] - def _parse_results( - self, html: str, query: str, max_results: int - ) -> list[SearchResult]: - """Parse 财联社 search page.""" - soup = BeautifulSoup(html, "lxml") - results = [] + @staticmethod + def _parse_content(content: str) -> tuple[str, str]: + """Parse 财联社 content into (title, snippet). - # cls.cn typically uses dynamic rendering, so parsing may be limited - articles = soup.select( - "a[href*='/detail'], .news-item, .search-result-item, article" - ) - for i, article in enumerate(articles[:max_results]): - title_el = article.select_one("h3, h4, .title, .news-title") - link = article.get("href", "") - snippet_el = article.select_one( - ".summary, .desc, .abstract, p" + Format examples: + "【日本地震】内容详情..." → ("日本地震", "内容详情...") + "财联社5月20日电,普通新闻" → ("", "财联社5月20日电,普通新闻") + """ + if content.startswith("【"): + end = content.find("】") + if end != -1: + title = content[1:end] + snippet = content[end + 1:].strip() + return title, snippet + return "", content + + @staticmethod + def _format_time(ts: int) -> str: + """Format Unix timestamp to readable time.""" + import datetime + + try: + dt = datetime.datetime.fromtimestamp( + ts, + tz=datetime.timezone(datetime.timedelta(hours=8)), ) - - title = title_el.get_text(strip=True) if title_el else "" - snippet = snippet_el.get_text(strip=True) if snippet_el else "" - - if not title and not link: - continue - - full_url = link if link.startswith("http") else f"https://www.cls.cn{link}" - - results.append( - SearchResult( - title=title or f"财联社: {query}", - url=full_url, - snippet=snippet or f"Search result from 财联社", - source=self.name, - rank=i + 1, - category="news", - extra={"region": "cn"}, - ) - ) - - if not results: - # Fallback: return URL for manual browsing - search_url = self.build_url(query) - results.append( - SearchResult( - title=f"财联社: {query}", - url=search_url, - snippet=f"Search 财联社 for '{query}' (site requires JS)", - source=self.name, - rank=1, - category="news", - extra={"search_url": search_url, "note": "JS-rendered page"}, - ) - ) - - return results + return dt.strftime("%Y-%m-%d %H:%M") + except (OSError, ValueError): + return str(ts) async def close(self): await self._client.aclose() diff --git a/scripts/unified_search.py b/scripts/unified_search.py index e25de0f..bfbc9c1 100644 --- a/scripts/unified_search.py +++ b/scripts/unified_search.py @@ -210,7 +210,7 @@ class UnifiedSearch: engine = DuckDuckGoEngine(self.config) elif engine_type == "baidu_qianfan" or "baidu qianfan" in name.lower(): engine = BaiduQianFanEngine(self.config) - elif "cls" in name.lower(): + elif "cls" in name.lower() or "财联社" in name: engine = CLSNewsEngine(self.config) elif "wallstreet" in name.lower() or "华尔街" in name: engine = WallStreetCNEngine(self.config)