""" 财联社 (cls.cn) news search engine. Uses the public telegraph API (nodeapi/telegraphList) to fetch real-time financial news without JS rendering. Falls back to search page URL for keyword-based search queries. API details (discovered via frontend reverse-engineering): GET https://www.cls.cn/nodeapi/telegraphList - Parameters: app=CailianpressWeb, os=web, sv=7.7.5, rn=N, last_time=timestamp - Sign: SHA1(plaintext) → MD5(hex) (anti-scraping measure) - Returns: {"data":{"roll_data":[{"ctime":..., "content":"..."}]}} """ from __future__ import annotations import hashlib import logging import time from typing import Optional from urllib.parse import quote_plus import httpx from scripts.core.base import SearchEngine, SearchResult logger = logging.getLogger(__name__) API_URL = "https://www.cls.cn/nodeapi/telegraphList" SEARCH_URL = "https://www.cls.cn/searchPage" API_APP = "CailianpressWeb" API_OS = "web" API_SV = "7.7.5" class CLSNewsEngine(SearchEngine): """财联社 (cls.cn) news search engine.""" def __init__(self, config: Optional[dict] = None): super().__init__("cls", config) timeout = (config or {}).get("timeout", 15) self._client = httpx.AsyncClient( timeout=timeout, headers={ "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/120.0.0.0 Safari/537.36" ), "Referer": "https://www.cls.cn/telegraph", }, follow_redirects=True, ) def build_url(self, query: str, **kwargs) -> str: return f"{SEARCH_URL}?keyword={quote_plus(query)}&type=all" def _build_api_sign(self, last_time: int, rn: int) -> str: """Build SHA1 + MD5 double-hashed sign for API request.""" plain = ( f"app={API_APP}&last_time={last_time}" f"&os={API_OS}&rn={rn}&sv={API_SV}" ) sha1_hex = hashlib.sha1(plain.encode()).hexdigest() return hashlib.md5(sha1_hex.encode()).hexdigest() async def search( self, query: str, max_results: int = 10, **kwargs ) -> list[SearchResult]: """Search 财联社. Uses the telegraph API for latest news (ignores query). For keyword-specific results, falls back to the search page URL. """ # Step 1: Try to get real results from telegraph API api_results = await self._fetch_telegraph(max_results) if api_results: return api_results[:max_results] # Step 2: Fallback to search page URL search_url = self.build_url(query) return [ SearchResult( title=f"财联社: {query}", url=search_url, snippet=( f"Search 财联社 for '{query}' " "(JS-required search page)" ), source=self.name, rank=1, category="news", extra={"search_url": search_url, "note": "JS-rendered page"}, ) ] async def _fetch_telegraph( self, max_results: int = 10 ) -> list[SearchResult]: """Fetch real-time telegraph news via API.""" try: rn = min(max(max_results, 5), 100) last_time = int(time.time()) sign = self._build_api_sign(last_time, rn) params = { "app": API_APP, "last_time": str(last_time), "os": API_OS, "rn": str(rn), "sv": API_SV, "sign": sign, } response = await self._client.get(API_URL, params=params) response.raise_for_status() data = response.json() items = data.get("data", {}).get("roll_data", []) results = [] for i, item in enumerate(items[:max_results]): content = item.get("content", "") ctime = item.get("ctime", 0) # Extract title from content (财联社 format: # "【title】content" or plain text) title, snippet = self._parse_content(content) # Use unique URL per item to avoid dedup collision item_url = ( f"https://www.cls.cn/telegraph#{ctime}-{i}" ) results.append( SearchResult( title=title or content[:80], url=item_url, snippet=snippet or content[:200], source=self.name, rank=i + 1, category="news", extra={ "time": self._format_time(ctime), "timestamp": ctime, }, ) ) return results except (httpx.HTTPError, ValueError, KeyError) as e: logger.warning(f"财联社 API failed: {e}") return [] @staticmethod def _parse_content(content: str) -> tuple[str, str]: """Parse 财联社 content into (title, snippet). Format examples: "【日本地震】内容详情..." → ("日本地震", "内容详情...") "财联社5月20日电,普通新闻" → ("", "财联社5月20日电,普通新闻") """ if content.startswith("【"): end = content.find("】") if end != -1: title = content[1:end] snippet = content[end + 1:].strip() return title, snippet return "", content @staticmethod def _format_time(ts: int) -> str: """Format Unix timestamp to readable time.""" import datetime try: dt = datetime.datetime.fromtimestamp( ts, tz=datetime.timezone(datetime.timedelta(hours=8)), ) return dt.strftime("%Y-%m-%d %H:%M") except (OSError, ValueError): return str(ts) async def close(self): await self._client.aclose()