fix: 财联社改用公开 API,替代 JS 渲染搜索页

- Rewrite cls.py to use nodeapi/telegraphList public API
  (SHA1+MD5 signed, no API key required)
- Returns real-time financial news with parsed titles and timestamps
- Unique per-item URLs to avoid dedup collision
- Update engine name from 'cls' to '财联社' for CLI usability
- Update SKILL.md known limitations
This commit is contained in:
vex1023 committed 2026-05-20 12:38:10 +08:00
1 parent 9578080ec4
commit 4a42a00156
4 files changed
+143 -71

No files matched your search

+139 -68
View File
@@ -1,115 +1,186 @@
"""
财联社 (cls.cn) news search engine.
Scrapes cls.cn search results for financial news.
"""
Uses the public telegraph API (nodeapi/telegraphList) to fetch real-time
financial news without JS rendering. Falls back to search page URL for
keyword-based search queries.
API details (discovered via frontend reverse-engineering):
GET https://www.cls.cn/nodeapi/telegraphList
- Parameters: app=CailianpressWeb, os=web, sv=7.7.5, rn=N, last_time=timestamp
- Sign: SHA1(plaintext) → MD5(hex) (anti-scraping measure)
- Returns: {"data":{"roll_data":[{"ctime":..., "content":"..."}]}}
"""
from __future__ import annotations
import hashlib
import logging
import time
from typing import Optional
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
API_URL = "https://www.cls.cn/nodeapi/telegraphList"
SEARCH_URL = "https://www.cls.cn/searchPage"
API_APP = "CailianpressWeb"
API_OS = "web"
API_SV = "7.7.5"
class CLSNewsEngine(SearchEngine):
"""财联社 (cls.cn) news search engine."""
SEARCH_URL = "https://www.cls.cn/searchPage"
def __init__(self, config: Optional[dict] = None):
super().__init__("cls", config)
timeout = (config or {}).get("timeout", 15)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"User-Agent": (config or {}).get(
"user_agent",
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36",
"Chrome/120.0.0.0 Safari/537.36"
),
"Referer": "https://www.cls.cn/",
"Referer": "https://www.cls.cn/telegraph",
},
follow_redirects=True,
)
def build_url(self, query: str, **kwargs) -> str:
return f"{self.SEARCH_URL}?keyword={quote_plus(query)}&type=all"
return f"{SEARCH_URL}?keyword={quote_plus(query)}&type=all"
def _build_api_sign(self, last_time: int, rn: int) -> str:
"""Build SHA1 + MD5 double-hashed sign for API request."""
plain = (
f"app={API_APP}&last_time={last_time}"
f"&os={API_OS}&rn={rn}&sv={API_SV}"
)
sha1_hex = hashlib.sha1(plain.encode()).hexdigest()
return hashlib.md5(sha1_hex.encode()).hexdigest()
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search 财联社 and parse results."""
url = self.build_url(query)
"""Search 财联社.
Uses the telegraph API for latest news (ignores query).
For keyword-specific results, falls back to the search page URL.
"""
# Step 1: Try to get real results from telegraph API
api_results = await self._fetch_telegraph(max_results)
if api_results:
return api_results[:max_results]
# Step 2: Fallback to search page URL
search_url = self.build_url(query)
return [
SearchResult(
title=f"财联社: {query}",
url=search_url,
snippet=(
f"Search 财联社 for '{query}' "
"(JS-required search page)"
),
source=self.name,
rank=1,
category="news",
extra={"search_url": search_url, "note": "JS-rendered page"},
)
]
async def _fetch_telegraph(
self, max_results: int = 10
) -> list[SearchResult]:
"""Fetch real-time telegraph news via API."""
try:
response = await self._client.get(url)
rn = min(max(max_results, 5), 100)
last_time = int(time.time())
sign = self._build_api_sign(last_time, rn)
params = {
"app": API_APP,
"last_time": str(last_time),
"os": API_OS,
"rn": str(rn),
"sv": API_SV,
"sign": sign,
}
response = await self._client.get(API_URL, params=params)
response.raise_for_status()
return self._parse_results(response.text, query, max_results)
except httpx.HTTPError as e:
logger.error(f"财联社 search failed: {e}")
data = response.json()
items = data.get("data", {}).get("roll_data", [])
results = []
for i, item in enumerate(items[:max_results]):
content = item.get("content", "")
ctime = item.get("ctime", 0)
# Extract title from content (财联社 format:
# "【title】content" or plain text)
title, snippet = self._parse_content(content)
# Use unique URL per item to avoid dedup collision
item_url = (
f"https://www.cls.cn/telegraph#{ctime}-{i}"
)
results.append(
SearchResult(
title=title or content[:80],
url=item_url,
snippet=snippet or content[:200],
source=self.name,
rank=i + 1,
category="news",
extra={
"time": self._format_time(ctime),
"timestamp": ctime,
},
)
)
return results
except (httpx.HTTPError, ValueError, KeyError) as e:
logger.warning(f"财联社 API failed: {e}")
return []
def _parse_results(
self, html: str, query: str, max_results: int
) -> list[SearchResult]:
"""Parse 财联社 search page."""
soup = BeautifulSoup(html, "lxml")
results = []
@staticmethod
def _parse_content(content: str) -> tuple[str, str]:
"""Parse 财联社 content into (title, snippet).
# cls.cn typically uses dynamic rendering, so parsing may be limited
articles = soup.select(
"a[href*='/detail'], .news-item, .search-result-item, article"
)
for i, article in enumerate(articles[:max_results]):
title_el = article.select_one("h3, h4, .title, .news-title")
link = article.get("href", "")
snippet_el = article.select_one(
".summary, .desc, .abstract, p"
Format examples:
"【日本地震】内容详情..." → ("日本地震", "内容详情...")
"财联社5月20日电,普通新闻" → ("", "财联社5月20日电,普通新闻")
"""
if content.startswith("【"):
end = content.find("】")
if end != -1:
title = content[1:end]
snippet = content[end + 1:].strip()
return title, snippet
return "", content
@staticmethod
def _format_time(ts: int) -> str:
"""Format Unix timestamp to readable time."""
import datetime
try:
dt = datetime.datetime.fromtimestamp(
ts,
tz=datetime.timezone(datetime.timedelta(hours=8)),
)
title = title_el.get_text(strip=True) if title_el else ""
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if not title and not link:
continue
full_url = link if link.startswith("http") else f"https://www.cls.cn{link}"
results.append(
SearchResult(
title=title or f"财联社: {query}",
url=full_url,
snippet=snippet or f"Search result from 财联社",
source=self.name,
rank=i + 1,
category="news",
extra={"region": "cn"},
)
)
if not results:
# Fallback: return URL for manual browsing
search_url = self.build_url(query)
results.append(
SearchResult(
title=f"财联社: {query}",
url=search_url,
snippet=f"Search 财联社 for '{query}' (site requires JS)",
source=self.name,
rank=1,
category="news",
extra={"search_url": search_url, "note": "JS-rendered page"},
)
)
return results
return dt.strftime("%Y-%m-%d %H:%M")
except (OSError, ValueError):
return str(ts)
async def close(self):
await self._client.aclose()