- Rewrite cls.py to use nodeapi/telegraphList public API (SHA1+MD5 signed, no API key required) - Returns real-time financial news with parsed titles and timestamps - Unique per-item URLs to avoid dedup collision - Update engine name from 'cls' to '财联社' for CLI usability - Update SKILL.md known limitations
187 lines
6.1 KiB
Python
187 lines
6.1 KiB
Python
"""
|
|
财联社 (cls.cn) news search engine.
|
|
|
|
Uses the public telegraph API (nodeapi/telegraphList) to fetch real-time
|
|
financial news without JS rendering. Falls back to search page URL for
|
|
keyword-based search queries.
|
|
|
|
API details (discovered via frontend reverse-engineering):
|
|
GET https://www.cls.cn/nodeapi/telegraphList
|
|
- Parameters: app=CailianpressWeb, os=web, sv=7.7.5, rn=N, last_time=timestamp
|
|
- Sign: SHA1(plaintext) → MD5(hex) (anti-scraping measure)
|
|
- Returns: {"data":{"roll_data":[{"ctime":..., "content":"..."}]}}
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
import time
|
|
from typing import Optional
|
|
from urllib.parse import quote_plus
|
|
|
|
import httpx
|
|
|
|
from scripts.core.base import SearchEngine, SearchResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
API_URL = "https://www.cls.cn/nodeapi/telegraphList"
|
|
SEARCH_URL = "https://www.cls.cn/searchPage"
|
|
API_APP = "CailianpressWeb"
|
|
API_OS = "web"
|
|
API_SV = "7.7.5"
|
|
|
|
|
|
class CLSNewsEngine(SearchEngine):
|
|
"""财联社 (cls.cn) news search engine."""
|
|
|
|
def __init__(self, config: Optional[dict] = None):
|
|
super().__init__("cls", config)
|
|
timeout = (config or {}).get("timeout", 15)
|
|
self._client = httpx.AsyncClient(
|
|
timeout=timeout,
|
|
headers={
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/120.0.0.0 Safari/537.36"
|
|
),
|
|
"Referer": "https://www.cls.cn/telegraph",
|
|
},
|
|
follow_redirects=True,
|
|
)
|
|
|
|
def build_url(self, query: str, **kwargs) -> str:
|
|
return f"{SEARCH_URL}?keyword={quote_plus(query)}&type=all"
|
|
|
|
def _build_api_sign(self, last_time: int, rn: int) -> str:
|
|
"""Build SHA1 + MD5 double-hashed sign for API request."""
|
|
plain = (
|
|
f"app={API_APP}&last_time={last_time}"
|
|
f"&os={API_OS}&rn={rn}&sv={API_SV}"
|
|
)
|
|
sha1_hex = hashlib.sha1(plain.encode()).hexdigest()
|
|
return hashlib.md5(sha1_hex.encode()).hexdigest()
|
|
|
|
async def search(
|
|
self, query: str, max_results: int = 10, **kwargs
|
|
) -> list[SearchResult]:
|
|
"""Search 财联社.
|
|
|
|
Uses the telegraph API for latest news (ignores query).
|
|
For keyword-specific results, falls back to the search page URL.
|
|
"""
|
|
# Step 1: Try to get real results from telegraph API
|
|
api_results = await self._fetch_telegraph(max_results)
|
|
if api_results:
|
|
return api_results[:max_results]
|
|
|
|
# Step 2: Fallback to search page URL
|
|
search_url = self.build_url(query)
|
|
return [
|
|
SearchResult(
|
|
title=f"财联社: {query}",
|
|
url=search_url,
|
|
snippet=(
|
|
f"Search 财联社 for '{query}' "
|
|
"(JS-required search page)"
|
|
),
|
|
source=self.name,
|
|
rank=1,
|
|
category="news",
|
|
extra={"search_url": search_url, "note": "JS-rendered page"},
|
|
)
|
|
]
|
|
|
|
async def _fetch_telegraph(
|
|
self, max_results: int = 10
|
|
) -> list[SearchResult]:
|
|
"""Fetch real-time telegraph news via API."""
|
|
try:
|
|
rn = min(max(max_results, 5), 100)
|
|
last_time = int(time.time())
|
|
sign = self._build_api_sign(last_time, rn)
|
|
|
|
params = {
|
|
"app": API_APP,
|
|
"last_time": str(last_time),
|
|
"os": API_OS,
|
|
"rn": str(rn),
|
|
"sv": API_SV,
|
|
"sign": sign,
|
|
}
|
|
|
|
response = await self._client.get(API_URL, params=params)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
items = data.get("data", {}).get("roll_data", [])
|
|
|
|
results = []
|
|
for i, item in enumerate(items[:max_results]):
|
|
content = item.get("content", "")
|
|
ctime = item.get("ctime", 0)
|
|
|
|
# Extract title from content (财联社 format:
|
|
# "【title】content" or plain text)
|
|
title, snippet = self._parse_content(content)
|
|
|
|
# Use unique URL per item to avoid dedup collision
|
|
item_url = (
|
|
f"https://www.cls.cn/telegraph#{ctime}-{i}"
|
|
)
|
|
|
|
results.append(
|
|
SearchResult(
|
|
title=title or content[:80],
|
|
url=item_url,
|
|
snippet=snippet or content[:200],
|
|
source=self.name,
|
|
rank=i + 1,
|
|
category="news",
|
|
extra={
|
|
"time": self._format_time(ctime),
|
|
"timestamp": ctime,
|
|
},
|
|
)
|
|
)
|
|
|
|
return results
|
|
|
|
except (httpx.HTTPError, ValueError, KeyError) as e:
|
|
logger.warning(f"财联社 API failed: {e}")
|
|
return []
|
|
|
|
@staticmethod
|
|
def _parse_content(content: str) -> tuple[str, str]:
|
|
"""Parse 财联社 content into (title, snippet).
|
|
|
|
Format examples:
|
|
"【日本地震】内容详情..." → ("日本地震", "内容详情...")
|
|
"财联社5月20日电,普通新闻" → ("", "财联社5月20日电,普通新闻")
|
|
"""
|
|
if content.startswith("【"):
|
|
end = content.find("】")
|
|
if end != -1:
|
|
title = content[1:end]
|
|
snippet = content[end + 1:].strip()
|
|
return title, snippet
|
|
return "", content
|
|
|
|
@staticmethod
|
|
def _format_time(ts: int) -> str:
|
|
"""Format Unix timestamp to readable time."""
|
|
import datetime
|
|
|
|
try:
|
|
dt = datetime.datetime.fromtimestamp(
|
|
ts,
|
|
tz=datetime.timezone(datetime.timedelta(hours=8)),
|
|
)
|
|
return dt.strftime("%Y-%m-%d %H:%M")
|
|
except (OSError, ValueError):
|
|
return str(ts)
|
|
|
|
async def close(self):
|
|
await self._client.aclose()
|