Files
web-search-skills/scripts/news/rss.py
T
vex1023 87935b59ce refactor: restructure project with CLI tool, multi-engine support, and skill-builder compliant SKILL.md
- Replace setup.py with pyproject.toml (uv-compatible)
- Add CLI entry point (web-search) with 9 subcommands
- Add multi-engine architecture (14 web + 7 special sources)
- Add WeChat/CLS/WallStreetCN/ArXiv/Twitter search engines
- Remove old src/ and examples/ directories
- Remove redundant requirements.txt and setup.py
- Update README.md and SKILL.md with full documentation
- SKILL.md now compliant with skill-builder standards (frontmatter, triggers, structure)
2026-05-14 16:31:54 +08:00

289 lines
10 KiB
Python

"""
RSS news aggregation engine.
Fetches and parses RSS/Atom feeds from trusted news sources.
Inspired by the news-summary skill's RSS-based approach.
Supported sources:
- BBC (World, Business, Technology, Top Stories)
- Reuters (World News)
- NPR (US news)
- Al Jazeera (Global South perspective)
- 财联社 (RSS if available)
- 华尔街见闻 (RSS if available)
"""
from __future__ import annotations
import logging
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from typing import Any, Optional
from urllib.parse import urlparse
import httpx
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
# RSS feed definitions (from news-summary skill + additional sources)
RSS_FEEDS = {
"BBC World": {
"url": "https://feeds.bbci.co.uk/news/world/rss.xml",
"category": "news",
"lang": "en",
},
"BBC Top Stories": {
"url": "https://feeds.bbci.co.uk/news/rss.xml",
"category": "news",
"lang": "en",
},
"BBC Business": {
"url": "https://feeds.bbci.co.uk/news/business/rss.xml",
"category": "business",
"lang": "en",
},
"BBC Technology": {
"url": "https://feeds.bbci.co.uk/news/technology/rss.xml",
"category": "technology",
"lang": "en",
},
"Reuters": {
"url": "https://www.reutersagency.com/feed/?best-regions=world&post_type=best",
"category": "news",
"lang": "en",
},
"NPR": {
"url": "https://feeds.npr.org/1001/rss.xml",
"category": "news",
"lang": "en",
},
"Al Jazeera": {
"url": "https://www.aljazeera.com/xml/rss/all.xml",
"category": "news",
"lang": "en",
},
}
class RSSNewsEngine(SearchEngine):
"""
RSS-based news aggregation engine.
Fetches and parses RSS/Atom feeds from international news sources.
Supports filtering by category, keyword, and time range.
"""
def __init__(self, config: Optional[dict] = None):
super().__init__("RSS News", config)
timeout = (config or {}).get("timeout", 30)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"User-Agent": (config or {}).get(
"user_agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36",
),
"Accept": "application/rss+xml, application/xml, text/xml, */*",
},
follow_redirects=True,
)
def build_url(self, query: str, **kwargs) -> str:
"""Build search URL - for RSS this is not a search URL but a feed URL."""
feed_name = kwargs.get("feed", "BBC World")
feed = RSS_FEEDS.get(feed_name, RSS_FEEDS["BBC World"])
return feed["url"]
async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]:
"""
Search news across RSS feeds. If query is empty, returns latest headlines.
Args:
query: Search keyword to filter articles (case-insensitive).
If empty/blank, returns all headlines.
max_results: Maximum articles per feed.
feeds: Specific feeds to query (list of feed names or "all").
category: Filter by category (news, business, technology).
hours: Only include articles from the last N hours.
Returns:
List of SearchResult objects sorted by date (newest first).
"""
feeds_to_query = self._select_feeds(
feed_names=kwargs.get("feeds", "all"),
category=kwargs.get("category"),
)
results = []
for feed_name, feed_info in feeds_to_query.items():
try:
response = await self._client.get(feed_info["url"])
response.raise_for_status()
parsed = self._parse_feed(response.text, feed_name, feed_info)
for article in parsed:
# Filter by keyword if query provided
if query:
q = query.lower()
if q in article.title.lower() or q in article.snippet.lower():
results.append(article)
else:
results.append(article)
except httpx.HTTPError as e:
logger.warning(f"RSS feed '{feed_name}' failed: {e}")
continue
# Apply time filter
hours = kwargs.get("hours")
if hours:
from datetime import timedelta
cutoff = datetime.now(timezone.utc) - timedelta(hours=int(hours))
results = [r for r in results if r.extra.get("parsed_date", datetime.max.replace(tzinfo=timezone.utc)) >= cutoff]
# Sort by date (newest first), limit results
results.sort(key=lambda r: r.extra.get("parsed_date", datetime.min.replace(tzinfo=timezone.utc)), reverse=True)
return results[:max_results * max(1, len(feeds_to_query))]
def _select_feeds(self, feed_names="all", category=None):
"""Select feeds based on filters."""
if feed_names != "all":
if isinstance(feed_names, str):
feed_names = [feed_names]
return {k: v for k, v in RSS_FEEDS.items() if k in feed_names}
if category:
return {k: v for k, v in RSS_FEEDS.items() if v.get("category") == category}
return RSS_FEEDS
def _parse_feed(self, xml_text: str, feed_name: str, feed_info: dict) -> list[SearchResult]:
"""Parse RSS/Atom XML into SearchResult objects."""
results = []
try:
root = ET.fromstring(xml_text)
except ET.ParseError as e:
logger.error(f"Failed to parse RSS from {feed_name}: {e}")
return []
# Detect RSS vs Atom format
if root.tag == "rss":
return self._parse_rss(root, feed_name, feed_info)
elif root.tag.endswith("feed"):
return self._parse_atom(root, feed_name, feed_info)
else:
logger.warning(f"Unknown feed format for {feed_name}: {root.tag}")
return results
def _parse_rss(self, root: ET.Element, feed_name: str, feed_info: dict) -> list[SearchResult]:
"""Parse RSS 2.0 format."""
results = []
namespaces = {"content": "http://purl.org/rss/1.0/modules/content/"}
for i, item in enumerate(root.findall(".//item")):
title = item.findtext("title", "")
link = item.findtext("link", "")
description = item.findtext("description", "")
pub_date_str = item.findtext("pubDate", "")
creator = item.findtext("dc:creator", "") or item.findtext("author", "")
# Clean up description (remove HTML tags)
import re
snippet = re.sub(r"<[^>]+>", "", description).strip()
# Truncate
snippet = snippet[:300] + "..." if len(snippet) > 300 else snippet
# Parse date
parsed_date = self._parse_date(pub_date_str)
results.append(
SearchResult(
title=title,
url=link,
snippet=snippet,
source=feed_name,
rank=i + 1,
category=feed_info.get("category", "news"),
extra={
"feed_name": feed_name,
"author": creator,
"published": pub_date_str,
"parsed_date": parsed_date,
"feed_url": feed_info["url"],
},
)
)
return results
def _parse_atom(self, root: ET.Element, feed_name: str, feed_info: dict) -> list[SearchResult]:
"""Parse Atom format."""
results = []
ns = {"a": "http://www.w3.org/2005/Atom"}
feed_title = root.findtext("a:title", "", ns)
for i, entry in enumerate(root.findall("a:entry", ns)):
title = entry.findtext("a:title", "", ns)
link_el = entry.find("a:link", ns)
link = link_el.get("href", "") if link_el is not None else ""
summary = entry.findtext("a:summary", "", ns) or entry.findtext("a:content", "", ns)
published = entry.findtext("a:published", "", ns) or entry.findtext("a:updated", "", ns)
author = entry.findtext("a:author/a:name", "", ns)
import re
snippet = re.sub(r"<[^>]+>", "", summary).strip()
snippet = snippet[:300] + "..." if len(snippet) > 300 else snippet
parsed_date = self._parse_date(published)
results.append(
SearchResult(
title=title,
url=link,
snippet=snippet,
source=feed_name,
rank=i + 1,
category=feed_info.get("category", "news"),
extra={
"feed_name": feed_name,
"author": author,
"published": published,
"parsed_date": parsed_date,
"feed_url": feed_info["url"],
},
)
)
return results
def _parse_date(self, date_str: str) -> datetime:
"""Parse various date formats to datetime."""
if not date_str:
return datetime.min.replace(tzinfo=timezone.utc)
formats = [
"%a, %d %b %Y %H:%M:%S %z", # RSS: Sat, 01 Jan 2024 12:00:00 +0000
"%a, %d %b %Y %H:%M:%S %Z", # RSS with timezone name
"%Y-%m-%dT%H:%M:%S%z", # Atom ISO 8601
"%Y-%m-%dT%H:%M:%S.%f%z", # Atom with microseconds
"%Y-%m-%dT%H:%M:%SZ", # Atom UTC
"%Y-%m-%d", # Simple date
]
for fmt in formats:
try:
return datetime.strptime(date_str.strip(), fmt).replace(tzinfo=timezone.utc)
except (ValueError, TypeError):
continue
return datetime.min.replace(tzinfo=timezone.utc)
async def close(self):
await self._client.aclose()