"""News search engine — multi-source news aggregation.""" from __future__ import annotations import time import logging from urllib.parse import quote_plus from xml.etree import ElementTree as ET import httpx from bs4 import BeautifulSoup from scripts.engines.base import BaseEngine, EngineResult logger = logging.getLogger(__name__) HEADERS = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", } class BingNewsEngine(BaseEngine): """Bing News scraping — good for both CN and EN news.""" name = "bing_news" async def search(self, query: str, max_results: int = 10, region: str = "cn", **kwargs) -> list[EngineResult]: t0 = time.time() base = "cn.bing.com" if region == "cn" else "www.bing.com" url = f"https://{base}/news/search?q={quote_plus(query)}&setlang=zh-CN" try: async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client: resp = await client.get(url, headers=HEADERS) resp.raise_for_status() elapsed = time.time() - t0 results = self._parse(resp.text, max_results) logger.info(f"[bing_news] {len(results)} results in {elapsed:.2f}s") return results except Exception as e: logger.warning(f"[bing_news] failed: {e}") return [] def _parse(self, html: str, max_results: int) -> list[EngineResult]: soup = BeautifulSoup(html, "html.parser") results = [] for item in soup.select("a.title"): title = item.get_text(strip=True) href = item.get("href", "") # Try to get snippet from parent container parent = item.find_parent("div", class_=True) snippet = "" if parent: snippet_el = parent.select_one("div.snippet") snippet = snippet_el.get_text(strip=True) if snippet_el else "" if title and href and href.startswith("http"): results.append(EngineResult( title=title, url=href, snippet=snippet, source="bing_news", category="news", )) if len(results) >= max_results: break return results class RSSNewsEngine(BaseEngine): """RSS-based news — multi-source aggregation.""" name = "rss_news" # Free RSS feeds, no key needed FEEDS = { # Chinese tech/business "36kr": "https://36kr.com/feed", "少数派": "https://sspai.com/feed", "IT之家": "https://www.ithome.com/rss/", "钛媒体": "https://www.tmtpost.com/rss.xml", "Solidot": "https://www.solidot.org/index.rss", "InfoQ CN": "https://www.infoq.cn/feed", "OSChina": "https://www.oschina.net/news/rss", # English tech "HackerNews": "https://hnrss.org/best?count=20", "TechCrunch": "https://techcrunch.com/feed/", "ArsTechnica": "https://feeds.arstechnica.com/arstechnica/index", "TheVerge": "https://www.theverge.com/rss/index.xml", "MIT Tech Review": "https://www.technologyreview.com/feed/", "Wired": "https://www.wired.com/feed/rss", "Lobste.rs": "https://lobste.rs/rss", # English general "ChinaDaily": "https://www.chinadaily.com.cn/rss/china_rss.xml", } async def search(self, query: str, max_results: int = 10, **kwargs) -> list[EngineResult]: t0 = time.time() all_results = [] import asyncio tasks = [self._fetch_feed(name, url, query) for name, url in self.FEEDS.items()] feeds = await asyncio.gather(*tasks, return_exceptions=True) for feed_results in feeds: if isinstance(feed_results, list): all_results.extend(feed_results) elapsed = time.time() - t0 logger.info(f"[rss_news] {len(all_results)} results in {elapsed:.2f}s") return all_results[:max_results] async def _fetch_feed(self, name: str, url: str, query: str) -> list[EngineResult]: try: async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client: resp = await client.get(url, headers=HEADERS) resp.raise_for_status() return self._parse_feed(name, resp.text, query) except Exception as e: logger.warning(f"[rss_news:{name}] failed: {e}") return [] def _parse_feed(self, name: str, xml_text: str, query: str) -> list[EngineResult]: try: root = ET.fromstring(xml_text) except ET.ParseError: return [] results = [] # RSS 2.0: //channel/item, Atom: //entry ns = {"atom": "http://www.w3.org/2005/Atom"} items = root.findall(".//item") or root.findall(".//atom:entry", ns) q_lower = query.lower() q_words = [w for w in q_lower.split() if len(w) > 1] for item in items[:50]: title = (item.findtext("title") or item.findtext("atom:title", namespaces=ns) or "") link = (item.findtext("link") or "") if not link: link_el = item.find("atom:link", ns) link = link_el.get("href", "") if link_el is not None else "" desc = (item.findtext("description") or item.findtext("atom:summary", namespaces=ns) or "") # Clean HTML tags from description import re desc = re.sub(r'<[^>]+>', '', desc) # Keyword scoring (not strict filter) text_lower = f"{title} {desc}".lower() if q_words: match_count = sum(1 for w in q_words if w in text_lower) if match_count == 0: continue # Skip completely irrelevant items results.append(EngineResult( title=title.strip(), url=link.strip(), snippet=desc[:300].strip(), source=f"rss_{name}", category="news", )) return results class NewsSearch(BaseEngine): """Unified news search — Bing News + RSS in parallel.""" name = "news" async def search(self, query: str, max_results: int = 10, region: str = "cn", **kwargs) -> list[EngineResult]: import asyncio bing = BingNewsEngine(timeout=self.timeout) rss = RSSNewsEngine(timeout=8.0) # RSS feeds may be slow bing_results, rss_results = await asyncio.gather( bing.search(query, max_results=max_results, region=region), rss.search(query, max_results=max_results), return_exceptions=True, ) all_results = [] if isinstance(bing_results, list): all_results.extend(bing_results) if isinstance(rss_results, list): all_results.extend(rss_results) # Deduplicate by URL seen = set() deduped = [] for r in all_results: if r.url not in seen: seen.add(r.url) deduped.append(r) return deduped[:max_results]