refactor: cleanup dead code, fix model confusion, streamline CLI
- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/) - Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py) - scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py) - scripts/engines/base.py: re-exports from core.base for compatibility - scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py) - scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py) - scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources) - scripts/unified_search.py: clean async Python API (was a stub) - ws.py: lightweight launcher - README.md + SKILL.md: updated to reflect actual 15+ engines - All EngineResult → SearchResult - All engines return list[SearchResult]
This commit is contained in:
1 parent
e20478fcbd
commit
c077a7ae40
31 files changed
+1188
-3148
No files matched your search
@@ -1,40 +1,62 @@
|
||||
"""News search engine — multi-source news aggregation."""
|
||||
"""News search — multi-source aggregation (Bing News + RSS)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
from urllib.parse import quote_plus
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADERS = {
|
||||
_HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
# Free RSS feeds — no API key needed
|
||||
_RSS_FEEDS = {
|
||||
# Chinese tech/business
|
||||
"36kr": "https://36kr.com/feed",
|
||||
"少数派": "https://sspai.com/feed",
|
||||
"IT之家": "https://www.ithome.com/rss/",
|
||||
"钛媒体": "https://www.tmtpost.com/rss.xml",
|
||||
"Solidot": "https://www.solidot.org/index.rss",
|
||||
"InfoQ CN": "https://www.infoq.cn/feed",
|
||||
"OSChina": "https://www.oschina.net/news/rss",
|
||||
# English tech
|
||||
"HackerNews": "https://hnrss.org/best?count=20",
|
||||
"TechCrunch": "https://techcrunch.com/feed/",
|
||||
"ArsTechnica": "https://feeds.arstechnica.com/arstechnica/index",
|
||||
"TheVerge": "https://www.theverge.com/rss/index.xml",
|
||||
"MIT Tech Review": "https://www.technologyreview.com/feed/",
|
||||
"Wired": "https://www.wired.com/feed/rss",
|
||||
"Lobste.rs": "https://lobste.rs/rss",
|
||||
# English general
|
||||
"ChinaDaily": "https://www.chinadaily.com.cn/rss/china_rss.xml",
|
||||
}
|
||||
|
||||
|
||||
class BingNewsEngine(BaseEngine):
|
||||
"""Bing News scraping — good for both CN and EN news."""
|
||||
"""Bing News scraping — works for both CN and global news."""
|
||||
|
||||
name = "bing_news"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
region: str = "cn", **kwargs) -> list[EngineResult]:
|
||||
region: str = "cn", **kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
base = "cn.bing.com" if region == "cn" else "www.bing.com"
|
||||
url = f"https://{base}/news/search?q={quote_plus(query)}&setlang=zh-CN"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp = await client.get(url, headers=_HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
@@ -44,20 +66,20 @@ class BingNewsEngine(BaseEngine):
|
||||
logger.warning(f"[bing_news] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
for item in soup.select("a.title"):
|
||||
title = item.get_text(strip=True)
|
||||
href = item.get("href", "")
|
||||
# Try to get snippet from parent container
|
||||
parent = item.find_parent("div", class_=True)
|
||||
snippet = ""
|
||||
if parent:
|
||||
snippet_el = parent.select_one("div.snippet")
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
if title and href and href.startswith("http"):
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title, url=href, snippet=snippet,
|
||||
source="bing_news", category="news",
|
||||
))
|
||||
@@ -67,40 +89,18 @@ class BingNewsEngine(BaseEngine):
|
||||
|
||||
|
||||
class RSSNewsEngine(BaseEngine):
|
||||
"""RSS-based news — multi-source aggregation."""
|
||||
"""RSS feed aggregation — multi-source news from curated feeds."""
|
||||
|
||||
name = "rss_news"
|
||||
|
||||
# Free RSS feeds, no key needed
|
||||
FEEDS = {
|
||||
# Chinese tech/business
|
||||
"36kr": "https://36kr.com/feed",
|
||||
"少数派": "https://sspai.com/feed",
|
||||
"IT之家": "https://www.ithome.com/rss/",
|
||||
"钛媒体": "https://www.tmtpost.com/rss.xml",
|
||||
"Solidot": "https://www.solidot.org/index.rss",
|
||||
"InfoQ CN": "https://www.infoq.cn/feed",
|
||||
"OSChina": "https://www.oschina.net/news/rss",
|
||||
# English tech
|
||||
"HackerNews": "https://hnrss.org/best?count=20",
|
||||
"TechCrunch": "https://techcrunch.com/feed/",
|
||||
"ArsTechnica": "https://feeds.arstechnica.com/arstechnica/index",
|
||||
"TheVerge": "https://www.theverge.com/rss/index.xml",
|
||||
"MIT Tech Review": "https://www.technologyreview.com/feed/",
|
||||
"Wired": "https://www.wired.com/feed/rss",
|
||||
"Lobste.rs": "https://lobste.rs/rss",
|
||||
# English general
|
||||
"ChinaDaily": "https://www.chinadaily.com.cn/rss/china_rss.xml",
|
||||
}
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
**kwargs) -> list[SearchResult]:
|
||||
t0 = time.time()
|
||||
all_results = []
|
||||
|
||||
import asyncio
|
||||
tasks = [self._fetch_feed(name, url, query)
|
||||
for name, url in self.FEEDS.items()]
|
||||
for name, url in _RSS_FEEDS.items()]
|
||||
feeds = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
for feed_results in feeds:
|
||||
@@ -111,52 +111,45 @@ class RSSNewsEngine(BaseEngine):
|
||||
logger.info(f"[rss_news] {len(all_results)} results in {elapsed:.2f}s")
|
||||
return all_results[:max_results]
|
||||
|
||||
async def _fetch_feed(self, name: str, url: str, query: str) -> list[EngineResult]:
|
||||
async def _fetch_feed(self, name: str, url: str, query: str) -> list[SearchResult]:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp = await client.get(url, headers=_HEADERS)
|
||||
resp.raise_for_status()
|
||||
return self._parse_feed(name, resp.text, query)
|
||||
except Exception as e:
|
||||
logger.warning(f"[rss_news:{name}] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_feed(self, name: str, xml_text: str, query: str) -> list[EngineResult]:
|
||||
def _parse_feed(self, name: str, xml_text: str, query: str) -> list[SearchResult]:
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError:
|
||||
return []
|
||||
|
||||
results = []
|
||||
# RSS 2.0: //channel/item, Atom: //entry
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||
|
||||
items = root.findall(".//item") or root.findall(".//atom:entry", ns)
|
||||
|
||||
q_lower = query.lower()
|
||||
q_words = [w for w in q_lower.split() if len(w) > 1]
|
||||
|
||||
results = []
|
||||
for item in items[:50]:
|
||||
title = (item.findtext("title") or
|
||||
item.findtext("atom:title", namespaces=ns) or "")
|
||||
link = (item.findtext("link") or "")
|
||||
link = item.findtext("link") or ""
|
||||
if not link:
|
||||
link_el = item.find("atom:link", ns)
|
||||
link = link_el.get("href", "") if link_el is not None else ""
|
||||
desc = (item.findtext("description") or
|
||||
item.findtext("atom:summary", namespaces=ns) or "")
|
||||
# Clean HTML tags from description
|
||||
import re
|
||||
desc = re.sub(r'<[^>]+>', '', desc)
|
||||
|
||||
# Keyword scoring (not strict filter)
|
||||
text_lower = f"{title} {desc}".lower()
|
||||
if q_words:
|
||||
match_count = sum(1 for w in q_words if w in text_lower)
|
||||
if match_count == 0:
|
||||
continue # Skip completely irrelevant items
|
||||
if q_words and not any(w in text_lower for w in q_words):
|
||||
continue # Skip items with no query keyword match
|
||||
|
||||
results.append(EngineResult(
|
||||
results.append(SearchResult(
|
||||
title=title.strip(),
|
||||
url=link.strip(),
|
||||
snippet=desc[:300].strip(),
|
||||
@@ -168,15 +161,15 @@ class RSSNewsEngine(BaseEngine):
|
||||
|
||||
|
||||
class NewsSearch(BaseEngine):
|
||||
"""Unified news search — Bing News + RSS in parallel."""
|
||||
"""Unified news search — Bing News + RSS feeds in parallel."""
|
||||
|
||||
name = "news"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
region: str = "cn", **kwargs) -> list[EngineResult]:
|
||||
region: str = "cn", **kwargs) -> list[SearchResult]:
|
||||
import asyncio
|
||||
bing = BingNewsEngine(timeout=self.timeout)
|
||||
rss = RSSNewsEngine(timeout=8.0) # RSS feeds may be slow
|
||||
rss = RSSNewsEngine(timeout=8.0)
|
||||
|
||||
bing_results, rss_results = await asyncio.gather(
|
||||
bing.search(query, max_results=max_results, region=region),
|
||||
|
||||
Reference in new issue
Block a user