- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/) - Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py) - scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py) - scripts/engines/base.py: re-exports from core.base for compatibility - scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py) - scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py) - scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources) - scripts/unified_search.py: clean async Python API (was a stub) - ws.py: lightweight launcher - README.md + SKILL.md: updated to reflect actual 15+ engines - All EngineResult → SearchResult - All engines return list[SearchResult]
115 lines
4.0 KiB
Python
115 lines
4.0 KiB
Python
"""Parsing-based search engines — DuckDuckGo, Brave (fallback HTML scraper)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from urllib.parse import quote_plus
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
from scripts.core.base import BaseEngine, SearchResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_HEADERS = {
|
|
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/125.0.0.0 Safari/537.36"),
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
}
|
|
|
|
|
|
class DuckDuckGoParser(BaseEngine):
|
|
"""DuckDuckGo HTML scraper (no API key needed)."""
|
|
|
|
name = "duckduckgo"
|
|
|
|
async def search(self, query: str, max_results: int = 10,
|
|
**kwargs) -> list[SearchResult]:
|
|
import time
|
|
t0 = time.time()
|
|
url = f"https://html.duckduckgo.com/html/?q={quote_plus(query)}"
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
|
resp = await client.get(url, headers=_HEADERS)
|
|
resp.raise_for_status()
|
|
elapsed = time.time() - t0
|
|
results = self._parse(resp.text, max_results)
|
|
logger.info(f"[duckduckgo] {len(results)} results in {elapsed:.2f}s")
|
|
return results
|
|
except Exception as e:
|
|
logger.warning(f"[duckduckgo] failed: {e}")
|
|
return []
|
|
|
|
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
results = []
|
|
|
|
for result in soup.select(".result")[:max_results]:
|
|
title_el = result.select_one(".result__title a")
|
|
snippet_el = result.select_one(".result__snippet")
|
|
if not title_el:
|
|
continue
|
|
results.append(SearchResult(
|
|
title=title_el.get_text(strip=True),
|
|
url=title_el.get("href", ""),
|
|
snippet=snippet_el.get_text(strip=True) if snippet_el else "",
|
|
source="duckduckgo",
|
|
category="web",
|
|
))
|
|
|
|
return results
|
|
|
|
|
|
class BraveParser(BaseEngine):
|
|
"""Brave Search fallback HTML scraper (no API key needed)."""
|
|
|
|
name = "brave"
|
|
|
|
async def search(self, query: str, max_results: int = 10,
|
|
**kwargs) -> list[SearchResult]:
|
|
import time
|
|
t0 = time.time()
|
|
url = f"https://search.brave.com/search?q={quote_plus(query)}&source=web"
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
|
resp = await client.get(url, headers=_HEADERS)
|
|
resp.raise_for_status()
|
|
elapsed = time.time() - t0
|
|
results = self._parse(resp.text, max_results)
|
|
logger.info(f"[brave] {len(results)} results in {elapsed:.2f}s")
|
|
return results
|
|
except Exception as e:
|
|
logger.warning(f"[brave] failed: {e}")
|
|
return []
|
|
|
|
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
results = []
|
|
|
|
for item in soup.select("div.snippet, div.result")[:max_results]:
|
|
title_el = item.select_one("a.result-header, a.snippet-title, a")
|
|
if not title_el:
|
|
continue
|
|
title = title_el.get_text(strip=True)
|
|
href = title_el.get("href", "")
|
|
snippet_el = item.select_one("p.snippet-description, div.snippet-description")
|
|
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
|
|
|
if title and href and href.startswith("http"):
|
|
results.append(SearchResult(
|
|
title=title, url=href, snippet=snippet,
|
|
source="brave", category="web",
|
|
))
|
|
|
|
return results
|
|
|
|
|
|
# Backwards-compatibility aliases
|
|
DuckDuckGoEngine = DuckDuckGoParser
|
|
BraveScraper = BraveParser
|