refactor: cleanup dead code, fix model confusion, streamline CLI
- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/) - Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py) - scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py) - scripts/engines/base.py: re-exports from core.base for compatibility - scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py) - scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py) - scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources) - scripts/unified_search.py: clean async Python API (was a stub) - ws.py: lightweight launcher - README.md + SKILL.md: updated to reflect actual 15+ engines - All EngineResult → SearchResult - All engines return list[SearchResult]
This commit is contained in:
1 parent
e20478fcbd
commit
c077a7ae40
31 files changed
+1188
-3148
No files matched your search
@@ -1,80 +1,114 @@
|
||||
"""
|
||||
Parsing-based search engines (e.g. DuckDuckGo HTML scraper).
|
||||
"""
|
||||
"""Parsing-based search engines — DuckDuckGo, Brave (fallback HTML scraper)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
from scripts.core.base import BaseEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_HEADERS = {
|
||||
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/125.0.0.0 Safari/537.36"),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
class DuckDuckGoEngine(SearchEngine):
|
||||
"""DuckDuckGo HTML search engine (no API key required)."""
|
||||
|
||||
SEARCH_URL = "https://html.duckduckgo.com/html/"
|
||||
class DuckDuckGoParser(BaseEngine):
|
||||
"""DuckDuckGo HTML scraper (no API key needed)."""
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("DuckDuckGo", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
name = "duckduckgo"
|
||||
|
||||
def build_url(self, query: str, **kwargs) -> str:
|
||||
"""Build DuckDuckGo search URL."""
|
||||
from urllib.parse import quote_plus
|
||||
return f"https://duckduckgo.com/?q={quote_plus(query)}"
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[SearchResult]:
|
||||
import time
|
||||
t0 = time.time()
|
||||
url = f"https://html.duckduckgo.com/html/?q={quote_plus(query)}"
|
||||
|
||||
async def search(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[SearchResult]:
|
||||
"""Search DuckDuckGo and parse HTML results."""
|
||||
params = {"q": query}
|
||||
try:
|
||||
response = await self._client.post(self.SEARCH_URL, data=params)
|
||||
response.raise_for_status()
|
||||
return self._parse_results(response.text, max_results)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"DuckDuckGo search failed: {e}")
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=_HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
logger.info(f"[duckduckgo] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[duckduckgo] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_results(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
"""Parse DuckDuckGo HTML search results."""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
for i, result in enumerate(soup.select(".result")[:max_results]):
|
||||
for result in soup.select(".result")[:max_results]:
|
||||
title_el = result.select_one(".result__title a")
|
||||
snippet_el = result.select_one(".result__snippet")
|
||||
if title_el:
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title_el.get_text(strip=True),
|
||||
url=title_el.get("href", ""),
|
||||
snippet=snippet_el.get_text(strip=True)
|
||||
if snippet_el else "",
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="web",
|
||||
)
|
||||
)
|
||||
if not title_el:
|
||||
continue
|
||||
results.append(SearchResult(
|
||||
title=title_el.get_text(strip=True),
|
||||
url=title_el.get("href", ""),
|
||||
snippet=snippet_el.get_text(strip=True) if snippet_el else "",
|
||||
source="duckduckgo",
|
||||
category="web",
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
|
||||
class BraveParser(BaseEngine):
|
||||
"""Brave Search fallback HTML scraper (no API key needed)."""
|
||||
|
||||
name = "brave"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[SearchResult]:
|
||||
import time
|
||||
t0 = time.time()
|
||||
url = f"https://search.brave.com/search?q={quote_plus(query)}&source=web"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=_HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
logger.info(f"[brave] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[brave] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
for item in soup.select("div.snippet, div.result")[:max_results]:
|
||||
title_el = item.select_one("a.result-header, a.snippet-title, a")
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
href = title_el.get("href", "")
|
||||
snippet_el = item.select_one("p.snippet-description, div.snippet-description")
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href and href.startswith("http"):
|
||||
results.append(SearchResult(
|
||||
title=title, url=href, snippet=snippet,
|
||||
source="brave", category="web",
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# Backwards-compatibility aliases
|
||||
DuckDuckGoEngine = DuckDuckGoParser
|
||||
BraveScraper = BraveParser
|
||||
Reference in new issue
Block a user