refactor: cleanup dead code, fix model confusion, streamline CLI

- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/)
- Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py)
- scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py)
- scripts/engines/base.py: re-exports from core.base for compatibility
- scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py)
- scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py)
- scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources)
- scripts/unified_search.py: clean async Python API (was a stub)
- ws.py: lightweight launcher
- README.md + SKILL.md: updated to reflect actual 15+ engines
- All EngineResult → SearchResult
- All engines return list[SearchResult]
This commit is contained in:
openbot-coder committed 2026-05-30 16:25:41 +08:00
1 parent e20478fcbd
commit c077a7ae40
31 files changed
+1188 -3148

No files matched your search

+19 -23
View File
@@ -1,4 +1,4 @@
"""Academic search engine — ArXiv + CrossRef APIs (free, no key)."""
"""Academic search — ArXiv + CrossRef APIs (free, no API key)."""
from __future__ import annotations
@@ -9,18 +9,18 @@ from xml.etree import ElementTree as ET
import httpx
from scripts.engines.base import BaseEngine, EngineResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
class ArxivEngine(BaseEngine):
"""ArXiv paper search — free, no key needed."""
"""ArXiv paper search via public API."""
name = "arxiv"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
**kwargs) -> list[SearchResult]:
t0 = time.time()
url = (
f"http://export.arxiv.org/api/query?"
@@ -40,8 +40,12 @@ class ArxivEngine(BaseEngine):
logger.warning(f"[arxiv] failed: {e}")
return []
def _parse_xml(self, xml_text: str, max_results: int) -> list[EngineResult]:
root = ET.fromstring(xml_text)
def _parse_xml(self, xml_text: str, max_results: int) -> list[SearchResult]:
try:
root = ET.fromstring(xml_text)
except ET.ParseError:
return []
ns = {"atom": "http://www.w3.org/2005/Atom"}
results = []
@@ -53,14 +57,11 @@ class ArxivEngine(BaseEngine):
link_el = entry.find("atom:id", ns)
url = link_el.get("href", "") if link_el is not None else ""
published = entry.findtext("atom:published", "", ns)
authors = [a.findtext("atom:name", "", ns)
for a in entry.findall("atom:author", ns)]
categories = [c.get("term", "")
for c in entry.findall("atom:category", ns)]
authors = [a.findtext("atom:name", "", ns) for a in entry.findall("atom:author", ns)]
categories = [c.get("term", "") for c in entry.findall("atom:category", ns)]
if title:
results.append(EngineResult(
results.append(SearchResult(
title=title, url=url, snippet=summary[:300],
source="arxiv", category="academic",
published=published,
@@ -71,20 +72,16 @@ class ArxivEngine(BaseEngine):
class CrossRefEngine(BaseEngine):
"""CrossRef academic search — free, no key needed."""
"""CrossRef academic search via public API."""
name = "crossref"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
**kwargs) -> list[SearchResult]:
t0 = time.time()
url = "https://api.crossref.org/works"
params = {
"query": query,
"rows": min(max_results, 20),
"sort": "relevance",
"order": "desc",
}
params = {"query": query, "rows": min(max_results, 20),
"sort": "relevance", "order": "desc"}
headers = {"User-Agent": "web-search-skills/1.0 (mailto:search@example.com)"}
try:
@@ -109,7 +106,7 @@ class CrossRefEngine(BaseEngine):
for a in item.get("author", [])]
if title:
results.append(EngineResult(
results.append(SearchResult(
title=title, url=url_link, snippet=abstract,
source="crossref", category="academic",
published=pub_date,
@@ -134,7 +131,7 @@ class AcademicSearch(BaseEngine):
name = "academic"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
**kwargs) -> list[SearchResult]:
import asyncio
arxiv = ArxivEngine(timeout=self.timeout)
crossref = CrossRefEngine(timeout=self.timeout)
@@ -151,7 +148,6 @@ class AcademicSearch(BaseEngine):
if isinstance(crossref_results, list):
all_results.extend(crossref_results)
# Deduplicate by URL
seen = set()
deduped = []
for r in all_results:
+25 -54
View File
@@ -18,7 +18,7 @@ from typing import Any, Optional
import httpx
from scripts.core.base import SearchEngine, SearchResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
@@ -51,18 +51,18 @@ def _get_api_key() -> str:
return env_from_file.get(ENV_KEY, "")
class BaiduQianFanEngine(SearchEngine):
class BaiduQianFanEngine(BaseEngine):
"""百度千帆 AI 搜索 API 引擎."""
def __init__(self, config: Optional[dict] = None):
super().__init__("Baidu Qianfan", config)
name = "baidu_qianfan"
def __init__(self, config: Optional[dict] = None, timeout: float = 30.0):
super().__init__(timeout=timeout)
self.api_key = (config or {}).get("api_key", "") or _get_api_key()
if not self.api_key:
logger.warning(
f"BAIDU_API_KEY not set! Set it in .env file or as environment variable."
"BAIDU_API_KEY not set. Set it in .env or as environment variable."
)
timeout = (config or {}).get("timeout", 30)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
@@ -76,17 +76,7 @@ class BaiduQianFanEngine(SearchEngine):
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search via Baidu Qianfan AI Search API.
Args:
query: Search keywords.
max_results: Max results to return.
**kwargs:
site: Restrict search to a specific site (e.g. "mp.weixin.qq.com").
recency: Time filter: "week", "month", "semiyear", "year" (default: "year").
edition: "standard" (default) or "lite".
safe_search: bool, default False.
"""
"""Search via Baidu Qianfan AI Search API."""
if not self.api_key:
return [
SearchResult(
@@ -100,19 +90,14 @@ class BaiduQianFanEngine(SearchEngine):
)
]
# Build resource_type_filter based on max_results
top_k = min(max_results, 50)
resource_type_filter = [
{"type": "web", "top_k": top_k},
]
resource_type_filter = [{"type": "web", "top_k": top_k}]
# Build search_filter if site is specified
search_filter = {}
site = kwargs.get("site")
if site:
search_filter["match"] = {"site": [site]}
search_filter: dict[str, Any] = {}
if kwargs.get("site"):
search_filter["match"] = {"site": [kwargs["site"]]}
request_body = {
request_body: dict[str, Any] = {
"messages": [{"content": query, "role": "user"}],
"edition": kwargs.get("edition", "standard"),
"search_source": "baidu_search_v2",
@@ -134,38 +119,24 @@ class BaiduQianFanEngine(SearchEngine):
logger.error(f"Baidu Qianfan API error: {data.get('message', '')}")
return []
references = data.get("references", [])
results = []
for i, ref in enumerate(references[:max_results]):
results.append(
SearchResult(
title=ref.get("title", ""),
url=ref.get("url", ""),
snippet=ref.get("snippet", ""),
source=self.name,
rank=i + 1,
category="web",
extra={
k: ref[k]
for k in ("site", "date", "type", "source")
if k in ref
},
)
)
for i, ref in enumerate(data.get("references", [])[:max_results]):
results.append(SearchResult(
title=ref.get("title", ""),
url=ref.get("url", ""),
snippet=ref.get("snippet", ""),
source=self.name,
rank=i + 1,
category="web",
extra={k: ref[k] for k in ("site", "date", "type", "source") if k in ref},
))
return results
except httpx.HTTPStatusError as e:
logger.error(f"Baidu Qianfan API HTTP error: {e.response.status_code}")
if e.response.status_code == 401:
logger.error("API Key 无效或未授权,请检查 BAIDU_API_KEY")
elif e.response.status_code == 429:
logger.error("API 配额不足")
logger.error(f"Baidu Qianfan HTTP {e.response.status_code}: {e}")
return []
except httpx.RequestError as e:
logger.error(f"Baidu Qianfan API request failed: {e}")
return []
except json.JSONDecodeError as e:
logger.error(f"Baidu Qianfan API response parse failed: {e}")
logger.error(f"Baidu Qianfan request failed: {e}")
return []
async def close(self):
+4 -4
View File
@@ -9,7 +9,7 @@ from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
@@ -27,7 +27,7 @@ class BaiduScraper(BaseEngine):
name = "baidu"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
**kwargs) -> list[SearchResult]:
t0 = time.time()
url = f"https://www.baidu.com/s?wd={quote_plus(query)}&rn={max_results}"
@@ -43,7 +43,7 @@ class BaiduScraper(BaseEngine):
logger.warning(f"[baidu] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
@@ -60,7 +60,7 @@ class BaiduScraper(BaseEngine):
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href:
results.append(EngineResult(
results.append(SearchResult(
title=title, url=href, snippet=snippet, source="baidu",
))
if len(results) >= max_results:
+3 -55
View File
@@ -6,59 +6,7 @@ from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any
from scripts.core.execution_log import execution_log
# Re-export SearchResult/BaseEngine from scripts.core.base for engine code compatibility
from scripts.core.base import BaseEngine, SearchResult
@dataclass
class EngineResult:
"""Unified result from any engine."""
title: str
url: str
snippet: str = ""
source: str = ""
category: str = "" # "general" | "news" | "academic" | "github"
published: str = ""
extra: dict[str, Any] = field(default_factory=dict)
class BaseEngine(ABC):
"""Base class for all search engines.
Automatically tracks execution timing and errors via execution_log.
"""
name: str = "unknown"
timeout: float = 10.0
def __init__(self, timeout: float = 10.0):
self.timeout = timeout
async def search_with_log(
self, query: str, max_results: int = 10, **kwargs
) -> list[EngineResult]:
"""Search with automatic execution logging.
Subclasses override `search()`, not this method.
"""
with execution_log.track(self.name, "search", query=query) as ctx:
results = await self.search(query, max_results=max_results, **kwargs)
ctx.result_count = len(results)
return results
@abstractmethod
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
...
async def health(self) -> dict:
"""Quick connectivity check."""
with execution_log.track(self.name, "health") as ctx:
try:
results = await self.search("test", max_results=1)
ctx.result_count = len(results)
return {"name": self.name, "status": "ok", "count": len(results)}
except Exception as e:
ctx.success = False
ctx.error = str(e)
ctx.error_type = type(e).__name__
return {"name": self.name, "status": "fail", "error": str(e)}
__all__ = ["BaseEngine", "SearchResult"]
+4 -4
View File
@@ -10,7 +10,7 @@ from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
@@ -28,7 +28,7 @@ class BingScraper(BaseEngine):
name = "bing"
async def search(self, query: str, max_results: int = 10,
region: str = "global", **kwargs) -> list[EngineResult]:
region: str = "global", **kwargs) -> list[SearchResult]:
t0 = time.time()
url = f"https://cn.bing.com/search?q={quote_plus(query)}&setlang=zh-CN"
if region == "global":
@@ -46,7 +46,7 @@ class BingScraper(BaseEngine):
logger.warning(f"[bing] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
@@ -60,7 +60,7 @@ class BingScraper(BaseEngine):
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href:
results.append(EngineResult(
results.append(SearchResult(
title=title, url=href, snippet=snippet, source="bing",
))
if len(results) >= max_results:
+4 -4
View File
@@ -9,7 +9,7 @@ from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
@@ -27,7 +27,7 @@ class BraveScraper(BaseEngine):
name = "brave"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
**kwargs) -> list[SearchResult]:
t0 = time.time()
url = f"https://search.brave.com/search?q={quote_plus(query)}&source=web"
@@ -43,7 +43,7 @@ class BraveScraper(BaseEngine):
logger.warning(f"[brave] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
@@ -60,7 +60,7 @@ class BraveScraper(BaseEngine):
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href and href.startswith("http"):
results.append(EngineResult(
results.append(SearchResult(
title=title, url=href, snippet=snippet, source="brave",
))
if len(results) >= max_results:
+5 -5
View File
@@ -8,7 +8,7 @@ from urllib.parse import quote_plus
import httpx
from scripts.engines.base import BaseEngine, EngineResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
@@ -24,7 +24,7 @@ class GitHubEngine(BaseEngine):
name = "github"
async def search(self, query: str, max_results: int = 10,
search_type: str = "repositories", **kwargs) -> list[EngineResult]:
search_type: str = "repositories", **kwargs) -> list[SearchResult]:
"""search_type: 'repositories' | 'code' | 'issues'"""
t0 = time.time()
endpoint = f"https://api.github.com/search/{search_type}"
@@ -40,7 +40,7 @@ class GitHubEngine(BaseEngine):
results = []
for item in data.get("items", [])[:max_results]:
if search_type == "repositories":
results.append(EngineResult(
results.append(SearchResult(
title=item.get("full_name", ""),
url=item.get("html_url", ""),
snippet=item.get("description", "") or "",
@@ -53,7 +53,7 @@ class GitHubEngine(BaseEngine):
},
))
elif search_type == "code":
results.append(EngineResult(
results.append(SearchResult(
title=item.get("name", ""),
url=item.get("html_url", ""),
snippet=item.get("text_matches", [{}])[0].get("fragment", "")
@@ -66,7 +66,7 @@ class GitHubEngine(BaseEngine):
},
))
elif search_type == "issues":
results.append(EngineResult(
results.append(SearchResult(
title=item.get("title", ""),
url=item.get("html_url", ""),
snippet=item.get("body", "")[:300] if item.get("body") else "",
+46 -53
View File
@@ -1,40 +1,62 @@
"""News search engine — multi-source news aggregation."""
"""News search — multi-source aggregation (Bing News + RSS)."""
from __future__ import annotations
import re
import time
import logging
from urllib.parse import quote_plus
from xml.etree import ElementTree as ET
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
HEADERS = {
_HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
# Free RSS feeds — no API key needed
_RSS_FEEDS = {
# Chinese tech/business
"36kr": "https://36kr.com/feed",
"少数派": "https://sspai.com/feed",
"IT之家": "https://www.ithome.com/rss/",
"钛媒体": "https://www.tmtpost.com/rss.xml",
"Solidot": "https://www.solidot.org/index.rss",
"InfoQ CN": "https://www.infoq.cn/feed",
"OSChina": "https://www.oschina.net/news/rss",
# English tech
"HackerNews": "https://hnrss.org/best?count=20",
"TechCrunch": "https://techcrunch.com/feed/",
"ArsTechnica": "https://feeds.arstechnica.com/arstechnica/index",
"TheVerge": "https://www.theverge.com/rss/index.xml",
"MIT Tech Review": "https://www.technologyreview.com/feed/",
"Wired": "https://www.wired.com/feed/rss",
"Lobste.rs": "https://lobste.rs/rss",
# English general
"ChinaDaily": "https://www.chinadaily.com.cn/rss/china_rss.xml",
}
class BingNewsEngine(BaseEngine):
"""Bing News scraping — good for both CN and EN news."""
"""Bing News scraping — works for both CN and global news."""
name = "bing_news"
async def search(self, query: str, max_results: int = 10,
region: str = "cn", **kwargs) -> list[EngineResult]:
region: str = "cn", **kwargs) -> list[SearchResult]:
t0 = time.time()
base = "cn.bing.com" if region == "cn" else "www.bing.com"
url = f"https://{base}/news/search?q={quote_plus(query)}&setlang=zh-CN"
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=HEADERS)
resp = await client.get(url, headers=_HEADERS)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse(resp.text, max_results)
@@ -44,20 +66,20 @@ class BingNewsEngine(BaseEngine):
logger.warning(f"[bing_news] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, "html.parser")
results = []
for item in soup.select("a.title"):
title = item.get_text(strip=True)
href = item.get("href", "")
# Try to get snippet from parent container
parent = item.find_parent("div", class_=True)
snippet = ""
if parent:
snippet_el = parent.select_one("div.snippet")
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href and href.startswith("http"):
results.append(EngineResult(
results.append(SearchResult(
title=title, url=href, snippet=snippet,
source="bing_news", category="news",
))
@@ -67,40 +89,18 @@ class BingNewsEngine(BaseEngine):
class RSSNewsEngine(BaseEngine):
"""RSS-based news — multi-source aggregation."""
"""RSS feed aggregation — multi-source news from curated feeds."""
name = "rss_news"
# Free RSS feeds, no key needed
FEEDS = {
# Chinese tech/business
"36kr": "https://36kr.com/feed",
"少数派": "https://sspai.com/feed",
"IT之家": "https://www.ithome.com/rss/",
"钛媒体": "https://www.tmtpost.com/rss.xml",
"Solidot": "https://www.solidot.org/index.rss",
"InfoQ CN": "https://www.infoq.cn/feed",
"OSChina": "https://www.oschina.net/news/rss",
# English tech
"HackerNews": "https://hnrss.org/best?count=20",
"TechCrunch": "https://techcrunch.com/feed/",
"ArsTechnica": "https://feeds.arstechnica.com/arstechnica/index",
"TheVerge": "https://www.theverge.com/rss/index.xml",
"MIT Tech Review": "https://www.technologyreview.com/feed/",
"Wired": "https://www.wired.com/feed/rss",
"Lobste.rs": "https://lobste.rs/rss",
# English general
"ChinaDaily": "https://www.chinadaily.com.cn/rss/china_rss.xml",
}
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
**kwargs) -> list[SearchResult]:
t0 = time.time()
all_results = []
import asyncio
tasks = [self._fetch_feed(name, url, query)
for name, url in self.FEEDS.items()]
for name, url in _RSS_FEEDS.items()]
feeds = await asyncio.gather(*tasks, return_exceptions=True)
for feed_results in feeds:
@@ -111,52 +111,45 @@ class RSSNewsEngine(BaseEngine):
logger.info(f"[rss_news] {len(all_results)} results in {elapsed:.2f}s")
return all_results[:max_results]
async def _fetch_feed(self, name: str, url: str, query: str) -> list[EngineResult]:
async def _fetch_feed(self, name: str, url: str, query: str) -> list[SearchResult]:
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=HEADERS)
resp = await client.get(url, headers=_HEADERS)
resp.raise_for_status()
return self._parse_feed(name, resp.text, query)
except Exception as e:
logger.warning(f"[rss_news:{name}] failed: {e}")
return []
def _parse_feed(self, name: str, xml_text: str, query: str) -> list[EngineResult]:
def _parse_feed(self, name: str, xml_text: str, query: str) -> list[SearchResult]:
try:
root = ET.fromstring(xml_text)
except ET.ParseError:
return []
results = []
# RSS 2.0: //channel/item, Atom: //entry
ns = {"atom": "http://www.w3.org/2005/Atom"}
items = root.findall(".//item") or root.findall(".//atom:entry", ns)
q_lower = query.lower()
q_words = [w for w in q_lower.split() if len(w) > 1]
results = []
for item in items[:50]:
title = (item.findtext("title") or
item.findtext("atom:title", namespaces=ns) or "")
link = (item.findtext("link") or "")
link = item.findtext("link") or ""
if not link:
link_el = item.find("atom:link", ns)
link = link_el.get("href", "") if link_el is not None else ""
desc = (item.findtext("description") or
item.findtext("atom:summary", namespaces=ns) or "")
# Clean HTML tags from description
import re
desc = re.sub(r'<[^>]+>', '', desc)
# Keyword scoring (not strict filter)
text_lower = f"{title} {desc}".lower()
if q_words:
match_count = sum(1 for w in q_words if w in text_lower)
if match_count == 0:
continue # Skip completely irrelevant items
if q_words and not any(w in text_lower for w in q_words):
continue # Skip items with no query keyword match
results.append(EngineResult(
results.append(SearchResult(
title=title.strip(),
url=link.strip(),
snippet=desc[:300].strip(),
@@ -168,15 +161,15 @@ class RSSNewsEngine(BaseEngine):
class NewsSearch(BaseEngine):
"""Unified news search — Bing News + RSS in parallel."""
"""Unified news search — Bing News + RSS feeds in parallel."""
name = "news"
async def search(self, query: str, max_results: int = 10,
region: str = "cn", **kwargs) -> list[EngineResult]:
region: str = "cn", **kwargs) -> list[SearchResult]:
import asyncio
bing = BingNewsEngine(timeout=self.timeout)
rss = RSSNewsEngine(timeout=8.0) # RSS feeds may be slow
rss = RSSNewsEngine(timeout=8.0)
bing_results, rss_results = await asyncio.gather(
bing.search(query, max_results=max_results, region=region),
+4 -4
View File
@@ -8,7 +8,7 @@ import logging
from dataclasses import dataclass, field
from typing import Any
from scripts.engines.base import BaseEngine, EngineResult
from scripts.core.base import BaseEngine, SearchResult
from scripts.engines.bing_scraper import BingScraper
from scripts.engines.sogou_scraper import SogouScraper
from scripts.engines.baidu_scraper import BaiduScraper
@@ -59,7 +59,7 @@ class ParallelSearch:
max_results: int = 10,
region: str = "cn",
**kwargs,
) -> tuple[list[EngineResult], SearchStats]:
) -> tuple[list[SearchResult], SearchStats]:
"""
Search across specified engines in parallel.
@@ -138,7 +138,7 @@ class ParallelSearch:
async def _run_engine(
self, engine: BaseEngine, query: str, max_results: int, kwargs: dict
) -> tuple[list[EngineResult], float]:
) -> tuple[list[SearchResult], float]:
"""Run a single engine with timing."""
with execution_log.track(engine.name, "search", query=query) as ctx:
t0 = time.time()
@@ -162,7 +162,7 @@ class ParallelSearch:
vertical: str,
max_results: int = 10,
**kwargs,
) -> tuple[list[EngineResult], SearchStats]:
) -> tuple[list[SearchResult], SearchStats]:
"""Search a specific vertical (news/academic/github)."""
if vertical == "github":
# GitHub supports sub-types
+89 -55
View File
@@ -1,80 +1,114 @@
"""
Parsing-based search engines (e.g. DuckDuckGo HTML scraper).
"""
"""Parsing-based search engines — DuckDuckGo, Brave (fallback HTML scraper)."""
from __future__ import annotations
import logging
from typing import Optional
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.core.base import SearchEngine, SearchResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
_HEADERS = {
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/125.0.0.0 Safari/537.36"),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
class DuckDuckGoEngine(SearchEngine):
"""DuckDuckGo HTML search engine (no API key required)."""
SEARCH_URL = "https://html.duckduckgo.com/html/"
class DuckDuckGoParser(BaseEngine):
"""DuckDuckGo HTML scraper (no API key needed)."""
def __init__(self, config: Optional[dict] = None):
super().__init__("DuckDuckGo", config)
timeout = (config or {}).get("timeout", 15)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"User-Agent": (config or {}).get(
"user_agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36",
),
},
follow_redirects=True,
)
name = "duckduckgo"
def build_url(self, query: str, **kwargs) -> str:
"""Build DuckDuckGo search URL."""
from urllib.parse import quote_plus
return f"https://duckduckgo.com/?q={quote_plus(query)}"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[SearchResult]:
import time
t0 = time.time()
url = f"https://html.duckduckgo.com/html/?q={quote_plus(query)}"
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Search DuckDuckGo and parse HTML results."""
params = {"q": query}
try:
response = await self._client.post(self.SEARCH_URL, data=params)
response.raise_for_status()
return self._parse_results(response.text, max_results)
except httpx.HTTPError as e:
logger.error(f"DuckDuckGo search failed: {e}")
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=_HEADERS)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse(resp.text, max_results)
logger.info(f"[duckduckgo] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[duckduckgo] failed: {e}")
return []
def _parse_results(self, html: str, max_results: int) -> list[SearchResult]:
"""Parse DuckDuckGo HTML search results."""
soup = BeautifulSoup(html, "lxml")
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
for i, result in enumerate(soup.select(".result")[:max_results]):
for result in soup.select(".result")[:max_results]:
title_el = result.select_one(".result__title a")
snippet_el = result.select_one(".result__snippet")
if title_el:
results.append(
SearchResult(
title=title_el.get_text(strip=True),
url=title_el.get("href", ""),
snippet=snippet_el.get_text(strip=True)
if snippet_el else "",
source=self.name,
rank=i + 1,
category="web",
)
)
if not title_el:
continue
results.append(SearchResult(
title=title_el.get_text(strip=True),
url=title_el.get("href", ""),
snippet=snippet_el.get_text(strip=True) if snippet_el else "",
source="duckduckgo",
category="web",
))
return results
async def close(self):
await self._client.aclose()
class BraveParser(BaseEngine):
"""Brave Search fallback HTML scraper (no API key needed)."""
name = "brave"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[SearchResult]:
import time
t0 = time.time()
url = f"https://search.brave.com/search?q={quote_plus(query)}&source=web"
try:
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=_HEADERS)
resp.raise_for_status()
elapsed = time.time() - t0
results = self._parse(resp.text, max_results)
logger.info(f"[brave] {len(results)} results in {elapsed:.2f}s")
return results
except Exception as e:
logger.warning(f"[brave] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
for item in soup.select("div.snippet, div.result")[:max_results]:
title_el = item.select_one("a.result-header, a.snippet-title, a")
if not title_el:
continue
title = title_el.get_text(strip=True)
href = title_el.get("href", "")
snippet_el = item.select_one("p.snippet-description, div.snippet-description")
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href and href.startswith("http"):
results.append(SearchResult(
title=title, url=href, snippet=snippet,
source="brave", category="web",
))
return results
# Backwards-compatibility aliases
DuckDuckGoEngine = DuckDuckGoParser
BraveScraper = BraveParser
+4 -4
View File
@@ -9,7 +9,7 @@ from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
@@ -27,7 +27,7 @@ class Search360Scraper(BaseEngine):
name = "360"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
**kwargs) -> list[SearchResult]:
t0 = time.time()
url = f"https://www.so.com/s?q={quote_plus(query)}"
@@ -43,7 +43,7 @@ class Search360Scraper(BaseEngine):
logger.warning(f"[360] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
@@ -57,7 +57,7 @@ class Search360Scraper(BaseEngine):
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href:
results.append(EngineResult(
results.append(SearchResult(
title=title, url=href, snippet=snippet, source="360",
))
if len(results) >= max_results:
+4 -4
View File
@@ -9,7 +9,7 @@ from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.engines.base import BaseEngine, EngineResult
from scripts.core.base import BaseEngine, SearchResult
logger = logging.getLogger(__name__)
@@ -27,7 +27,7 @@ class SogouScraper(BaseEngine):
name = "sogou"
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[EngineResult]:
**kwargs) -> list[SearchResult]:
t0 = time.time()
url = f"https://www.sogou.com/web?query={quote_plus(query)}"
@@ -43,7 +43,7 @@ class SogouScraper(BaseEngine):
logger.warning(f"[sogou] failed: {e}")
return []
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
def _parse(self, html: str, max_results: int) -> list[SearchResult]:
soup = BeautifulSoup(html, "html.parser")
results = []
@@ -57,7 +57,7 @@ class SogouScraper(BaseEngine):
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
if title and href and not href.startswith("/"):
results.append(EngineResult(
results.append(SearchResult(
title=title, url=href, snippet=snippet, source="sogou",
))
if len(results) >= max_results:
+10 -13
View File
@@ -1,31 +1,31 @@
"""
Generic URL-based search engine.
Builds search URLs from templates defined in engines.json
Builds search URLs from templates (config/engines.json)
and returns them as URL results.
"""
from __future__ import annotations
import logging
from typing import Any, Optional
from urllib.parse import quote_plus
from typing import Any
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
from scripts.core.base import BaseEngine, SearchResult
class UrlSearchEngine(SearchEngine):
class UrlSearchEngine(BaseEngine):
"""
Generic search engine that builds URLs from templates.
Generic engine that builds search URLs from templates.
Reads URL templates from config (engines.json) and returns
the constructed search URL as a SearchResult.
"""
def __init__(self, name: str, engine_def: dict[str, Any]):
super().__init__(name, engine_def.get("config", {}))
name: str = "url"
def __init__(self, name: str, engine_def: dict[str, Any], timeout: float = 10.0):
super().__init__(timeout=timeout)
self.name = name
self.url_template = engine_def.get("url", "")
self.region = engine_def.get("region", "global")
@@ -55,6 +55,3 @@ class UrlSearchEngine(SearchEngine):
extra={"url": url, "region": self.region},
)
]
async def close(self):
pass