feat: 添加执行日志系统,跟踪各引擎/接口调用耗时和错误信息
新增功能: - execution_log 模块:自动记录引擎调用耗时、成功/失败、错误详情 - 所有搜索引擎自动跟踪每次调用的性能指标 - CLI 搜索后自动显示执行摘要 - health 命令使用 ParallelSearch 检测所有引擎连通性 - 新增 log 命令查看历史执行日志
This commit is contained in:
1 parent
f2d8919860
commit
a60ba71663
18 files changed
+1671
-332
No files matched your search
@@ -0,0 +1,162 @@
|
||||
"""Academic search engine — ArXiv + CrossRef APIs (free, no key)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import logging
|
||||
from urllib.parse import quote_plus
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ArxivEngine(BaseEngine):
|
||||
"""ArXiv paper search — free, no key needed."""
|
||||
|
||||
name = "arxiv"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
t0 = time.time()
|
||||
url = (
|
||||
f"http://export.arxiv.org/api/query?"
|
||||
f"search_query=all:{quote_plus(query)}&start=0&max_results={max_results}"
|
||||
f"&sortBy=relevance&sortOrder=descending"
|
||||
)
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse_xml(resp.text, max_results)
|
||||
logger.info(f"[arxiv] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[arxiv] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_xml(self, xml_text: str, max_results: int) -> list[EngineResult]:
|
||||
root = ET.fromstring(xml_text)
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||
results = []
|
||||
|
||||
for entry in root.findall("atom:entry", ns)[:max_results]:
|
||||
title = entry.findtext("atom:title", "", ns).strip().replace("\n", " ")
|
||||
summary = entry.findtext("atom:summary", "", ns).strip().replace("\n", " ")
|
||||
link_el = entry.find("atom:link[@type='text/html']", ns)
|
||||
if link_el is None:
|
||||
link_el = entry.find("atom:id", ns)
|
||||
url = link_el.get("href", "") if link_el is not None else ""
|
||||
published = entry.findtext("atom:published", "", ns)
|
||||
|
||||
authors = [a.findtext("atom:name", "", ns)
|
||||
for a in entry.findall("atom:author", ns)]
|
||||
categories = [c.get("term", "")
|
||||
for c in entry.findall("atom:category", ns)]
|
||||
|
||||
if title:
|
||||
results.append(EngineResult(
|
||||
title=title, url=url, snippet=summary[:300],
|
||||
source="arxiv", category="academic",
|
||||
published=published,
|
||||
extra={"authors": authors[:5], "categories": categories},
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
class CrossRefEngine(BaseEngine):
|
||||
"""CrossRef academic search — free, no key needed."""
|
||||
|
||||
name = "crossref"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
t0 = time.time()
|
||||
url = "https://api.crossref.org/works"
|
||||
params = {
|
||||
"query": query,
|
||||
"rows": min(max_results, 20),
|
||||
"sort": "relevance",
|
||||
"order": "desc",
|
||||
}
|
||||
headers = {"User-Agent": "web-search-skills/1.0 (mailto:search@example.com)"}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, params=params, headers=headers)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
data = resp.json()
|
||||
results = []
|
||||
for item in data.get("message", {}).get("items", [])[:max_results]:
|
||||
title_list = item.get("title", [])
|
||||
title = title_list[0] if title_list else ""
|
||||
doi = item.get("DOI", "")
|
||||
url_link = item.get("URL", f"https://doi.org/{doi}")
|
||||
abstract = item.get("abstract", "")[:300]
|
||||
pub_date = ""
|
||||
if item.get("published-print"):
|
||||
dp = item["published-print"].get("date-parts", [[]])[0]
|
||||
pub_date = "-".join(str(p) for p in dp if p) if dp else ""
|
||||
|
||||
authors = [f"{a.get('given', '')} {a.get('family', '')}".strip()
|
||||
for a in item.get("author", [])]
|
||||
|
||||
if title:
|
||||
results.append(EngineResult(
|
||||
title=title, url=url_link, snippet=abstract,
|
||||
source="crossref", category="academic",
|
||||
published=pub_date,
|
||||
extra={
|
||||
"doi": doi,
|
||||
"authors": authors[:5],
|
||||
"journal": (item.get("container-title", [""])[0]
|
||||
if item.get("container-title") else ""),
|
||||
},
|
||||
))
|
||||
|
||||
logger.info(f"[crossref] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[crossref] failed: {e}")
|
||||
return []
|
||||
|
||||
|
||||
class AcademicSearch(BaseEngine):
|
||||
"""Unified academic search — ArXiv + CrossRef in parallel."""
|
||||
|
||||
name = "academic"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
import asyncio
|
||||
arxiv = ArxivEngine(timeout=self.timeout)
|
||||
crossref = CrossRefEngine(timeout=self.timeout)
|
||||
|
||||
arxiv_results, crossref_results = await asyncio.gather(
|
||||
arxiv.search(query, max_results=max_results),
|
||||
crossref.search(query, max_results=max_results),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
all_results = []
|
||||
if isinstance(arxiv_results, list):
|
||||
all_results.extend(arxiv_results)
|
||||
if isinstance(crossref_results, list):
|
||||
all_results.extend(crossref_results)
|
||||
|
||||
# Deduplicate by URL
|
||||
seen = set()
|
||||
deduped = []
|
||||
for r in all_results:
|
||||
if r.url not in seen:
|
||||
seen.add(r.url)
|
||||
deduped.append(r)
|
||||
|
||||
return deduped[:max_results]
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Baidu HTML scraping engine — free, fastest in China, best Chinese results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import logging
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
|
||||
|
||||
class BaiduScraper(BaseEngine):
|
||||
"""Baidu web scraping — fastest in China, best Chinese content."""
|
||||
|
||||
name = "baidu"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
t0 = time.time()
|
||||
url = f"https://www.baidu.com/s?wd={quote_plus(query)}&rn={max_results}"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
logger.info(f"[baidu] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[baidu] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
for item in soup.select("div.result, div.c-container"):
|
||||
title_el = item.select_one("h3 a")
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
href = title_el.get("href", "")
|
||||
snippet_el = item.select_one(
|
||||
"span.content-right_8Zs40, div.c-abstract, "
|
||||
"span[class*='content'], div[class*='abstract']"
|
||||
)
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href:
|
||||
results.append(EngineResult(
|
||||
title=title, url=href, snippet=snippet, source="baidu",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
break
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Base engine class and shared data models."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from scripts.core.execution_log import execution_log
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineResult:
|
||||
"""Unified result from any engine."""
|
||||
title: str
|
||||
url: str
|
||||
snippet: str = ""
|
||||
source: str = ""
|
||||
category: str = "" # "general" | "news" | "academic" | "github"
|
||||
published: str = ""
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class BaseEngine(ABC):
|
||||
"""Base class for all search engines.
|
||||
|
||||
Automatically tracks execution timing and errors via execution_log.
|
||||
"""
|
||||
|
||||
name: str = "unknown"
|
||||
timeout: float = 10.0
|
||||
|
||||
def __init__(self, timeout: float = 10.0):
|
||||
self.timeout = timeout
|
||||
|
||||
async def search_with_log(
|
||||
self, query: str, max_results: int = 10, **kwargs
|
||||
) -> list[EngineResult]:
|
||||
"""Search with automatic execution logging.
|
||||
|
||||
Subclasses override `search()`, not this method.
|
||||
"""
|
||||
with execution_log.track(self.name, "search", query=query) as ctx:
|
||||
results = await self.search(query, max_results=max_results, **kwargs)
|
||||
ctx.result_count = len(results)
|
||||
return results
|
||||
|
||||
@abstractmethod
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
...
|
||||
|
||||
async def health(self) -> dict:
|
||||
"""Quick connectivity check."""
|
||||
with execution_log.track(self.name, "health") as ctx:
|
||||
try:
|
||||
results = await self.search("test", max_results=1)
|
||||
ctx.result_count = len(results)
|
||||
return {"name": self.name, "status": "ok", "count": len(results)}
|
||||
except Exception as e:
|
||||
ctx.success = False
|
||||
ctx.error = str(e)
|
||||
ctx.error_type = type(e).__name__
|
||||
return {"name": self.name, "status": "fail", "error": str(e)}
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Bing HTML scraping engine — free, no API key."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
|
||||
class BingScraper(BaseEngine):
|
||||
"""Bing web scraping — fast, good English results."""
|
||||
|
||||
name = "bing"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
region: str = "global", **kwargs) -> list[EngineResult]:
|
||||
t0 = time.time()
|
||||
url = f"https://cn.bing.com/search?q={quote_plus(query)}&setlang=zh-CN"
|
||||
if region == "global":
|
||||
url = f"https://www.bing.com/search?q={quote_plus(query)}"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
logger.info(f"[bing] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[bing] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
for item in soup.select("li.b_algo"):
|
||||
title_el = item.select_one("h2 a")
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
href = title_el.get("href", "")
|
||||
snippet_el = item.select_one("p, .b_caption p")
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href:
|
||||
results.append(EngineResult(
|
||||
title=title, url=href, snippet=snippet, source="bing",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
break
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Brave Search HTML scraping engine — free, privacy-focused."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import logging
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
|
||||
class BraveScraper(BaseEngine):
|
||||
"""Brave Search scraping — privacy-focused, good results."""
|
||||
|
||||
name = "brave"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
t0 = time.time()
|
||||
url = f"https://search.brave.com/search?q={quote_plus(query)}&source=web"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
logger.info(f"[brave] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[brave] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
# Brave uses div.snippet or article elements
|
||||
for item in soup.select("div.snippet, div.result"):
|
||||
title_el = item.select_one("a.result-header, a.snippet-title")
|
||||
if not title_el:
|
||||
title_el = item.select_one("a")
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
href = title_el.get("href", "")
|
||||
snippet_el = item.select_one("p.snippet-description, div.snippet-description")
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href and href.startswith("http"):
|
||||
results.append(EngineResult(
|
||||
title=title, url=href, snippet=snippet, source="brave",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
break
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,86 @@
|
||||
"""GitHub code search engine — uses GitHub API (no key needed for basic search)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import logging
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADERS = {
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
"User-Agent": "web-search-skills/1.0",
|
||||
}
|
||||
|
||||
|
||||
class GitHubEngine(BaseEngine):
|
||||
"""GitHub code & repo search via public API."""
|
||||
|
||||
name = "github"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
search_type: str = "repositories", **kwargs) -> list[EngineResult]:
|
||||
"""search_type: 'repositories' | 'code' | 'issues'"""
|
||||
t0 = time.time()
|
||||
endpoint = f"https://api.github.com/search/{search_type}"
|
||||
params = {"q": query, "per_page": min(max_results, 30), "sort": "stars"}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(endpoint, headers=HEADERS, params=params)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
elapsed = time.time() - t0
|
||||
|
||||
results = []
|
||||
for item in data.get("items", [])[:max_results]:
|
||||
if search_type == "repositories":
|
||||
results.append(EngineResult(
|
||||
title=item.get("full_name", ""),
|
||||
url=item.get("html_url", ""),
|
||||
snippet=item.get("description", "") or "",
|
||||
source="github",
|
||||
category="github",
|
||||
extra={
|
||||
"stars": item.get("stargazers_count", 0),
|
||||
"language": item.get("language", ""),
|
||||
"updated": item.get("updated_at", ""),
|
||||
},
|
||||
))
|
||||
elif search_type == "code":
|
||||
results.append(EngineResult(
|
||||
title=item.get("name", ""),
|
||||
url=item.get("html_url", ""),
|
||||
snippet=item.get("text_matches", [{}])[0].get("fragment", "")
|
||||
if item.get("text_matches") else "",
|
||||
source="github",
|
||||
category="github",
|
||||
extra={
|
||||
"repo": item.get("repository", {}).get("full_name", ""),
|
||||
"language": item.get("language", ""),
|
||||
},
|
||||
))
|
||||
elif search_type == "issues":
|
||||
results.append(EngineResult(
|
||||
title=item.get("title", ""),
|
||||
url=item.get("html_url", ""),
|
||||
snippet=item.get("body", "")[:300] if item.get("body") else "",
|
||||
source="github",
|
||||
category="github",
|
||||
extra={
|
||||
"state": item.get("state", ""),
|
||||
"repo": item.get("repository_url", "").split("/")[-2:]
|
||||
if item.get("repository_url") else [],
|
||||
},
|
||||
))
|
||||
|
||||
logger.info(f"[github] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[github] failed: {e}")
|
||||
return []
|
||||
@@ -0,0 +1,201 @@
|
||||
"""News search engine — multi-source news aggregation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import logging
|
||||
from urllib.parse import quote_plus
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
|
||||
class BingNewsEngine(BaseEngine):
|
||||
"""Bing News scraping — good for both CN and EN news."""
|
||||
|
||||
name = "bing_news"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
region: str = "cn", **kwargs) -> list[EngineResult]:
|
||||
t0 = time.time()
|
||||
base = "cn.bing.com" if region == "cn" else "www.bing.com"
|
||||
url = f"https://{base}/news/search?q={quote_plus(query)}&setlang=zh-CN"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
logger.info(f"[bing_news] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[bing_news] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
for item in soup.select("a.title"):
|
||||
title = item.get_text(strip=True)
|
||||
href = item.get("href", "")
|
||||
# Try to get snippet from parent container
|
||||
parent = item.find_parent("div", class_=True)
|
||||
snippet = ""
|
||||
if parent:
|
||||
snippet_el = parent.select_one("div.snippet")
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
if title and href and href.startswith("http"):
|
||||
results.append(EngineResult(
|
||||
title=title, url=href, snippet=snippet,
|
||||
source="bing_news", category="news",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
class RSSNewsEngine(BaseEngine):
|
||||
"""RSS-based news — multi-source aggregation."""
|
||||
|
||||
name = "rss_news"
|
||||
|
||||
# Free RSS feeds, no key needed
|
||||
FEEDS = {
|
||||
# Chinese tech/business
|
||||
"36kr": "https://36kr.com/feed",
|
||||
"少数派": "https://sspai.com/feed",
|
||||
"IT之家": "https://www.ithome.com/rss/",
|
||||
"钛媒体": "https://www.tmtpost.com/rss.xml",
|
||||
"Solidot": "https://www.solidot.org/index.rss",
|
||||
"InfoQ CN": "https://www.infoq.cn/feed",
|
||||
"OSChina": "https://www.oschina.net/news/rss",
|
||||
# English tech
|
||||
"HackerNews": "https://hnrss.org/best?count=20",
|
||||
"TechCrunch": "https://techcrunch.com/feed/",
|
||||
"ArsTechnica": "https://feeds.arstechnica.com/arstechnica/index",
|
||||
"TheVerge": "https://www.theverge.com/rss/index.xml",
|
||||
"MIT Tech Review": "https://www.technologyreview.com/feed/",
|
||||
"Wired": "https://www.wired.com/feed/rss",
|
||||
"Lobste.rs": "https://lobste.rs/rss",
|
||||
# English general
|
||||
"ChinaDaily": "https://www.chinadaily.com.cn/rss/china_rss.xml",
|
||||
}
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
t0 = time.time()
|
||||
all_results = []
|
||||
|
||||
import asyncio
|
||||
tasks = [self._fetch_feed(name, url, query)
|
||||
for name, url in self.FEEDS.items()]
|
||||
feeds = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
for feed_results in feeds:
|
||||
if isinstance(feed_results, list):
|
||||
all_results.extend(feed_results)
|
||||
|
||||
elapsed = time.time() - t0
|
||||
logger.info(f"[rss_news] {len(all_results)} results in {elapsed:.2f}s")
|
||||
return all_results[:max_results]
|
||||
|
||||
async def _fetch_feed(self, name: str, url: str, query: str) -> list[EngineResult]:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp.raise_for_status()
|
||||
return self._parse_feed(name, resp.text, query)
|
||||
except Exception as e:
|
||||
logger.warning(f"[rss_news:{name}] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse_feed(self, name: str, xml_text: str, query: str) -> list[EngineResult]:
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError:
|
||||
return []
|
||||
|
||||
results = []
|
||||
# RSS 2.0: //channel/item, Atom: //entry
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||
|
||||
items = root.findall(".//item") or root.findall(".//atom:entry", ns)
|
||||
|
||||
q_lower = query.lower()
|
||||
q_words = [w for w in q_lower.split() if len(w) > 1]
|
||||
|
||||
for item in items[:50]:
|
||||
title = (item.findtext("title") or
|
||||
item.findtext("atom:title", namespaces=ns) or "")
|
||||
link = (item.findtext("link") or "")
|
||||
if not link:
|
||||
link_el = item.find("atom:link", ns)
|
||||
link = link_el.get("href", "") if link_el is not None else ""
|
||||
desc = (item.findtext("description") or
|
||||
item.findtext("atom:summary", namespaces=ns) or "")
|
||||
# Clean HTML tags from description
|
||||
import re
|
||||
desc = re.sub(r'<[^>]+>', '', desc)
|
||||
|
||||
# Keyword scoring (not strict filter)
|
||||
text_lower = f"{title} {desc}".lower()
|
||||
if q_words:
|
||||
match_count = sum(1 for w in q_words if w in text_lower)
|
||||
if match_count == 0:
|
||||
continue # Skip completely irrelevant items
|
||||
|
||||
results.append(EngineResult(
|
||||
title=title.strip(),
|
||||
url=link.strip(),
|
||||
snippet=desc[:300].strip(),
|
||||
source=f"rss_{name}",
|
||||
category="news",
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
class NewsSearch(BaseEngine):
|
||||
"""Unified news search — Bing News + RSS in parallel."""
|
||||
|
||||
name = "news"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
region: str = "cn", **kwargs) -> list[EngineResult]:
|
||||
import asyncio
|
||||
bing = BingNewsEngine(timeout=self.timeout)
|
||||
rss = RSSNewsEngine(timeout=8.0) # RSS feeds may be slow
|
||||
|
||||
bing_results, rss_results = await asyncio.gather(
|
||||
bing.search(query, max_results=max_results, region=region),
|
||||
rss.search(query, max_results=max_results),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
all_results = []
|
||||
if isinstance(bing_results, list):
|
||||
all_results.extend(bing_results)
|
||||
if isinstance(rss_results, list):
|
||||
all_results.extend(rss_results)
|
||||
|
||||
# Deduplicate by URL
|
||||
seen = set()
|
||||
deduped = []
|
||||
for r in all_results:
|
||||
if r.url not in seen:
|
||||
seen.add(r.url)
|
||||
deduped.append(r)
|
||||
|
||||
return deduped[:max_results]
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Parallel search orchestrator — runs multiple engines concurrently."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
from scripts.engines.bing_scraper import BingScraper
|
||||
from scripts.engines.sogou_scraper import SogouScraper
|
||||
from scripts.engines.baidu_scraper import BaiduScraper
|
||||
from scripts.engines.search360_scraper import Search360Scraper
|
||||
from scripts.engines.news_engine import NewsSearch
|
||||
from scripts.engines.academic_engine import AcademicSearch
|
||||
from scripts.engines.github_engine import GitHubEngine
|
||||
from scripts.core.execution_log import execution_log
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchStats:
|
||||
"""Search statistics."""
|
||||
total_engines: int = 0
|
||||
successful_engines: int = 0
|
||||
failed_engines: list[str] = field(default_factory=list)
|
||||
total_results: int = 0
|
||||
deduplicated: int = 0
|
||||
duration_ms: int = 0
|
||||
engine_timings: dict[str, float] = field(default_factory=dict)
|
||||
|
||||
|
||||
class ParallelSearch:
|
||||
"""Run multiple engines in parallel and merge results."""
|
||||
|
||||
# Default engine sets
|
||||
GENERAL_ENGINES = ["bing", "sogou", "baidu", "360"]
|
||||
VERTICAL_ENGINES = ["news", "academic", "github"]
|
||||
|
||||
def __init__(self, timeout: float = 10.0):
|
||||
self.timeout = timeout
|
||||
self._engines: dict[str, BaseEngine] = {
|
||||
"bing": BingScraper(timeout=timeout),
|
||||
"sogou": SogouScraper(timeout=timeout),
|
||||
"baidu": BaiduScraper(timeout=timeout),
|
||||
"360": Search360Scraper(timeout=timeout),
|
||||
"news": NewsSearch(timeout=timeout),
|
||||
"academic": AcademicSearch(timeout=timeout),
|
||||
"github": GitHubEngine(timeout=timeout),
|
||||
}
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
engines: list[str] | None = None,
|
||||
max_results: int = 10,
|
||||
region: str = "cn",
|
||||
**kwargs,
|
||||
) -> tuple[list[EngineResult], SearchStats]:
|
||||
"""
|
||||
Search across specified engines in parallel.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
engines: List of engine names to use (default: all general)
|
||||
max_results: Max results per engine
|
||||
region: 'cn' or 'global'
|
||||
**kwargs: Extra params passed to engines
|
||||
|
||||
Returns:
|
||||
(results, stats)
|
||||
"""
|
||||
if engines is None:
|
||||
engines = self.GENERAL_ENGINES.copy()
|
||||
|
||||
t0 = time.time()
|
||||
stats = SearchStats(total_engines=len(engines))
|
||||
|
||||
# Build tasks
|
||||
tasks = {}
|
||||
for name in engines:
|
||||
engine = self._engines.get(name)
|
||||
if engine is None:
|
||||
logger.warning(f"Unknown engine: {name}")
|
||||
continue
|
||||
kwargs_extra = dict(kwargs)
|
||||
if name in ("bing", "bing_news"):
|
||||
kwargs_extra["region"] = region
|
||||
tasks[name] = self._run_engine(engine, query, max_results, kwargs_extra)
|
||||
|
||||
# Run all in parallel
|
||||
results_map = await asyncio.gather(
|
||||
*tasks.values(), return_exceptions=True
|
||||
)
|
||||
|
||||
all_results = []
|
||||
for name, result in zip(tasks.keys(), results_map):
|
||||
if isinstance(result, Exception):
|
||||
stats.failed_engines.append(name)
|
||||
logger.warning(f"[{name}] exception: {result}")
|
||||
# 记录异常到 execution_log
|
||||
execution_log.record(execution_log._logs[-1].__class__(
|
||||
engine=name,
|
||||
operation="search",
|
||||
success=False,
|
||||
error=str(result),
|
||||
error_type=type(result).__name__,
|
||||
duration_ms=int((time.time() - t0) * 1000),
|
||||
))
|
||||
elif isinstance(result, tuple):
|
||||
engine_results, elapsed = result
|
||||
stats.engine_timings[name] = elapsed
|
||||
if engine_results:
|
||||
stats.successful_engines += 1
|
||||
all_results.extend(engine_results)
|
||||
else:
|
||||
stats.failed_engines.append(name)
|
||||
else:
|
||||
stats.failed_engines.append(name)
|
||||
|
||||
# Deduplicate by URL
|
||||
seen_urls = set()
|
||||
deduped = []
|
||||
for r in all_results:
|
||||
key = r.url or r.title # fallback to title if no URL
|
||||
if key not in seen_urls:
|
||||
seen_urls.add(key)
|
||||
deduped.append(r)
|
||||
|
||||
stats.total_results = len(all_results)
|
||||
stats.deduplicated = len(deduped)
|
||||
stats.duration_ms = int((time.time() - t0) * 1000)
|
||||
|
||||
return deduped, stats
|
||||
|
||||
async def _run_engine(
|
||||
self, engine: BaseEngine, query: str, max_results: int, kwargs: dict
|
||||
) -> tuple[list[EngineResult], float]:
|
||||
"""Run a single engine with timing."""
|
||||
with execution_log.track(engine.name, "search", query=query) as ctx:
|
||||
t0 = time.time()
|
||||
try:
|
||||
results = await engine.search(
|
||||
query, max_results=max_results, **kwargs
|
||||
)
|
||||
elapsed = time.time() - t0
|
||||
ctx.result_count = len(results)
|
||||
return results, elapsed
|
||||
except Exception as e:
|
||||
elapsed = time.time() - t0
|
||||
ctx.success = False
|
||||
ctx.error = str(e)
|
||||
ctx.error_type = type(e).__name__
|
||||
raise
|
||||
|
||||
async def search_vertical(
|
||||
self,
|
||||
query: str,
|
||||
vertical: str,
|
||||
max_results: int = 10,
|
||||
**kwargs,
|
||||
) -> tuple[list[EngineResult], SearchStats]:
|
||||
"""Search a specific vertical (news/academic/github)."""
|
||||
if vertical == "github":
|
||||
# GitHub supports sub-types
|
||||
search_type = kwargs.pop("search_type", "repositories")
|
||||
engine = self._engines["github"]
|
||||
with execution_log.track("github", "search",
|
||||
search_type=search_type) as ctx:
|
||||
t0 = time.time()
|
||||
results = await engine.search(
|
||||
query, max_results=max_results,
|
||||
search_type=search_type, **kwargs
|
||||
)
|
||||
ctx.result_count = len(results)
|
||||
stats = SearchStats(
|
||||
total_engines=1,
|
||||
successful_engines=1 if results else 0,
|
||||
total_results=len(results),
|
||||
duration_ms=int((time.time() - t0) * 1000),
|
||||
)
|
||||
return results, stats
|
||||
else:
|
||||
return await self.search(
|
||||
query, engines=[vertical], max_results=max_results, **kwargs
|
||||
)
|
||||
|
||||
async def health(self) -> dict[str, dict]:
|
||||
"""Check health of all engines."""
|
||||
results = {}
|
||||
for name, engine in self._engines.items():
|
||||
try:
|
||||
results[name] = await engine.health()
|
||||
except Exception as e:
|
||||
results[name] = {"name": name, "status": "error", "error": str(e)}
|
||||
return results
|
||||
@@ -0,0 +1,66 @@
|
||||
"""360 Search HTML scraping engine — free, good Chinese results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import logging
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
|
||||
|
||||
class Search360Scraper(BaseEngine):
|
||||
"""360 Search scraping — good Chinese results, fast in China."""
|
||||
|
||||
name = "360"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
t0 = time.time()
|
||||
url = f"https://www.so.com/s?q={quote_plus(query)}"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
logger.info(f"[360] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[360] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
for item in soup.select("li.res-list"):
|
||||
title_el = item.select_one("h3 a")
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
href = title_el.get("href", "")
|
||||
snippet_el = item.select_one("p.res-desc, .res-rich")
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href:
|
||||
results.append(EngineResult(
|
||||
title=title, url=href, snippet=snippet, source="360",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
break
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Sogou HTML scraping engine — free, good Chinese results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import logging
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.engines.base import BaseEngine, EngineResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
|
||||
|
||||
class SogouScraper(BaseEngine):
|
||||
"""Sogou web scraping — best for Chinese content."""
|
||||
|
||||
name = "sogou"
|
||||
|
||||
async def search(self, query: str, max_results: int = 10,
|
||||
**kwargs) -> list[EngineResult]:
|
||||
t0 = time.time()
|
||||
url = f"https://www.sogou.com/web?query={quote_plus(query)}"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=HEADERS)
|
||||
resp.raise_for_status()
|
||||
elapsed = time.time() - t0
|
||||
results = self._parse(resp.text, max_results)
|
||||
logger.info(f"[sogou] {len(results)} results in {elapsed:.2f}s")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"[sogou] failed: {e}")
|
||||
return []
|
||||
|
||||
def _parse(self, html: str, max_results: int) -> list[EngineResult]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
results = []
|
||||
|
||||
for item in soup.select("div.results > div"):
|
||||
title_el = item.select_one("h3 a, a.account")
|
||||
if not title_el:
|
||||
continue
|
||||
title = title_el.get_text(strip=True)
|
||||
href = title_el.get("href", "")
|
||||
snippet_el = item.select_one("p.str_info, div.str-text, .star-wiki")
|
||||
snippet = snippet_el.get_text(strip=True) if snippet_el else ""
|
||||
|
||||
if title and href and not href.startswith("/"):
|
||||
results.append(EngineResult(
|
||||
title=title, url=href, snippet=snippet, source="sogou",
|
||||
))
|
||||
if len(results) >= max_results:
|
||||
break
|
||||
|
||||
return results
|
||||
Reference in new issue
Block a user