refactor: cleanup dead code, fix model confusion, streamline CLI

- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/)
- Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py)
- scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py)
- scripts/engines/base.py: re-exports from core.base for compatibility
- scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py)
- scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py)
- scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources)
- scripts/unified_search.py: clean async Python API (was a stub)
- ws.py: lightweight launcher
- README.md + SKILL.md: updated to reflect actual 15+ engines
- All EngineResult → SearchResult
- All engines return list[SearchResult]
This commit is contained in:
openbot-coder committed 2026-05-30 16:25:41 +08:00
1 parent e20478fcbd
commit c077a7ae40
31 files changed
+1188 -3148

No files matched your search

+48 -27
View File
@@ -1,48 +1,69 @@
"""
Core data types and base classes for search engines.
Unified data models — single source of truth for all search results.
Provides:
- SearchResult: Standard result dataclass used across all engines
- SearchEngine: Abstract base class for all search engine implementations
Defines:
- SearchResult: unified result from any engine (immutable dataclass)
- BaseEngine: abstract base for all engines
- rank field: assigned by aggregator, not engines
All engine-specific models must convert to SearchResult before returning.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Optional
from typing import Any
@dataclass
class SearchResult:
"""Represents a single search result."""
title: str = ""
url: str = ""
"""Unified search result — returned by all engines."""
title: str
url: str
snippet: str = ""
source: str = ""
rank: int = 0
category: str = ""
source: str = "" # engine name (e.g. "bing", "arxiv")
category: str = "" # "web" | "news" | "academic" | "github" | "social"
rank: int = 0 # assigned by aggregator, not engine
published: str = "" # ISO date string (optional)
extra: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return {
"title": self.title,
"url": self.url,
"snippet": self.snippet,
"source": self.source,
"category": self.category,
"rank": self.rank,
"published": self.published,
"extra": self.extra,
}
class SearchEngine(ABC):
"""Abstract base class for search engines."""
def __init__(self, name: str, config: Optional[dict[str, Any]] = None):
self.name = name
self.config = config or {}
class BaseEngine(ABC):
"""Abstract base for all search engines.
Subclasses override `search()`, which returns list[SearchResult].
`health()` provides a quick connectivity check.
"""
name: str = "unknown"
timeout: float = 15.0
def __init__(self, timeout: float = 15.0):
self.timeout = timeout
@abstractmethod
async def search(
self, query: str, max_results: int = 10, **kwargs
) -> list[SearchResult]:
"""Execute a search and return results."""
async def search(self, query: str, max_results: int = 10,
**kwargs) -> list[SearchResult]:
"""Search and return unified results."""
...
def build_url(self, query: str, **kwargs) -> str:
"""Build a search URL for this engine (optional)."""
return ""
async def close(self):
"""Clean up resources."""
pass
async def health(self) -> dict[str, Any]:
"""Quick connectivity check. Returns {"status": "ok"} or {"status": "fail", "error": "..."}."""
try:
results = await self.search("test", max_results=1)
return {"name": self.name, "status": "ok", "count": len(results)}
except Exception as e:
return {"name": self.name, "status": "fail", "error": str(e)}