Files
openbot-coder c077a7ae40 refactor: cleanup dead code, fix model confusion, streamline CLI
- Remove 3 dead modules: academic/, wechat/, social/ (merged into engines/)
- Remove 4 stale news scripts (tech_news.py, wallstreetcn.py, cls.py, rss.py)
- scripts/core/base.py: canonical SearchResult + BaseEngine (was in engines/base.py)
- scripts/engines/base.py: re-exports from core.base for compatibility
- scripts/engines/news_engine.py: Bing News + RSS unified (was in scripts/news/rss.py)
- scripts/engines/academic_engine.py: ArXiv + CrossRef unified (was in scripts/academic/arxiv.py)
- scripts/cli.py: complete rewrite — 10 commands (search/web/news/academic/wechat/social/rss/baidu/health/sources)
- scripts/unified_search.py: clean async Python API (was a stub)
- ws.py: lightweight launcher
- README.md + SKILL.md: updated to reflect actual 15+ engines
- All EngineResult → SearchResult
- All engines return list[SearchResult]
2026-05-30 16:25:41 +08:00

607 lines
25 KiB
Python

#!/usr/bin/env python3
"""
Web Search Skills CLI — unified entry point for all search commands.
Usage:
web-search <command> [options]
python scripts/cli.py <command> [options]
Commands:
search Full multi-source search (all ~16 engines in parallel)
web Web search engines only (Bing, Sogou, Baidu, 360, DuckDuckGo, Brave)
news News search (Bing News + 15 RSS feeds in parallel)
wechat WeChat public account search (via Sogou)
academic Academic paper search (ArXiv + CrossRef in parallel)
social Social media search (Twitter/X via Nitter)
rss RSS feed aggregation only
baidu Baidu Qianfan API (requires BAIDU_API_KEY env)
health Check all engine connectivity
sources List all available sources
Examples:
web-search search "quantum computing" -n 10
web-search news "人工智能" -n 5
web-search academic "machine learning"
web-search wechat 蒙面财经
web-search health
"""
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import os
import sys
import time
from typing import Any
from scripts.core.base import BaseEngine, SearchResult
from scripts.engines.bing_scraper import BingScraper
from scripts.engines.sogou_scraper import SogouScraper
from scripts.engines.baidu_scraper import BaiduScraper
from scripts.engines.search360_scraper import Search360Scraper
from scripts.engines.parser_engines import DuckDuckGoParser, BraveParser
from scripts.engines.news_engine import NewsSearch
from scripts.engines.academic_engine import AcademicSearch
from scripts.engines.github_engine import GitHubEngine
from scripts.core.aggregator import ResultAggregator
# ── Logging ──────────────────────────────────────────────────────────────────
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
datefmt="%H:%M:%S",
)
logger = logging.getLogger("web-search")
# ── WeChat ────────────────────────────────────────────────────────────────────
async def cmd_wechat(args: argparse.Namespace) -> None:
"""Search WeChat public accounts via Sogou."""
# Sogou WeChat search — simple URL-based approach (no JS rendering needed)
from urllib.parse import quote_plus
url = (
f"https://weixin.sogou.com/weixin?type=2&query={quote_plus(args.query)}"
f"&s_from=input&_sug_=n&_sug_type_="
)
headers = {
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/125.0.0.0 Safari/537.36"),
"Accept-Language": "zh-CN,zh;q=0.9",
"Referer": "https://weixin.sogou.com/",
}
try:
import httpx
async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
resp = await client.get(url, headers=headers)
resp.raise_for_status()
from bs4 import BeautifulSoup
soup = BeautifulSoup(resp.text, "html.parser")
items = soup.select("div.txt-box")
if not items:
items = soup.select("div.weixin-rss-item")
if not items:
# Fallback: just show the search URL
print(f"\n🔍 微信搜索: {args.query}")
print(f" 访问: {url}")
print(f" (Sogou 反爬限制,无法直接解析内容)")
return
results = []
for item in items[: args.max_results]:
title_el = item.select_one("h3") or item.select_one("a.title")
title = title_el.get_text(strip=True) if title_el else ""
link_el = item.select_one("a") or item.select_one("h3 a")
link = link_el.get("href", "") if link_el else ""
if link and not link.startswith("http"):
link = "https://weixin.sogou.com" + link
account_el = item.select_one(".account")
account = account_el.get_text(strip=True) if account_el else ""
snippet_el = item.select_one("p")
snippet = snippet_el.get_text(strip=True)[:200] if snippet_el else ""
if title:
results.append(
f" [{len(results)+1}] {title}\n"
f" 公众号: {account}\n"
f" {link}"
f"{' | ' + snippet if snippet else ''}"
)
print(f"\n🔍 微信搜索: {args.query} ({len(results)} 条结果)\n")
for r in results:
print(r)
except Exception as e:
print(f"\n🔍 微信搜索: {args.query}")
print(f" Sogou 反爬限制,访问: {url}")
print(f" (错误: {e})")
# ── Social ────────────────────────────────────────────────────────────────────
async def cmd_social(args: argparse.Namespace) -> None:
"""Search Twitter/X via Nitter instances (no auth needed)."""
import httpx
from urllib.parse import quote_plus
# Try multiple Nitter instances
nitter_instances = [
"https://nitter.privacydev.net",
"https://nitter.poast.org",
"https://xcancel.com",
]
query_encoded = quote_plus(args.query)
results = []
found = False
for instance in nitter_instances:
url = f"{instance}/search?f=tweets&q={query_encoded}&since=&until=&near="
headers = {
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36"),
}
try:
async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
resp = await client.get(url, headers=headers)
if resp.status_code != 200:
continue
from bs4 import BeautifulSoup
soup = BeautifulSoup(resp.text, "html.parser")
items = soup.select("div.timeline-item")[: args.max_results]
if not items:
continue
found = True
print(f"\n🐦 Twitter/X 搜索: {args.query} (via {instance})\n")
for item in items:
text_el = item.select_one(".tweet-content")
author_el = item.select_one(".username")
time_el = item.select_one(".tweet-date")
link_el = item.select_one("a")
text = text_el.get_text(strip=True) if text_el else ""
author = author_el.get_text(strip=True) if author_el else ""
tweet_time = time_el.get("title", "") if time_el else ""
link = link_el.get("href", "") if link_el else ""
if link and not link.startswith("http"):
link = instance + link
if text:
print(f" @{author} · {tweet_time}")
print(f" {text[:200]}")
if link:
print(f" 🔗 {link}")
print()
break
except Exception:
continue
if not found:
print(f"\n🐦 Twitter/X 搜索: {args.query}")
print(f" Nitter 实例均不可用,请直接访问 twitter.com")
print(f" 查询: https://twitter.com/search?q={query_encoded}")
# ── RSS ────────────────────────────────────────────────────────────────────────
async def cmd_rss(args: argparse.Namespace) -> None:
"""RSS feed aggregation — AI newsletters, tech blogs, podcasts."""
from scripts.engines.news_engine import RSSNewsEngine
engine = RSSNewsEngine(timeout=10.0)
results = await engine.search(args.query, max_results=args.max_results)
await _print_results(args, results, f"RSS 订阅源: {args.query}")
# ── Baidu Qianfan ─────────────────────────────────────────────────────────────
async def cmd_baidu(args: argparse.Namespace) -> None:
"""Baidu Qianfan AI Search API (requires BAIDU_API_KEY env var)."""
api_key = os.environ.get("BAIDU_API_KEY") or os.environ.get("BAIDU_QIANFAN_API_KEY")
if not api_key:
print("❌ 需要设置 BAIDU_API_KEY 环境变量")
print(" export BAIDU_API_KEY=your_key_here")
return
from scripts.engines.baidu_qianfan import BaiduQianfanEngine
engine = BaiduQianfanEngine(timeout=15.0)
results = await engine.search(args.query, max_results=args.max_results)
await _print_results(args, results, f"Baidu Qianfan: {args.query}")
# ── Health ────────────────────────────────────────────────────────────────────
async def cmd_health(args: argparse.Namespace) -> None:
"""Check connectivity of all engines concurrently."""
from scripts.engines.parallel_search import ParallelSearch
ps = ParallelSearch(timeout=10.0)
status = await ps.health()
if args.json:
print(json.dumps(status, ensure_ascii=False, indent=2))
return
print("\n🔧 引擎健康检测\n")
print(f" {'引擎':<20} {'状态':<8} {'详情'}")
print(" " + "─" * 50)
ok_count = sum(1 for v in status.values() if v.get("status") == "ok")
for name, result in sorted(status.items()):
s = result.get("status", "?")
icon = "✅" if s == "ok" else "❌"
detail = result.get("error", result.get("count", ""))
print(f" {icon} {name:<18} {s:<8} {detail}")
print(f"\n ✅ 可用: {ok_count}/{len(status)}")
# ── Sources ───────────────────────────────────────────────────────────────────
async def cmd_sources(args: argparse.Namespace) -> None:
"""List all available search sources."""
sources = {
"🌐 Web (网页)": [
"Bing CN (cn.bing.com)",
"Bing INT (www.bing.com)",
"Sogou (sogou.com)",
"Baidu (baidu.com)",
"360 (so.com)",
"DuckDuckGo (html.duckduckgo.com)",
"Brave (search.brave.com)",
"Google (www.google.com, via serpapi optional)",
],
"📰 News (新闻)": [
"Bing News (cn.bing.com/news)",
"36氪 RSS",
"IT之家 RSS",
"钛媒体 RSS",
"Solidot RSS",
"Hacker News RSS",
"TechCrunch RSS",
"MIT Tech Review RSS",
"Wired RSS",
"The Verge RSS",
],
"🎓 Academic (学术)": [
"ArXiv API (export.arxiv.org/api)",
"CrossRef API (api.crossref.org)",
],
"💬 Social (社交)": [
"Nitter instances (Twitter/X 免登录)",
],
"🔧 API (需配置密钥)": [
"Baidu Qianfan (BAIDU_API_KEY)",
],
"📱 垂直搜索": [
"微信公众号 (weixin.sogou.com)",
],
}
for category, items in sources.items():
print(f"\n{category}")
for item in items:
print(f" • {item}")
# ── Shared helpers ────────────────────────────────────────────────────────────
def _build_engines(timeout: float = 10.0) -> dict[str, BaseEngine]:
return {
"bing": BingScraper(timeout=timeout),
"sogou": SogouScraper(timeout=timeout),
"baidu": BaiduScraper(timeout=timeout),
"360": Search360Scraper(timeout=timeout),
"duckduckgo": DuckDuckGoParser(timeout=timeout),
"brave": BraveParser(timeout=timeout),
}
def _build_vertical_engines(timeout: float = 10.0) -> dict[str, BaseEngine]:
return {
"news": NewsSearch(timeout=timeout),
"academic": AcademicSearch(timeout=timeout),
"github": GitHubEngine(timeout=timeout),
}
async def _run_engines(
engines: dict[str, BaseEngine],
query: str,
max_results: int,
region: str = "cn",
) -> list[SearchResult]:
"""Run engines in parallel with asyncio.gather."""
import asyncio
kwargs = {"region": region} if region else {}
async def run_one(name: str, eng: BaseEngine) -> tuple[str, list[SearchResult]]:
try:
results = await asyncio.wait_for(
eng.search(query, max_results=max_results, **kwargs),
timeout=eng.timeout + 2.0,
)
return name, list(results)
except asyncio.TimeoutError:
logger.warning(f"[{name}] timeout after {eng.timeout}s")
return name, []
except Exception as e:
logger.warning(f"[{name}] failed: {e}")
return name, []
tasks = [run_one(n, e) for n, e in engines.items()]
done = await asyncio.gather(*tasks, return_exceptions=True)
all_results = []
for item in done:
if isinstance(item, tuple):
_, results = item
all_results.extend(results)
elif isinstance(item, Exception):
logger.warning(f"engine task exception: {item}")
return all_results
def _format_result(r: SearchResult, index: int) -> str:
extra_parts = []
if r.published:
extra_parts.append(r.published)
if r.source:
extra_parts.append(r.source)
if r.extra:
keys = list(r.extra.keys())[:2]
extra_parts.extend(str(r.extra[k])[:30] for k in keys if r.extra.get(k))
header = f" {index}. [{r.source or r.category}] {r.title}"
url_line = f" {r.url}"
snippet_line = f" {r.snippet[:120]}..." if len(r.snippet) > 120 else f" {r.snippet}"
return "\n".join(filter(None, [header, url_line, snippet_line]))
async def _print_results(
args: argparse.Namespace,
results: list[SearchResult],
title: str,
) -> None:
if not results:
print(f"\n📭 未找到「{args.query}」的相关结果")
return
if args.json:
output = {
"query": args.query,
"total": len(results),
"results": [
{
"rank": i + 1,
"title": r.title,
"url": r.url,
"snippet": r.snippet,
"source": r.source,
"category": r.category,
"published": r.published,
}
for i, r in enumerate(results)
],
}
print(json.dumps(output, ensure_ascii=False, indent=2))
return
# Rank results
for i, r in enumerate(results):
r.rank = i + 1
print(f"\n🔍 「{args.query}」— {len(results)} 条结果\n")
for i, r in enumerate(results):
print(_format_result(r, i + 1))
print()
# ── Main commands ─────────────────────────────────────────────────────────────
async def cmd_search(args: argparse.Namespace) -> None:
"""Full search across all engines (~16 in parallel)."""
t0 = time.time()
all_results: list[SearchResult] = []
if args.sources:
# Run specific sources only
source_map = {**_build_engines(), **_build_vertical_engines()}
selected = {k: v for k, v in source_map.items() if k in args.sources}
if not selected:
print("❌ 未识别任何指定的源")
return
all_results = await _run_engines(selected, args.query, args.max_results, args.region)
elif args.category:
if args.category == "news":
engs = {"news": _build_vertical_engines()["news"]}
elif args.category == "academic":
engs = {"academic": _build_vertical_engines()["academic"]}
elif args.category == "social":
print("\n🐦 请使用: web-search social <query>")
return
elif args.category == "rss":
print("\n📡 请使用: web-search rss <query>")
return
else:
engs = _build_engines()
all_results = await _run_engines(engs, args.query, args.max_results, args.region)
else:
# Run all engines concurrently
web_engs = _build_engines()
vert_engs = _build_vertical_engines()
all_web = await _run_engines(web_engs, args.query, args.max_results, args.region)
all_vert = await _run_engines(vert_engs, args.query, args.max_results, args.region)
all_results = all_web + all_vert
elapsed = time.time() - t0
# Deduplicate and rank
aggregator = ResultAggregator()
merged = aggregator.merge_and_rank(all_results, max_results=args.max_results)
if args.json:
await _print_results(args, merged, "")
else:
print(f"\n🔍 「{args.query}」— {len(merged)} 条结果 (耗时 {elapsed:.1f}s)\n")
for i, r in enumerate(merged):
print(_format_result(r, i + 1))
print()
async def cmd_web(args: argparse.Namespace) -> None:
"""Web search engines only."""
t0 = time.time()
engines = _build_engines()
results = await _run_engines(engines, args.query, args.max_results, args.region)
elapsed = time.time() - t0
aggregator = ResultAggregator()
merged = aggregator.merge_and_rank(results, max_results=args.max_results)
if args.json:
await _print_results(args, merged, "")
else:
print(f"\n🌐 「{args.query}」— {len(merged)} 条结果 (耗时 {elapsed:.1f}s)\n")
for i, r in enumerate(merged):
print(_format_result(r, i + 1))
print()
async def cmd_news(args: argparse.Namespace) -> None:
"""News search — Bing News + RSS."""
from scripts.engines.news_engine import NewsSearch
engine = NewsSearch(timeout=15.0)
results = await engine.search(args.query, max_results=args.max_results, region=args.region)
await _print_results(args, results, "📰 新闻")
async def cmd_academic(args: argparse.Namespace) -> None:
"""Academic search — ArXiv + CrossRef."""
from scripts.engines.academic_engine import AcademicSearch
engine = AcademicSearch(timeout=15.0)
results = await engine.search(args.query, max_results=args.max_results)
await _print_results(args, results, "🎓 学术论文")
# ── CLI entry point ───────────────────────────────────────────────────────────
def main() -> None:
parser = argparse.ArgumentParser(
prog="web-search",
description="Multi-source web search CLI (28+ engines)",
)
sub = parser.add_subparsers(dest="command", required=True)
# ── search ──────────────────────────────────────────────────────────────
sp = sub.add_parser("search", help="Search all engines (default: all ~16 in parallel)")
sp.add_argument("query", help="Search query")
sp.add_argument("-n", "--max-results", type=int, default=10, help="Max results per engine")
sp.add_argument("-s", "--sources", nargs="+", metavar="SRC",
help="Run specific sources only (e.g. bing sogou news)")
sp.add_argument("-c", "--category", choices=["news", "academic", "social", "rss", "web"],
help="Run engines of a specific category")
sp.add_argument("-r", "--region", default="cn", choices=["cn", "global"],
help="Region: cn (default) or global")
sp.add_argument("-j", "--json", action="store_true", help="JSON output")
sp.add_argument("-v", "--verbose", action="store_true", help="Debug logging")
# ── web ─────────────────────────────────────────────────────────────────
wp = sub.add_parser("web", help="Web search engines only")
wp.add_argument("query", help="Search query")
wp.add_argument("-n", "--max-results", type=int, default=10)
wp.add_argument("-r", "--region", default="cn", choices=["cn", "global"])
wp.add_argument("-j", "--json", action="store_true")
wp.add_argument("-v", "--verbose", action="store_true")
# ── news ────────────────────────────────────────────────────────────────
np_ = sub.add_parser("news", help="News search (Bing News + RSS)")
np_.add_argument("query", help="Search query")
np_.add_argument("-n", "--max-results", type=int, default=10)
np_.add_argument("-r", "--region", default="cn", choices=["cn", "global"])
np_.add_argument("-j", "--json", action="store_true")
# ── academic ─────────────────────────────────────────────────────────────
ap = sub.add_parser("academic", help="Academic papers (ArXiv + CrossRef)")
ap.add_argument("query", help="Search query")
ap.add_argument("-n", "--max-results", type=int, default=10)
ap.add_argument("-j", "--json", action="store_true")
# ── wechat ──────────────────────────────────────────────────────────────
wcp = sub.add_parser("wechat", help="WeChat public account search (Sogou)")
wcp.add_argument("query", help="Search query")
wcp.add_argument("-n", "--max-results", type=int, default=5)
# ── social ───────────────────────────────────────────────────────────────
sop = sub.add_parser("social", help="Social media (Twitter/X via Nitter)")
sop.add_argument("query", help="Search query")
sop.add_argument("-n", "--max-results", type=int, default=10)
# ── rss ──────────────────────────────────────────────────────────────────
rp = sub.add_parser("rss", help="RSS feed aggregation")
rp.add_argument("query", help="Search query")
rp.add_argument("-n", "--max-results", type=int, default=10)
rp.add_argument("-j", "--json", action="store_true")
# ── baidu ────────────────────────────────────────────────────────────────
bp = sub.add_parser("baidu", help="Baidu Qianfan API (requires BAIDU_API_KEY)")
bp.add_argument("query", help="Search query")
bp.add_argument("-n", "--max-results", type=int, default=10)
bp.add_argument("-j", "--json", action="store_true")
# ── health ───────────────────────────────────────────────────────────────
hp = sub.add_parser("health", help="Check all engine connectivity")
hp.add_argument("-j", "--json", action="store_true")
# ── sources ──────────────────────────────────────────────────────────────
sub.add_parser("sources", help="List all available search sources")
args = parser.parse_args()
# Verbose logging
if getattr(args, "verbose", False):
logging.getLogger("web-search").setLevel(logging.DEBUG)
# Run the appropriate command
try:
if args.command == "search":
asyncio.run(cmd_search(args))
elif args.command == "web":
asyncio.run(cmd_web(args))
elif args.command == "news":
asyncio.run(cmd_news(args))
elif args.command == "academic":
asyncio.run(cmd_academic(args))
elif args.command == "wechat":
asyncio.run(cmd_wechat(args))
elif args.command == "social":
asyncio.run(cmd_social(args))
elif args.command == "rss":
asyncio.run(cmd_rss(args))
elif args.command == "baidu":
asyncio.run(cmd_baidu(args))
elif args.command == "health":
asyncio.run(cmd_health(args))
elif args.command == "sources":
asyncio.run(cmd_sources(args))
except KeyboardInterrupt:
print("\n\n⏹ 已取消")
sys.exit(130)
if __name__ == "__main__":
main()