#!/usr/bin/env python3 """ Web Search Skills CLI — unified entry point for all search commands. Usage: web-search [options] python scripts/cli.py [options] Commands: search Full multi-source search (all ~16 engines in parallel) web Web search engines only (Bing, Sogou, Baidu, 360, DuckDuckGo, Brave) news News search (Bing News + 15 RSS feeds in parallel) wechat WeChat public account search (via Sogou) academic Academic paper search (ArXiv + CrossRef in parallel) social Social media search (Twitter/X via Nitter) rss RSS feed aggregation only baidu Baidu Qianfan API (requires BAIDU_API_KEY env) health Check all engine connectivity sources List all available sources Examples: web-search search "quantum computing" -n 10 web-search news "人工智能" -n 5 web-search academic "machine learning" web-search wechat 蒙面财经 web-search health """ from __future__ import annotations import argparse import asyncio import json import logging import os import sys import time from typing import Any from scripts.core.base import BaseEngine, SearchResult from scripts.engines.bing_scraper import BingScraper from scripts.engines.sogou_scraper import SogouScraper from scripts.engines.baidu_scraper import BaiduScraper from scripts.engines.search360_scraper import Search360Scraper from scripts.engines.parser_engines import DuckDuckGoParser, BraveParser from scripts.engines.news_engine import NewsSearch from scripts.engines.academic_engine import AcademicSearch from scripts.engines.github_engine import GitHubEngine from scripts.core.aggregator import ResultAggregator # ── Logging ────────────────────────────────────────────────────────────────── logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", datefmt="%H:%M:%S", ) logger = logging.getLogger("web-search") # ── WeChat ──────────────────────────────────────────────────────────────────── async def cmd_wechat(args: argparse.Namespace) -> None: """Search WeChat public accounts via Sogou.""" # Sogou WeChat search — simple URL-based approach (no JS rendering needed) from urllib.parse import quote_plus url = ( f"https://weixin.sogou.com/weixin?type=2&query={quote_plus(args.query)}" f"&s_from=input&_sug_=n&_sug_type_=" ) headers = { "User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/125.0.0.0 Safari/537.36"), "Accept-Language": "zh-CN,zh;q=0.9", "Referer": "https://weixin.sogou.com/", } try: import httpx async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client: resp = await client.get(url, headers=headers) resp.raise_for_status() from bs4 import BeautifulSoup soup = BeautifulSoup(resp.text, "html.parser") items = soup.select("div.txt-box") if not items: items = soup.select("div.weixin-rss-item") if not items: # Fallback: just show the search URL print(f"\n🔍 微信搜索: {args.query}") print(f" 访问: {url}") print(f" (Sogou 反爬限制,无法直接解析内容)") return results = [] for item in items[: args.max_results]: title_el = item.select_one("h3") or item.select_one("a.title") title = title_el.get_text(strip=True) if title_el else "" link_el = item.select_one("a") or item.select_one("h3 a") link = link_el.get("href", "") if link_el else "" if link and not link.startswith("http"): link = "https://weixin.sogou.com" + link account_el = item.select_one(".account") account = account_el.get_text(strip=True) if account_el else "" snippet_el = item.select_one("p") snippet = snippet_el.get_text(strip=True)[:200] if snippet_el else "" if title: results.append( f" [{len(results)+1}] {title}\n" f" 公众号: {account}\n" f" {link}" f"{' | ' + snippet if snippet else ''}" ) print(f"\n🔍 微信搜索: {args.query} ({len(results)} 条结果)\n") for r in results: print(r) except Exception as e: print(f"\n🔍 微信搜索: {args.query}") print(f" Sogou 反爬限制,访问: {url}") print(f" (错误: {e})") # ── Social ──────────────────────────────────────────────────────────────────── async def cmd_social(args: argparse.Namespace) -> None: """Search Twitter/X via Nitter instances (no auth needed).""" import httpx from urllib.parse import quote_plus # Try multiple Nitter instances nitter_instances = [ "https://nitter.privacydev.net", "https://nitter.poast.org", "https://xcancel.com", ] query_encoded = quote_plus(args.query) results = [] found = False for instance in nitter_instances: url = f"{instance}/search?f=tweets&q={query_encoded}&since=&until=&near=" headers = { "User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36"), } try: async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client: resp = await client.get(url, headers=headers) if resp.status_code != 200: continue from bs4 import BeautifulSoup soup = BeautifulSoup(resp.text, "html.parser") items = soup.select("div.timeline-item")[: args.max_results] if not items: continue found = True print(f"\n🐦 Twitter/X 搜索: {args.query} (via {instance})\n") for item in items: text_el = item.select_one(".tweet-content") author_el = item.select_one(".username") time_el = item.select_one(".tweet-date") link_el = item.select_one("a") text = text_el.get_text(strip=True) if text_el else "" author = author_el.get_text(strip=True) if author_el else "" tweet_time = time_el.get("title", "") if time_el else "" link = link_el.get("href", "") if link_el else "" if link and not link.startswith("http"): link = instance + link if text: print(f" @{author} · {tweet_time}") print(f" {text[:200]}") if link: print(f" 🔗 {link}") print() break except Exception: continue if not found: print(f"\n🐦 Twitter/X 搜索: {args.query}") print(f" Nitter 实例均不可用,请直接访问 twitter.com") print(f" 查询: https://twitter.com/search?q={query_encoded}") # ── RSS ──────────────────────────────────────────────────────────────────────── async def cmd_rss(args: argparse.Namespace) -> None: """RSS feed aggregation — AI newsletters, tech blogs, podcasts.""" from scripts.engines.news_engine import RSSNewsEngine engine = RSSNewsEngine(timeout=10.0) results = await engine.search(args.query, max_results=args.max_results) await _print_results(args, results, f"RSS 订阅源: {args.query}") # ── Baidu Qianfan ───────────────────────────────────────────────────────────── async def cmd_baidu(args: argparse.Namespace) -> None: """Baidu Qianfan AI Search API (requires BAIDU_API_KEY env var).""" api_key = os.environ.get("BAIDU_API_KEY") or os.environ.get("BAIDU_QIANFAN_API_KEY") if not api_key: print("❌ 需要设置 BAIDU_API_KEY 环境变量") print(" export BAIDU_API_KEY=your_key_here") return from scripts.engines.baidu_qianfan import BaiduQianfanEngine engine = BaiduQianfanEngine(timeout=15.0) results = await engine.search(args.query, max_results=args.max_results) await _print_results(args, results, f"Baidu Qianfan: {args.query}") # ── Health ──────────────────────────────────────────────────────────────────── async def cmd_health(args: argparse.Namespace) -> None: """Check connectivity of all engines concurrently.""" from scripts.engines.parallel_search import ParallelSearch ps = ParallelSearch(timeout=10.0) status = await ps.health() if args.json: print(json.dumps(status, ensure_ascii=False, indent=2)) return print("\n🔧 引擎健康检测\n") print(f" {'引擎':<20} {'状态':<8} {'详情'}") print(" " + "─" * 50) ok_count = sum(1 for v in status.values() if v.get("status") == "ok") for name, result in sorted(status.items()): s = result.get("status", "?") icon = "✅" if s == "ok" else "❌" detail = result.get("error", result.get("count", "")) print(f" {icon} {name:<18} {s:<8} {detail}") print(f"\n ✅ 可用: {ok_count}/{len(status)}") # ── Sources ─────────────────────────────────────────────────────────────────── async def cmd_sources(args: argparse.Namespace) -> None: """List all available search sources.""" sources = { "🌐 Web (网页)": [ "Bing CN (cn.bing.com)", "Bing INT (www.bing.com)", "Sogou (sogou.com)", "Baidu (baidu.com)", "360 (so.com)", "DuckDuckGo (html.duckduckgo.com)", "Brave (search.brave.com)", "Google (www.google.com, via serpapi optional)", ], "📰 News (新闻)": [ "Bing News (cn.bing.com/news)", "36氪 RSS", "IT之家 RSS", "钛媒体 RSS", "Solidot RSS", "Hacker News RSS", "TechCrunch RSS", "MIT Tech Review RSS", "Wired RSS", "The Verge RSS", ], "🎓 Academic (学术)": [ "ArXiv API (export.arxiv.org/api)", "CrossRef API (api.crossref.org)", ], "💬 Social (社交)": [ "Nitter instances (Twitter/X 免登录)", ], "🔧 API (需配置密钥)": [ "Baidu Qianfan (BAIDU_API_KEY)", ], "📱 垂直搜索": [ "微信公众号 (weixin.sogou.com)", ], } for category, items in sources.items(): print(f"\n{category}") for item in items: print(f" • {item}") # ── Shared helpers ──────────────────────────────────────────────────────────── def _build_engines(timeout: float = 10.0) -> dict[str, BaseEngine]: return { "bing": BingScraper(timeout=timeout), "sogou": SogouScraper(timeout=timeout), "baidu": BaiduScraper(timeout=timeout), "360": Search360Scraper(timeout=timeout), "duckduckgo": DuckDuckGoParser(timeout=timeout), "brave": BraveParser(timeout=timeout), } def _build_vertical_engines(timeout: float = 10.0) -> dict[str, BaseEngine]: return { "news": NewsSearch(timeout=timeout), "academic": AcademicSearch(timeout=timeout), "github": GitHubEngine(timeout=timeout), } async def _run_engines( engines: dict[str, BaseEngine], query: str, max_results: int, region: str = "cn", ) -> list[SearchResult]: """Run engines in parallel with asyncio.gather.""" import asyncio kwargs = {"region": region} if region else {} async def run_one(name: str, eng: BaseEngine) -> tuple[str, list[SearchResult]]: try: results = await asyncio.wait_for( eng.search(query, max_results=max_results, **kwargs), timeout=eng.timeout + 2.0, ) return name, list(results) except asyncio.TimeoutError: logger.warning(f"[{name}] timeout after {eng.timeout}s") return name, [] except Exception as e: logger.warning(f"[{name}] failed: {e}") return name, [] tasks = [run_one(n, e) for n, e in engines.items()] done = await asyncio.gather(*tasks, return_exceptions=True) all_results = [] for item in done: if isinstance(item, tuple): _, results = item all_results.extend(results) elif isinstance(item, Exception): logger.warning(f"engine task exception: {item}") return all_results def _format_result(r: SearchResult, index: int) -> str: extra_parts = [] if r.published: extra_parts.append(r.published) if r.source: extra_parts.append(r.source) if r.extra: keys = list(r.extra.keys())[:2] extra_parts.extend(str(r.extra[k])[:30] for k in keys if r.extra.get(k)) header = f" {index}. [{r.source or r.category}] {r.title}" url_line = f" {r.url}" snippet_line = f" {r.snippet[:120]}..." if len(r.snippet) > 120 else f" {r.snippet}" return "\n".join(filter(None, [header, url_line, snippet_line])) async def _print_results( args: argparse.Namespace, results: list[SearchResult], title: str, ) -> None: if not results: print(f"\n📭 未找到「{args.query}」的相关结果") return if args.json: output = { "query": args.query, "total": len(results), "results": [ { "rank": i + 1, "title": r.title, "url": r.url, "snippet": r.snippet, "source": r.source, "category": r.category, "published": r.published, } for i, r in enumerate(results) ], } print(json.dumps(output, ensure_ascii=False, indent=2)) return # Rank results for i, r in enumerate(results): r.rank = i + 1 print(f"\n🔍 「{args.query}」— {len(results)} 条结果\n") for i, r in enumerate(results): print(_format_result(r, i + 1)) print() # ── Main commands ───────────────────────────────────────────────────────────── async def cmd_search(args: argparse.Namespace) -> None: """Full search across all engines (~16 in parallel).""" t0 = time.time() all_results: list[SearchResult] = [] if args.sources: # Run specific sources only source_map = {**_build_engines(), **_build_vertical_engines()} selected = {k: v for k, v in source_map.items() if k in args.sources} if not selected: print("❌ 未识别任何指定的源") return all_results = await _run_engines(selected, args.query, args.max_results, args.region) elif args.category: if args.category == "news": engs = {"news": _build_vertical_engines()["news"]} elif args.category == "academic": engs = {"academic": _build_vertical_engines()["academic"]} elif args.category == "social": print("\n🐦 请使用: web-search social ") return elif args.category == "rss": print("\n📡 请使用: web-search rss ") return else: engs = _build_engines() all_results = await _run_engines(engs, args.query, args.max_results, args.region) else: # Run all engines concurrently web_engs = _build_engines() vert_engs = _build_vertical_engines() all_web = await _run_engines(web_engs, args.query, args.max_results, args.region) all_vert = await _run_engines(vert_engs, args.query, args.max_results, args.region) all_results = all_web + all_vert elapsed = time.time() - t0 # Deduplicate and rank aggregator = ResultAggregator() merged = aggregator.merge_and_rank(all_results, max_results=args.max_results) if args.json: await _print_results(args, merged, "") else: print(f"\n🔍 「{args.query}」— {len(merged)} 条结果 (耗时 {elapsed:.1f}s)\n") for i, r in enumerate(merged): print(_format_result(r, i + 1)) print() async def cmd_web(args: argparse.Namespace) -> None: """Web search engines only.""" t0 = time.time() engines = _build_engines() results = await _run_engines(engines, args.query, args.max_results, args.region) elapsed = time.time() - t0 aggregator = ResultAggregator() merged = aggregator.merge_and_rank(results, max_results=args.max_results) if args.json: await _print_results(args, merged, "") else: print(f"\n🌐 「{args.query}」— {len(merged)} 条结果 (耗时 {elapsed:.1f}s)\n") for i, r in enumerate(merged): print(_format_result(r, i + 1)) print() async def cmd_news(args: argparse.Namespace) -> None: """News search — Bing News + RSS.""" from scripts.engines.news_engine import NewsSearch engine = NewsSearch(timeout=15.0) results = await engine.search(args.query, max_results=args.max_results, region=args.region) await _print_results(args, results, "📰 新闻") async def cmd_academic(args: argparse.Namespace) -> None: """Academic search — ArXiv + CrossRef.""" from scripts.engines.academic_engine import AcademicSearch engine = AcademicSearch(timeout=15.0) results = await engine.search(args.query, max_results=args.max_results) await _print_results(args, results, "🎓 学术论文") # ── CLI entry point ─────────────────────────────────────────────────────────── def main() -> None: parser = argparse.ArgumentParser( prog="web-search", description="Multi-source web search CLI (28+ engines)", ) sub = parser.add_subparsers(dest="command", required=True) # ── search ────────────────────────────────────────────────────────────── sp = sub.add_parser("search", help="Search all engines (default: all ~16 in parallel)") sp.add_argument("query", help="Search query") sp.add_argument("-n", "--max-results", type=int, default=10, help="Max results per engine") sp.add_argument("-s", "--sources", nargs="+", metavar="SRC", help="Run specific sources only (e.g. bing sogou news)") sp.add_argument("-c", "--category", choices=["news", "academic", "social", "rss", "web"], help="Run engines of a specific category") sp.add_argument("-r", "--region", default="cn", choices=["cn", "global"], help="Region: cn (default) or global") sp.add_argument("-j", "--json", action="store_true", help="JSON output") sp.add_argument("-v", "--verbose", action="store_true", help="Debug logging") # ── web ───────────────────────────────────────────────────────────────── wp = sub.add_parser("web", help="Web search engines only") wp.add_argument("query", help="Search query") wp.add_argument("-n", "--max-results", type=int, default=10) wp.add_argument("-r", "--region", default="cn", choices=["cn", "global"]) wp.add_argument("-j", "--json", action="store_true") wp.add_argument("-v", "--verbose", action="store_true") # ── news ──────────────────────────────────────────────────────────────── np_ = sub.add_parser("news", help="News search (Bing News + RSS)") np_.add_argument("query", help="Search query") np_.add_argument("-n", "--max-results", type=int, default=10) np_.add_argument("-r", "--region", default="cn", choices=["cn", "global"]) np_.add_argument("-j", "--json", action="store_true") # ── academic ───────────────────────────────────────────────────────────── ap = sub.add_parser("academic", help="Academic papers (ArXiv + CrossRef)") ap.add_argument("query", help="Search query") ap.add_argument("-n", "--max-results", type=int, default=10) ap.add_argument("-j", "--json", action="store_true") # ── wechat ────────────────────────────────────────────────────────────── wcp = sub.add_parser("wechat", help="WeChat public account search (Sogou)") wcp.add_argument("query", help="Search query") wcp.add_argument("-n", "--max-results", type=int, default=5) # ── social ─────────────────────────────────────────────────────────────── sop = sub.add_parser("social", help="Social media (Twitter/X via Nitter)") sop.add_argument("query", help="Search query") sop.add_argument("-n", "--max-results", type=int, default=10) # ── rss ────────────────────────────────────────────────────────────────── rp = sub.add_parser("rss", help="RSS feed aggregation") rp.add_argument("query", help="Search query") rp.add_argument("-n", "--max-results", type=int, default=10) rp.add_argument("-j", "--json", action="store_true") # ── baidu ──────────────────────────────────────────────────────────────── bp = sub.add_parser("baidu", help="Baidu Qianfan API (requires BAIDU_API_KEY)") bp.add_argument("query", help="Search query") bp.add_argument("-n", "--max-results", type=int, default=10) bp.add_argument("-j", "--json", action="store_true") # ── health ─────────────────────────────────────────────────────────────── hp = sub.add_parser("health", help="Check all engine connectivity") hp.add_argument("-j", "--json", action="store_true") # ── sources ────────────────────────────────────────────────────────────── sub.add_parser("sources", help="List all available search sources") args = parser.parse_args() # Verbose logging if getattr(args, "verbose", False): logging.getLogger("web-search").setLevel(logging.DEBUG) # Run the appropriate command try: if args.command == "search": asyncio.run(cmd_search(args)) elif args.command == "web": asyncio.run(cmd_web(args)) elif args.command == "news": asyncio.run(cmd_news(args)) elif args.command == "academic": asyncio.run(cmd_academic(args)) elif args.command == "wechat": asyncio.run(cmd_wechat(args)) elif args.command == "social": asyncio.run(cmd_social(args)) elif args.command == "rss": asyncio.run(cmd_rss(args)) elif args.command == "baidu": asyncio.run(cmd_baidu(args)) elif args.command == "health": asyncio.run(cmd_health(args)) elif args.command == "sources": asyncio.run(cmd_sources(args)) except KeyboardInterrupt: print("\n\n⏹ 已取消") sys.exit(130) if __name__ == "__main__": main()