#!/usr/bin/env python3 """ Web Search Skills CLI — search across 20+ sources from the command line. Usage: web-search [options] Commands: search Search all sources (default) web Search web search engines (Google, Baidu, DuckDuckGo, etc.) news Search news (财联社, 华尔街见闻, RSS) wechat Search WeChat public account articles academic Search academic papers (ArXiv) social Search social media (Twitter/X) rss Search RSS news feeds sources List all available sources urls Get search URLs without executing Options: -n, --max-results N Max results per source (default: 10) -j, --json Output as JSON -o, --output FILE Save results to file -s, --source NAME Specific source name -r, --region REGION Region filter (cn/global) -v, --verbose Show debug logs -h, --help Show this help Examples: web-search wechat 蒙面财经 web-search news "人工智能" -n 5 web-search web "Python async" -r global -j web-search search "量子计算" -o results.json web-search sources web-search urls "机器学习" """ from __future__ import annotations import argparse import asyncio import json import logging import os import sys from datetime import datetime from typing import Any # Ensure scripts package is importable _SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) _PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR) if _PROJECT_ROOT not in sys.path: sys.path.insert(0, _PROJECT_ROOT) logging.basicConfig( level=logging.WARNING, format="%(levelname)s: %(message)s", ) logger = logging.getLogger("cli") # --------------------------------------------------------------------------- # Lazy engine imports (import on demand to keep startup fast) # --------------------------------------------------------------------------- def _get_unified(): from scripts.unified_search import UnifiedSearch return UnifiedSearch def _get_sogou(): from scripts.wechat.sogou_weixin import SogouWeChatEngine return SogouWeChatEngine def _get_rss(): from scripts.news.rss import RSSNewsEngine return RSSNewsEngine def _get_twitter(): from scripts.social.twitter import TwitterSearchEngine return TwitterSearchEngine def _get_ddg(): from scripts.engines.parser_engines import DuckDuckGoEngine return DuckDuckGoEngine def _get_config_loader(): from scripts.core.config_loader import ( get_search_engines, get_news_sources, get_academic_sources, get_wechat_sources, get_social_sources, get_special_engines, get_all_sources, list_source_categories, ) return { "get_search_engines": get_search_engines, "get_news_sources": get_news_sources, "get_academic_sources": get_academic_sources, "get_wechat_sources": get_wechat_sources, "get_social_sources": get_social_sources, "get_special_engines": get_special_engines, "get_all_sources": get_all_sources, "list_source_categories": list_source_categories, } # --------------------------------------------------------------------------- # Result formatting # --------------------------------------------------------------------------- def _format_result(r, fmt: str = "text") -> str: """Format a single SearchResult.""" if fmt == "json": return json.dumps({ "rank": r.rank, "title": r.title, "url": r.url, "snippet": r.snippet[:300], "source": r.source, "category": r.category, "extra": dict(r.extra) if r.extra else {}, }, ensure_ascii=False, indent=2) lines = [f"[{r.rank}] {r.title}"] lines.append(f" URL: {r.url}") if r.snippet: lines.append(f" {r.snippet[:200]}") if r.extra: extra_parts = [] for k, v in r.extra.items(): if k == "account" and v: extra_parts.append(f"公众号: {v}") elif k == "time" and v: extra_parts.append(f"时间: {v}") elif k == "search_url": continue elif v and k not in ("parsed_date", "note", "feed_url"): extra_parts.append(f"{k}: {v}") if extra_parts: lines.append(f" ({'; '.join(extra_parts)})") return "\n".join(lines) def _print_results(results, fmt: str = "text"): """Print search results.""" if not results: print(" 未找到结果") return if fmt == "json": output = json.dumps( [{ "rank": r.rank, "title": r.title, "url": r.url, "snippet": r.snippet[:300], "source": r.source, "category": r.category, "extra": dict(r.extra) if r.extra else {}, } for r in results], ensure_ascii=False, indent=2, ) print(output) else: for r in results: print(_format_result(r)) print() # --------------------------------------------------------------------------- # Async searchers # --------------------------------------------------------------------------- async def _search_all(query: str, max_results: int, **kwargs) -> list[Any]: us = _get_unified()() try: return await us.search(query, max_results=max_results, **kwargs) finally: await us.close() async def _search_web(query: str, max_results: int, region: str = None, **kwargs) -> list[Any]: us = _get_unified()() try: return await us.search_engines(query, region=region, max_results=max_results) finally: await us.close() async def _search_news(query: str, max_results: int, **kwargs) -> list[Any]: us = _get_unified()() try: return await us.search_news(query, max_results=max_results, **kwargs) finally: await us.close() async def _search_wechat(query: str, max_results: int) -> list[Any]: engine = _get_sogou()() try: return await engine.search(query, max_results=max_results) finally: await engine.close() async def _search_academic(query: str, max_results: int) -> list[Any]: us = _get_unified()() try: return await us.search_academic(query, max_results=max_results) finally: await us.close() async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]: us = _get_unified()() try: return await us.search_social(query, max_results=max_results, **kwargs) finally: await us.close() async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]: us = _get_unified()() try: return await us.search_rss(query, max_results=max_results, **kwargs) finally: await us.close() async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]: us = _get_unified()() try: return await us.get_search_urls(query, sources=sources) finally: await us.close() # --------------------------------------------------------------------------- # CLI actions # --------------------------------------------------------------------------- def cmd_sources(args): """List all available search sources.""" cl = _get_config_loader() categories = cl["list_source_categories"]() print("可用搜索源:\n") for cat_key, cat_name in [ ("search_engines", "Web 搜索引擎"), ("news_sources", "新闻源"), ("academic_sources", "学术源"), ("wechat_sources", "微信源"), ("social_sources", "社交源"), ("special_engines", "特殊源"), ]: count = categories.get(cat_key, 0) if count == 0: continue print(f" {cat_name} ({count} 个)") print(f"\n总计: {cl['get_all_sources']().__len__()} 个源") print("\n详细列表:") for s in cl["get_all_sources"](): print(f" - {s['name']:20s} ({s.get('region', '?')}) [{s.get('type', 'url')}]") def cmd_urls(args): """Get search URLs without executing.""" urls = asyncio.run(_get_urls(args.query, sources=args.source)) for name, url in urls.items(): print(f"{name}:") print(f" {url}") print() async def _run_search(args): """Route search to the right engine(s).""" if args.verbose: logging.getLogger().setLevel(logging.DEBUG) method_map = { "search": _search_all, "web": _search_web, "news": _search_news, "wechat": _search_wechat, "academic": _search_academic, "social": _search_social, "rss": _search_rss, } search_fn = method_map.get(args.command, _search_all) kwargs = {} if args.command == "web" and args.region: kwargs["region"] = args.region try: results = await search_fn( args.query, max_results=args.max_results, **kwargs, ) except Exception as e: print(f"搜索失败: {e}", file=sys.stderr) if args.verbose: import traceback traceback.print_exc() sys.exit(1) # Print header cmd_names = { "search": "全源搜索", "web": "网页搜索", "news": "新闻搜索", "wechat": "微信公众号搜索", "academic": "学术搜索", "social": "社交搜索", "rss": "RSS 新闻搜索", } header = cmd_names.get(args.command, "搜索") print(f"{header}: {args.query}") print(f"共 {len(results)} 条结果\n") _print_results(results, fmt="json" if args.json else "text") # Save to file if requested if args.output: output_data = json.dumps( [{ "query": args.query, "command": args.command, "timestamp": datetime.now().isoformat(), "total": len(results), "results": [ { "rank": r.rank, "title": r.title, "url": r.url, "snippet": r.snippet[:500], "source": r.source, "category": r.category, "extra": dict(r.extra) if r.extra else {}, } for r in results ], }], ensure_ascii=False, indent=2, ) output_path = os.path.abspath(args.output) with open(output_path, "w", encoding="utf-8") as f: f.write(output_data) print(f"\n结果已保存至: {output_path}") # --------------------------------------------------------------------------- # Argument parser # --------------------------------------------------------------------------- def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="web-search", description="Web Search Skills CLI — multi-source search tool", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__, ) parser.add_argument( "command", nargs="?", default="search", choices=["search", "web", "news", "wechat", "academic", "social", "rss", "sources", "urls"], help="Search type (default: search)", ) parser.add_argument( "query", nargs="?", default="", help="Search keyword", ) parser.add_argument( "-n", "--max-results", type=int, default=10, help="Max results per source (default: 10)", ) parser.add_argument( "-j", "--json", action="store_true", help="Output as JSON", ) parser.add_argument( "-o", "--output", type=str, help="Save results to JSON file", ) parser.add_argument( "-s", "--source", type=str, nargs="*", help="Specific source name(s)", ) parser.add_argument( "-r", "--region", type=str, choices=["cn", "global"], help="Region filter (web search only)", ) parser.add_argument( "-v", "--verbose", action="store_true", help="Show debug logs", ) return parser # --------------------------------------------------------------------------- # Entry point # --------------------------------------------------------------------------- def main(): parser = build_parser() args = parser.parse_args() if args.verbose: logging.getLogger().setLevel(logging.DEBUG) # sources and urls commands don't need a query if args.command == "sources": cmd_sources(args) return if args.command == "urls": if not args.query: print("错误: urls 命令需要提供搜索关键词", file=sys.stderr) sys.exit(1) cmd_urls(args) return # Other commands need a query if not args.query: print(f"错误: {args.command} 命令需要提供搜索关键词", file=sys.stderr) sys.exit(1) asyncio.run(_run_search(args)) if __name__ == "__main__": main()