Files
web-search-skills/scripts/cli.py
T
vex1023 87935b59ce refactor: restructure project with CLI tool, multi-engine support, and skill-builder compliant SKILL.md
- Replace setup.py with pyproject.toml (uv-compatible)
- Add CLI entry point (web-search) with 9 subcommands
- Add multi-engine architecture (14 web + 7 special sources)
- Add WeChat/CLS/WallStreetCN/ArXiv/Twitter search engines
- Remove old src/ and examples/ directories
- Remove redundant requirements.txt and setup.py
- Update README.md and SKILL.md with full documentation
- SKILL.md now compliant with skill-builder standards (frontmatter, triggers, structure)
2026-05-14 16:31:54 +08:00

459 lines
13 KiB
Python

#!/usr/bin/env python3
"""
Web Search Skills CLI — search across 20+ sources from the command line.
Usage:
web-search <command> <query> [options]
Commands:
search Search all sources (default)
web Search web search engines (Google, Baidu, DuckDuckGo, etc.)
news Search news (财联社, 华尔街见闻, RSS)
wechat Search WeChat public account articles
academic Search academic papers (ArXiv)
social Search social media (Twitter/X)
rss Search RSS news feeds
sources List all available sources
urls Get search URLs without executing
Options:
-n, --max-results N Max results per source (default: 10)
-j, --json Output as JSON
-o, --output FILE Save results to file
-s, --source NAME Specific source name
-r, --region REGION Region filter (cn/global)
-v, --verbose Show debug logs
-h, --help Show this help
Examples:
web-search wechat 蒙面财经
web-search news "人工智能" -n 5
web-search web "Python async" -r global -j
web-search search "量子计算" -o results.json
web-search sources
web-search urls "机器学习"
"""
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import os
import sys
from datetime import datetime
from typing import Any
# Ensure scripts package is importable
_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR)
if _PROJECT_ROOT not in sys.path:
sys.path.insert(0, _PROJECT_ROOT)
logging.basicConfig(
level=logging.WARNING,
format="%(levelname)s: %(message)s",
)
logger = logging.getLogger("cli")
# ---------------------------------------------------------------------------
# Lazy engine imports (import on demand to keep startup fast)
# ---------------------------------------------------------------------------
def _get_unified():
from scripts.unified_search import UnifiedSearch
return UnifiedSearch
def _get_sogou():
from scripts.wechat.sogou_weixin import SogouWeChatEngine
return SogouWeChatEngine
def _get_rss():
from scripts.news.rss import RSSNewsEngine
return RSSNewsEngine
def _get_twitter():
from scripts.social.twitter import TwitterSearchEngine
return TwitterSearchEngine
def _get_ddg():
from scripts.engines.parser_engines import DuckDuckGoEngine
return DuckDuckGoEngine
def _get_config_loader():
from scripts.core.config_loader import (
get_search_engines,
get_news_sources,
get_academic_sources,
get_wechat_sources,
get_social_sources,
get_special_engines,
get_all_sources,
list_source_categories,
)
return {
"get_search_engines": get_search_engines,
"get_news_sources": get_news_sources,
"get_academic_sources": get_academic_sources,
"get_wechat_sources": get_wechat_sources,
"get_social_sources": get_social_sources,
"get_special_engines": get_special_engines,
"get_all_sources": get_all_sources,
"list_source_categories": list_source_categories,
}
# ---------------------------------------------------------------------------
# Result formatting
# ---------------------------------------------------------------------------
def _format_result(r, fmt: str = "text") -> str:
"""Format a single SearchResult."""
if fmt == "json":
return json.dumps({
"rank": r.rank,
"title": r.title,
"url": r.url,
"snippet": r.snippet[:300],
"source": r.source,
"category": r.category,
"extra": dict(r.extra) if r.extra else {},
}, ensure_ascii=False, indent=2)
lines = [f"[{r.rank}] {r.title}"]
lines.append(f" URL: {r.url}")
if r.snippet:
lines.append(f" {r.snippet[:200]}")
if r.extra:
extra_parts = []
for k, v in r.extra.items():
if k == "account" and v:
extra_parts.append(f"公众号: {v}")
elif k == "time" and v:
extra_parts.append(f"时间: {v}")
elif k == "search_url":
continue
elif v and k not in ("parsed_date", "note", "feed_url"):
extra_parts.append(f"{k}: {v}")
if extra_parts:
lines.append(f" ({'; '.join(extra_parts)})")
return "\n".join(lines)
def _print_results(results, fmt: str = "text"):
"""Print search results."""
if not results:
print(" 未找到结果")
return
if fmt == "json":
output = json.dumps(
[{
"rank": r.rank,
"title": r.title,
"url": r.url,
"snippet": r.snippet[:300],
"source": r.source,
"category": r.category,
"extra": dict(r.extra) if r.extra else {},
} for r in results],
ensure_ascii=False,
indent=2,
)
print(output)
else:
for r in results:
print(_format_result(r))
print()
# ---------------------------------------------------------------------------
# Async searchers
# ---------------------------------------------------------------------------
async def _search_all(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()()
try:
return await us.search(query, max_results=max_results, **kwargs)
finally:
await us.close()
async def _search_web(query: str, max_results: int, region: str = None, **kwargs) -> list[Any]:
us = _get_unified()()
try:
return await us.search_engines(query, region=region, max_results=max_results)
finally:
await us.close()
async def _search_news(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()()
try:
return await us.search_news(query, max_results=max_results, **kwargs)
finally:
await us.close()
async def _search_wechat(query: str, max_results: int) -> list[Any]:
engine = _get_sogou()()
try:
return await engine.search(query, max_results=max_results)
finally:
await engine.close()
async def _search_academic(query: str, max_results: int) -> list[Any]:
us = _get_unified()()
try:
return await us.search_academic(query, max_results=max_results)
finally:
await us.close()
async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()()
try:
return await us.search_social(query, max_results=max_results, **kwargs)
finally:
await us.close()
async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]:
us = _get_unified()()
try:
return await us.search_rss(query, max_results=max_results, **kwargs)
finally:
await us.close()
async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]:
us = _get_unified()()
try:
return await us.get_search_urls(query, sources=sources)
finally:
await us.close()
# ---------------------------------------------------------------------------
# CLI actions
# ---------------------------------------------------------------------------
def cmd_sources(args):
"""List all available search sources."""
cl = _get_config_loader()
categories = cl["list_source_categories"]()
print("可用搜索源:\n")
for cat_key, cat_name in [
("search_engines", "Web 搜索引擎"),
("news_sources", "新闻源"),
("academic_sources", "学术源"),
("wechat_sources", "微信源"),
("social_sources", "社交源"),
("special_engines", "特殊源"),
]:
count = categories.get(cat_key, 0)
if count == 0:
continue
print(f" {cat_name} ({count} 个)")
print(f"\n总计: {cl['get_all_sources']().__len__()} 个源")
print("\n详细列表:")
for s in cl["get_all_sources"]():
print(f" - {s['name']:20s} ({s.get('region', '?')}) [{s.get('type', 'url')}]")
def cmd_urls(args):
"""Get search URLs without executing."""
urls = asyncio.run(_get_urls(args.query, sources=args.source))
for name, url in urls.items():
print(f"{name}:")
print(f" {url}")
print()
async def _run_search(args):
"""Route search to the right engine(s)."""
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)
method_map = {
"search": _search_all,
"web": _search_web,
"news": _search_news,
"wechat": _search_wechat,
"academic": _search_academic,
"social": _search_social,
"rss": _search_rss,
}
search_fn = method_map.get(args.command, _search_all)
kwargs = {}
if args.command == "web" and args.region:
kwargs["region"] = args.region
try:
results = await search_fn(
args.query,
max_results=args.max_results,
**kwargs,
)
except Exception as e:
print(f"搜索失败: {e}", file=sys.stderr)
if args.verbose:
import traceback
traceback.print_exc()
sys.exit(1)
# Print header
cmd_names = {
"search": "全源搜索",
"web": "网页搜索",
"news": "新闻搜索",
"wechat": "微信公众号搜索",
"academic": "学术搜索",
"social": "社交搜索",
"rss": "RSS 新闻搜索",
}
header = cmd_names.get(args.command, "搜索")
print(f"{header}: {args.query}")
print(f"共 {len(results)} 条结果\n")
_print_results(results, fmt="json" if args.json else "text")
# Save to file if requested
if args.output:
output_data = json.dumps(
[{
"query": args.query,
"command": args.command,
"timestamp": datetime.now().isoformat(),
"total": len(results),
"results": [
{
"rank": r.rank,
"title": r.title,
"url": r.url,
"snippet": r.snippet[:500],
"source": r.source,
"category": r.category,
"extra": dict(r.extra) if r.extra else {},
}
for r in results
],
}],
ensure_ascii=False,
indent=2,
)
output_path = os.path.abspath(args.output)
with open(output_path, "w", encoding="utf-8") as f:
f.write(output_data)
print(f"\n结果已保存至: {output_path}")
# ---------------------------------------------------------------------------
# Argument parser
# ---------------------------------------------------------------------------
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="web-search",
description="Web Search Skills CLI — multi-source search tool",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
parser.add_argument(
"command",
nargs="?",
default="search",
choices=["search", "web", "news", "wechat", "academic", "social",
"rss", "sources", "urls"],
help="Search type (default: search)",
)
parser.add_argument(
"query",
nargs="?",
default="",
help="Search keyword",
)
parser.add_argument(
"-n", "--max-results",
type=int,
default=10,
help="Max results per source (default: 10)",
)
parser.add_argument(
"-j", "--json",
action="store_true",
help="Output as JSON",
)
parser.add_argument(
"-o", "--output",
type=str,
help="Save results to JSON file",
)
parser.add_argument(
"-s", "--source",
type=str,
nargs="*",
help="Specific source name(s)",
)
parser.add_argument(
"-r", "--region",
type=str,
choices=["cn", "global"],
help="Region filter (web search only)",
)
parser.add_argument(
"-v", "--verbose",
action="store_true",
help="Show debug logs",
)
return parser
# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
def main():
parser = build_parser()
args = parser.parse_args()
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)
# sources and urls commands don't need a query
if args.command == "sources":
cmd_sources(args)
return
if args.command == "urls":
if not args.query:
print("错误: urls 命令需要提供搜索关键词", file=sys.stderr)
sys.exit(1)
cmd_urls(args)
return
# Other commands need a query
if not args.query:
print(f"错误: {args.command} 命令需要提供搜索关键词", file=sys.stderr)
sys.exit(1)
asyncio.run(_run_search(args))
if __name__ == "__main__":
main()