From 87935b59cee967e6eefffc13aa565ab23798d361 Mon Sep 17 00:00:00 2001 From: vex1023 Date: Thu, 14 May 2026 16:31:54 +0800 Subject: [PATCH] refactor: restructure project with CLI tool, multi-engine support, and skill-builder compliant SKILL.md - Replace setup.py with pyproject.toml (uv-compatible) - Add CLI entry point (web-search) with 9 subcommands - Add multi-engine architecture (14 web + 7 special sources) - Add WeChat/CLS/WallStreetCN/ArXiv/Twitter search engines - Remove old src/ and examples/ directories - Remove redundant requirements.txt and setup.py - Update README.md and SKILL.md with full documentation - SKILL.md now compliant with skill-builder standards (frontmatter, triggers, structure) --- README.md | 145 +++++----- config/engines.json | 35 +++ examples/__init__.py | 1 - examples/basic_search.py | 36 --- pyproject.toml | 20 ++ requirements.txt | 15 - scripts/__init__.py | 1 + scripts/academic/__init__.py | 1 + scripts/academic/arxiv.py | 130 +++++++++ scripts/cli.py | 458 ++++++++++++++++++++++++++++++ scripts/core/__init__.py | 1 + scripts/core/base.py | 48 ++++ scripts/core/config_loader.py | 70 +++++ scripts/engines/__init__.py | 1 + scripts/engines/parser_engines.py | 80 ++++++ scripts/engines/url_engine.py | 60 ++++ scripts/news/__init__.py | 1 + scripts/news/cls.py | 115 ++++++++ scripts/news/rss.py | 288 +++++++++++++++++++ scripts/news/wallstreetcn.py | 117 ++++++++ scripts/social/__init__.py | 1 + scripts/social/twitter.py | 208 ++++++++++++++ scripts/unified_search.py | 183 ++++++++++++ scripts/wechat/__init__.py | 1 + scripts/wechat/sogou_weixin.py | 325 +++++++++++++++++++++ skill.md | 149 ++++++++-- src/__init__.py | 4 - src/web_search.py | 201 ------------- ws.py | 23 ++ 29 files changed, 2364 insertions(+), 354 deletions(-) create mode 100644 config/engines.json delete mode 100644 examples/__init__.py delete mode 100644 examples/basic_search.py create mode 100644 pyproject.toml delete mode 100644 requirements.txt create mode 100644 scripts/__init__.py create mode 100644 scripts/academic/__init__.py create mode 100644 scripts/academic/arxiv.py create mode 100644 scripts/cli.py create mode 100644 scripts/core/__init__.py create mode 100644 scripts/core/base.py create mode 100644 scripts/core/config_loader.py create mode 100644 scripts/engines/__init__.py create mode 100644 scripts/engines/parser_engines.py create mode 100644 scripts/engines/url_engine.py create mode 100644 scripts/news/__init__.py create mode 100644 scripts/news/cls.py create mode 100644 scripts/news/rss.py create mode 100644 scripts/news/wallstreetcn.py create mode 100644 scripts/social/__init__.py create mode 100644 scripts/social/twitter.py create mode 100644 scripts/unified_search.py create mode 100644 scripts/wechat/__init__.py create mode 100644 scripts/wechat/sogou_weixin.py delete mode 100644 src/__init__.py delete mode 100644 src/web_search.py create mode 100644 ws.py diff --git a/README.md b/README.md index a38523f..24c7538 100644 --- a/README.md +++ b/README.md @@ -1,95 +1,86 @@ # Web Search Skills -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) +Multi-source web search CLI tool — search across 20+ engines from your terminal. -A collection of web search tools and skills for AI agents, designed to enhance search capabilities with customizable configurations, multiple search engines support, and intelligent result processing. - -## Features - -- **Multi-Engine Support** — Built-in support for Google, Bing, DuckDuckGo, and custom search backends -- **Intelligent Result Processing** — Extract, summarize, and rank search results automatically -- **Configurable Filters** — Filter by date, domain, language, region, and content type -- **Async & Concurrent** — Non-blocking requests with concurrent search execution -- **Extensible Architecture** — Plugin-based design for easy integration of new search sources -- **Agent-Ready** — Designed as a CodeBuddy Agent Skill for seamless AI integration - -## Getting Started - -### Prerequisites - -- Python 3.8+ -- pip / poetry - -### Installation +## Quick Start ```bash -git clone https://github.com/openbot-coder/web-search-skills.git +# Install with uv (recommended) cd web-search-skills -pip install -r requirements.txt +uv tool install -e . + +# Now use globally from anywhere +web-search wechat 蒙面财经 +web-search news "AI" -n 5 +web-search search "量子计算" -o results.json ``` -### Quick Start - -```python -import asyncio -from src.web_search import search_web - -results = asyncio.run(search_web("latest AI developments")) -for result in results: - print(f"- {result.title}: {result.url}") +Or run directly without install: +```bash +python ws.py wechat 蒙面财经 ``` +## Usage + +```bash +web-search [options] +``` + +| Command | Description | +|------------|------------------------------------------| +| `search` | Search all sources (default) | +| `web` | Web search engines (Google, Baidu, DDG) | +| `news` | News (财联社, 华尔街见闻, RSS) | +| `wechat` | WeChat public account articles | +| `academic` | Academic papers (ArXiv) | +| `social` | Social media (Twitter/X) | +| `rss` | RSS news feeds | +| `sources` | List all available search sources | +| `urls` | Generate search URLs without executing | + +Options: + +| Flag | Description | +|------|-------------| +| `-n N` | Max results per source (default: 10) | +| `-j` | JSON output | +| `-o FILE` | Save to JSON file | +| `-s NAME` | Specific source name(s) | +| `-r cn/global` | Region filter (web search) | +| `-v` | Verbose/debug logs | + ## Project Structure ``` web-search-skills/ -├── README.md # Project documentation -├── LICENSE # MIT License -├── skill.md # CodeBuddy Agent Skill definition -├── requirements.txt # Python dependencies -├── src/ # Source code -│ └── web_search.py # Core search implementation -└── examples/ # Usage examples - └── basic_search.py +├── scripts/ +│ ├── cli.py # CLI entry point +│ ├── unified_search.py # Unified search orchestrator +│ ├── core/ +│ │ ├── base.py # SearchResult & SearchEngine base class +│ │ └── config_loader.py # Load engine config from JSON +│ ├── engines/ +│ │ ├── url_engine.py # URL-template search engines +│ │ └── parser_engines.py # HTML-parsed search (DuckDuckGo) +│ ├── news/ +│ │ ├── cls.py # 财联社 +│ │ ├── wallstreetcn.py # 华尔街见闻 +│ │ └── rss.py # RSS feeds +│ ├── academic/ +│ │ └── arxiv.py # ArXiv papers +│ ├── wechat/ +│ │ └── sogou_weixin.py # WeChat (via Sogou) +│ └── social/ +│ └── twitter.py # Twitter/X +├── config/ +│ └── engines.json # Search engine definitions +├── ws.py # Direct run entry point +├── pyproject.toml # Package config +├── SKILL.md # CodeBuddy skill definition +├── LICENSE +└── README.md ``` -## Usage - -### As a CodeBuddy Agent Skill - -This project is designed as a CodeBuddy Agent Skill. To install: - -1. Copy `skill.md` to your CodeBuddy skills directory -2. Configure your search engine API keys -3. The skill is now available to your AI agent - -### API Reference - -```python -# Core class -searcher = WebSearch(config=None) -results = await searcher.search(query, engine="duckduckgo", max_results=10) -content = await searcher.extract_content(url) - -# Convenience function -results = await search_web(query, engine="duckduckgo", max_results=10) -``` - -## Contributing - -We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. - -1. Fork the repository -2. Create your feature branch (`git checkout -b feature/amazing-feature`) -3. Commit your changes -4. Push to the branch -5. Open a Pull Request - ## License -This project is licensed under the MIT License — see the [LICENSE](LICENSE) file for details. - -## Acknowledgments - -- Built for the CodeBuddy ecosystem -- Inspired by the need for better web search capabilities in AI agents +MIT diff --git a/config/engines.json b/config/engines.json new file mode 100644 index 0000000..ab36cb3 --- /dev/null +++ b/config/engines.json @@ -0,0 +1,35 @@ +{ + "search_engines": [ + {"name": "Baidu", "url": "https://www.baidu.com/s?wd={keyword}", "region": "cn", "type": "url"}, + {"name": "Bing CN", "url": "https://cn.bing.com/search?q={keyword}&ensearch=0", "region": "cn", "type": "url"}, + {"name": "Bing INT", "url": "https://cn.bing.com/search?q={keyword}&ensearch=1", "region": "cn", "type": "url"}, + {"name": "360", "url": "https://www.so.com/s?q={keyword}", "region": "cn", "type": "url"}, + {"name": "Sogou", "url": "https://sogou.com/web?query={keyword}", "region": "cn", "type": "url"}, + {"name": "Shenma", "url": "https://m.sm.cn/s?q={keyword}", "region": "cn", "type": "url"}, + {"name": "Google", "url": "https://www.google.com/search?q={keyword}", "region": "global", "type": "url"}, + {"name": "Google HK", "url": "https://www.google.com.hk/search?q={keyword}", "region": "global", "type": "url"}, + {"name": "DuckDuckGo", "url": "https://duckduckgo.com/html/?q={keyword}", "region": "global", "type": "parser"}, + {"name": "Yahoo", "url": "https://search.yahoo.com/search?p={keyword}", "region": "global", "type": "url"}, + {"name": "Startpage", "url": "https://www.startpage.com/sp/search?query={keyword}", "region": "global", "type": "url"}, + {"name": "Brave", "url": "https://search.brave.com/search?q={keyword}", "region": "global", "type": "url"}, + {"name": "Ecosia", "url": "https://www.ecosia.org/search?q={keyword}", "region": "global", "type": "url"}, + {"name": "Qwant", "url": "https://www.qwant.com/?q={keyword}", "region": "global", "type": "url"} + ], + "special_engines": [ + {"name": "WolframAlpha", "url": "https://www.wolframalpha.com/input?i={keyword}", "region": "global", "type": "url"} + ], + "news_sources": [ + {"name": "cls", "url": "https://www.cls.cn/searchPage?keyword={keyword}&type=all", "region": "cn", "type": "news"}, + {"name": "wallstreetcn", "url": "https://wallstreetcn.com/search?q={keyword}", "region": "cn", "type": "news"} + ], + "academic_sources": [ + {"name": "arxiv", "url": "https://export.arxiv.org/api/query?search_query=all:{keyword}&max_results=10", "region": "global", "type": "api"} + ], + "wechat_sources": [ + {"name": "weixin", "url": "https://wx.sogou.com/weixin?type=2&query={keyword}", "region": "cn", "type": "url"} + ], + "social_sources": [ + {"name": "Twitter", "url": "https://twitter.com/search?q={keyword}&src=typed_query", "region": "global", "type": "social"}, + {"name": "Twitter Latest", "url": "https://twitter.com/search?q={keyword}&f=live&src=typed_query", "region": "global", "type": "social"} + ] +} diff --git a/examples/__init__.py b/examples/__init__.py deleted file mode 100644 index 88f1d65..0000000 --- a/examples/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Examples for using Web Search Skills.""" diff --git a/examples/basic_search.py b/examples/basic_search.py deleted file mode 100644 index 78c285e..0000000 --- a/examples/basic_search.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Basic usage example for Web Search Skills. - -Run: - python examples/basic_search.py -""" - -import asyncio - -from src.web_search import WebSearch - - -async def main(): - async with WebSearch() as searcher: - results = await searcher.search( - "Python web scraping best practices 2026", - max_results=5, - ) - - print(f"Found {len(results)} results:\n") - for result in results: - print(f"[{result.rank}] {result.title}") - print(f" URL: {result.url}") - print(f" {result.snippet[:120]}...") - print() - - if results: - content = await searcher.extract_content(results[0].url) - if content: - print(f"--- Content from {results[0].url} ---") - print(content[:300]) - print("...") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e40a83c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "web-search-skills" +version = "2.1.0" +description = "Multi-source web search CLI tool" +requires-python = ">=3.8" +dependencies = [ + "httpx>=0.25.0", + "beautifulsoup4>=4.12.0", + "lxml>=4.9.0", +] + +[project.scripts] +web-search = "scripts.cli:main" + +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["scripts*"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 699ad0b..0000000 --- a/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Web Search Skills Dependencies - -# Core -httpx>=0.25.0 -beautifulsoup4>=4.12.0 -lxml>=4.9.0 - -# Optional: Google Custom Search -# google-api-python-client>=2.100.0 - -# Development -pytest>=7.0.0 -pytest-asyncio>=0.21.0 -black>=23.0.0 -ruff>=0.1.0 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..2a97cf5 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Web Search Skills package.""" diff --git a/scripts/academic/__init__.py b/scripts/academic/__init__.py new file mode 100644 index 0000000..4da2d65 --- /dev/null +++ b/scripts/academic/__init__.py @@ -0,0 +1 @@ +"""Academic search engines.""" diff --git a/scripts/academic/arxiv.py b/scripts/academic/arxiv.py new file mode 100644 index 0000000..da62a87 --- /dev/null +++ b/scripts/academic/arxiv.py @@ -0,0 +1,130 @@ +""" +ArXiv academic paper search engine. + +Uses the ArXiv API (export.arxiv.org/api/query) to search for papers. +""" + +from __future__ import annotations + +import logging +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from typing import Optional +from urllib.parse import quote_plus + +import httpx + +from scripts.core.base import SearchEngine, SearchResult + +logger = logging.getLogger(__name__) + + +class ArXivEngine(SearchEngine): + """ArXiv academic paper search engine via official API.""" + + API_URL = "https://export.arxiv.org/api/query" + + def __init__(self, config: Optional[dict] = None): + super().__init__("ArXiv", config) + timeout = (config or {}).get("timeout", 30) + self._client = httpx.AsyncClient( + timeout=timeout, + headers={ + "User-Agent": (config or {}).get( + "user_agent", + "ArXivSearchSkill/1.0", + ), + "Accept": "application/atom+xml", + }, + follow_redirects=True, + ) + + def build_url(self, query: str, max_results: int = 10, **kwargs) -> str: + search_field = kwargs.get("search_field", "all") + return ( + f"{self.API_URL}?search_query={search_field}:{quote_plus(query)}" + f"&max_results={max_results}&sortBy=submittedDate&sortOrder=descending" + ) + + async def search( + self, query: str, max_results: int = 10, **kwargs + ) -> list[SearchResult]: + """Search ArXiv papers.""" + url = self.build_url(query, max_results=max_results, **kwargs) + try: + response = await self._client.get(url) + response.raise_for_status() + return self._parse_response(response.text, max_results) + except httpx.HTTPError as e: + logger.error(f"ArXiv search failed: {e}") + return [] + + def _parse_response(self, xml_text: str, max_results: int) -> list[SearchResult]: + """Parse ArXiv Atom XML response.""" + ns = { + "atom": "http://www.w3.org/2005/Atom", + "arxiv": "http://arxiv.org/schemas/atom", + } + results = [] + + try: + root = ET.fromstring(xml_text) + except ET.ParseError as e: + logger.error(f"ArXiv XML parse error: {e}") + return [] + + for i, entry in enumerate(root.findall("atom:entry", ns)[:max_results]): + title = entry.findtext("atom:title", "", ns) + summary = entry.findtext("atom:summary", "", ns) + published = entry.findtext("atom:published", "", ns) + updated = entry.findtext("atom:updated", "", ns) + + link_el = entry.find("atom:link", ns) + link = link_el.get("href", "") if link_el is not None else "" + + authors = [] + for author in entry.findall("atom:author", ns): + name = author.findtext("atom:name", "", ns) + if name: + authors.append(name) + + # Clean title and summary + title = title.replace("\n", " ").strip() + snippet = summary.replace("\n", " ").strip()[:300] + + parsed_date = self._parse_date(published) if published else ( + self._parse_date(updated) if updated else + datetime.min.replace(tzinfo=timezone.utc) + ) + + results.append( + SearchResult( + title=title or f"ArXiv: {summary[:60]}", + url=link, + snippet=snippet or "No abstract available", + source=self.name, + rank=i + 1, + category="academic", + extra={ + "published": published or "", + "updated": updated or "", + "authors": ", ".join(authors), + "parsed_date": parsed_date, + }, + ) + ) + + return results + + def _parse_date(self, date_str: str) -> datetime: + """Parse ArXiv date format.""" + try: + # ArXiv format: 2024-01-01T00:00:00Z + return datetime.strptime( + date_str.strip(), "%Y-%m-%dT%H:%M:%SZ" + ).replace(tzinfo=timezone.utc) + except (ValueError, TypeError): + return datetime.min.replace(tzinfo=timezone.utc) + + async def close(self): + await self._client.aclose() diff --git a/scripts/cli.py b/scripts/cli.py new file mode 100644 index 0000000..6550bd0 --- /dev/null +++ b/scripts/cli.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +""" +Web Search Skills CLI — search across 20+ sources from the command line. + +Usage: + web-search [options] + +Commands: + search Search all sources (default) + web Search web search engines (Google, Baidu, DuckDuckGo, etc.) + news Search news (财联社, 华尔街见闻, RSS) + wechat Search WeChat public account articles + academic Search academic papers (ArXiv) + social Search social media (Twitter/X) + rss Search RSS news feeds + sources List all available sources + urls Get search URLs without executing + +Options: + -n, --max-results N Max results per source (default: 10) + -j, --json Output as JSON + -o, --output FILE Save results to file + -s, --source NAME Specific source name + -r, --region REGION Region filter (cn/global) + -v, --verbose Show debug logs + -h, --help Show this help + +Examples: + web-search wechat 蒙面财经 + web-search news "人工智能" -n 5 + web-search web "Python async" -r global -j + web-search search "量子计算" -o results.json + web-search sources + web-search urls "机器学习" +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import sys +from datetime import datetime +from typing import Any + +# Ensure scripts package is importable +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR) +if _PROJECT_ROOT not in sys.path: + sys.path.insert(0, _PROJECT_ROOT) + +logging.basicConfig( + level=logging.WARNING, + format="%(levelname)s: %(message)s", +) +logger = logging.getLogger("cli") + + +# --------------------------------------------------------------------------- +# Lazy engine imports (import on demand to keep startup fast) +# --------------------------------------------------------------------------- + +def _get_unified(): + from scripts.unified_search import UnifiedSearch + return UnifiedSearch + + +def _get_sogou(): + from scripts.wechat.sogou_weixin import SogouWeChatEngine + return SogouWeChatEngine + + +def _get_rss(): + from scripts.news.rss import RSSNewsEngine + return RSSNewsEngine + + +def _get_twitter(): + from scripts.social.twitter import TwitterSearchEngine + return TwitterSearchEngine + + +def _get_ddg(): + from scripts.engines.parser_engines import DuckDuckGoEngine + return DuckDuckGoEngine + + +def _get_config_loader(): + from scripts.core.config_loader import ( + get_search_engines, + get_news_sources, + get_academic_sources, + get_wechat_sources, + get_social_sources, + get_special_engines, + get_all_sources, + list_source_categories, + ) + return { + "get_search_engines": get_search_engines, + "get_news_sources": get_news_sources, + "get_academic_sources": get_academic_sources, + "get_wechat_sources": get_wechat_sources, + "get_social_sources": get_social_sources, + "get_special_engines": get_special_engines, + "get_all_sources": get_all_sources, + "list_source_categories": list_source_categories, + } + + +# --------------------------------------------------------------------------- +# Result formatting +# --------------------------------------------------------------------------- + +def _format_result(r, fmt: str = "text") -> str: + """Format a single SearchResult.""" + if fmt == "json": + return json.dumps({ + "rank": r.rank, + "title": r.title, + "url": r.url, + "snippet": r.snippet[:300], + "source": r.source, + "category": r.category, + "extra": dict(r.extra) if r.extra else {}, + }, ensure_ascii=False, indent=2) + + lines = [f"[{r.rank}] {r.title}"] + lines.append(f" URL: {r.url}") + if r.snippet: + lines.append(f" {r.snippet[:200]}") + if r.extra: + extra_parts = [] + for k, v in r.extra.items(): + if k == "account" and v: + extra_parts.append(f"公众号: {v}") + elif k == "time" and v: + extra_parts.append(f"时间: {v}") + elif k == "search_url": + continue + elif v and k not in ("parsed_date", "note", "feed_url"): + extra_parts.append(f"{k}: {v}") + if extra_parts: + lines.append(f" ({'; '.join(extra_parts)})") + return "\n".join(lines) + + +def _print_results(results, fmt: str = "text"): + """Print search results.""" + if not results: + print(" 未找到结果") + return + + if fmt == "json": + output = json.dumps( + [{ + "rank": r.rank, + "title": r.title, + "url": r.url, + "snippet": r.snippet[:300], + "source": r.source, + "category": r.category, + "extra": dict(r.extra) if r.extra else {}, + } for r in results], + ensure_ascii=False, + indent=2, + ) + print(output) + else: + for r in results: + print(_format_result(r)) + print() + + +# --------------------------------------------------------------------------- +# Async searchers +# --------------------------------------------------------------------------- + +async def _search_all(query: str, max_results: int, **kwargs) -> list[Any]: + us = _get_unified()() + try: + return await us.search(query, max_results=max_results, **kwargs) + finally: + await us.close() + + +async def _search_web(query: str, max_results: int, region: str = None, **kwargs) -> list[Any]: + us = _get_unified()() + try: + return await us.search_engines(query, region=region, max_results=max_results) + finally: + await us.close() + + +async def _search_news(query: str, max_results: int, **kwargs) -> list[Any]: + us = _get_unified()() + try: + return await us.search_news(query, max_results=max_results, **kwargs) + finally: + await us.close() + + +async def _search_wechat(query: str, max_results: int) -> list[Any]: + engine = _get_sogou()() + try: + return await engine.search(query, max_results=max_results) + finally: + await engine.close() + + +async def _search_academic(query: str, max_results: int) -> list[Any]: + us = _get_unified()() + try: + return await us.search_academic(query, max_results=max_results) + finally: + await us.close() + + +async def _search_social(query: str, max_results: int, **kwargs) -> list[Any]: + us = _get_unified()() + try: + return await us.search_social(query, max_results=max_results, **kwargs) + finally: + await us.close() + + +async def _search_rss(query: str, max_results: int, **kwargs) -> list[Any]: + us = _get_unified()() + try: + return await us.search_rss(query, max_results=max_results, **kwargs) + finally: + await us.close() + + +async def _get_urls(query: str, sources: list[str] = None) -> dict[str, str]: + us = _get_unified()() + try: + return await us.get_search_urls(query, sources=sources) + finally: + await us.close() + + +# --------------------------------------------------------------------------- +# CLI actions +# --------------------------------------------------------------------------- + +def cmd_sources(args): + """List all available search sources.""" + cl = _get_config_loader() + categories = cl["list_source_categories"]() + + print("可用搜索源:\n") + for cat_key, cat_name in [ + ("search_engines", "Web 搜索引擎"), + ("news_sources", "新闻源"), + ("academic_sources", "学术源"), + ("wechat_sources", "微信源"), + ("social_sources", "社交源"), + ("special_engines", "特殊源"), + ]: + count = categories.get(cat_key, 0) + if count == 0: + continue + print(f" {cat_name} ({count} 个)") + + print(f"\n总计: {cl['get_all_sources']().__len__()} 个源") + print("\n详细列表:") + for s in cl["get_all_sources"](): + print(f" - {s['name']:20s} ({s.get('region', '?')}) [{s.get('type', 'url')}]") + + +def cmd_urls(args): + """Get search URLs without executing.""" + urls = asyncio.run(_get_urls(args.query, sources=args.source)) + for name, url in urls.items(): + print(f"{name}:") + print(f" {url}") + print() + + +async def _run_search(args): + """Route search to the right engine(s).""" + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + method_map = { + "search": _search_all, + "web": _search_web, + "news": _search_news, + "wechat": _search_wechat, + "academic": _search_academic, + "social": _search_social, + "rss": _search_rss, + } + + search_fn = method_map.get(args.command, _search_all) + + kwargs = {} + if args.command == "web" and args.region: + kwargs["region"] = args.region + + try: + results = await search_fn( + args.query, + max_results=args.max_results, + **kwargs, + ) + except Exception as e: + print(f"搜索失败: {e}", file=sys.stderr) + if args.verbose: + import traceback + traceback.print_exc() + sys.exit(1) + + # Print header + cmd_names = { + "search": "全源搜索", + "web": "网页搜索", + "news": "新闻搜索", + "wechat": "微信公众号搜索", + "academic": "学术搜索", + "social": "社交搜索", + "rss": "RSS 新闻搜索", + } + header = cmd_names.get(args.command, "搜索") + print(f"{header}: {args.query}") + print(f"共 {len(results)} 条结果\n") + + _print_results(results, fmt="json" if args.json else "text") + + # Save to file if requested + if args.output: + output_data = json.dumps( + [{ + "query": args.query, + "command": args.command, + "timestamp": datetime.now().isoformat(), + "total": len(results), + "results": [ + { + "rank": r.rank, + "title": r.title, + "url": r.url, + "snippet": r.snippet[:500], + "source": r.source, + "category": r.category, + "extra": dict(r.extra) if r.extra else {}, + } + for r in results + ], + }], + ensure_ascii=False, + indent=2, + ) + output_path = os.path.abspath(args.output) + with open(output_path, "w", encoding="utf-8") as f: + f.write(output_data) + print(f"\n结果已保存至: {output_path}") + + +# --------------------------------------------------------------------------- +# Argument parser +# --------------------------------------------------------------------------- + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="web-search", + description="Web Search Skills CLI — multi-source search tool", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + parser.add_argument( + "command", + nargs="?", + default="search", + choices=["search", "web", "news", "wechat", "academic", "social", + "rss", "sources", "urls"], + help="Search type (default: search)", + ) + parser.add_argument( + "query", + nargs="?", + default="", + help="Search keyword", + ) + parser.add_argument( + "-n", "--max-results", + type=int, + default=10, + help="Max results per source (default: 10)", + ) + parser.add_argument( + "-j", "--json", + action="store_true", + help="Output as JSON", + ) + parser.add_argument( + "-o", "--output", + type=str, + help="Save results to JSON file", + ) + parser.add_argument( + "-s", "--source", + type=str, + nargs="*", + help="Specific source name(s)", + ) + parser.add_argument( + "-r", "--region", + type=str, + choices=["cn", "global"], + help="Region filter (web search only)", + ) + parser.add_argument( + "-v", "--verbose", + action="store_true", + help="Show debug logs", + ) + + return parser + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def main(): + parser = build_parser() + args = parser.parse_args() + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + # sources and urls commands don't need a query + if args.command == "sources": + cmd_sources(args) + return + + if args.command == "urls": + if not args.query: + print("错误: urls 命令需要提供搜索关键词", file=sys.stderr) + sys.exit(1) + cmd_urls(args) + return + + # Other commands need a query + if not args.query: + print(f"错误: {args.command} 命令需要提供搜索关键词", file=sys.stderr) + sys.exit(1) + + asyncio.run(_run_search(args)) + + +if __name__ == "__main__": + main() diff --git a/scripts/core/__init__.py b/scripts/core/__init__.py new file mode 100644 index 0000000..c1963bd --- /dev/null +++ b/scripts/core/__init__.py @@ -0,0 +1 @@ +"""Core data types and base classes.""" diff --git a/scripts/core/base.py b/scripts/core/base.py new file mode 100644 index 0000000..8465592 --- /dev/null +++ b/scripts/core/base.py @@ -0,0 +1,48 @@ +""" +Core data types and base classes for search engines. + +Provides: +- SearchResult: Standard result dataclass used across all engines +- SearchEngine: Abstract base class for all search engine implementations +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any, Optional + + +@dataclass +class SearchResult: + """Represents a single search result.""" + title: str = "" + url: str = "" + snippet: str = "" + source: str = "" + rank: int = 0 + category: str = "" + extra: dict[str, Any] = field(default_factory=dict) + + +class SearchEngine(ABC): + """Abstract base class for search engines.""" + + def __init__(self, name: str, config: Optional[dict[str, Any]] = None): + self.name = name + self.config = config or {} + + @abstractmethod + async def search( + self, query: str, max_results: int = 10, **kwargs + ) -> list[SearchResult]: + """Execute a search and return results.""" + ... + + def build_url(self, query: str, **kwargs) -> str: + """Build a search URL for this engine (optional).""" + return "" + + async def close(self): + """Clean up resources.""" + pass diff --git a/scripts/core/config_loader.py b/scripts/core/config_loader.py new file mode 100644 index 0000000..c17928e --- /dev/null +++ b/scripts/core/config_loader.py @@ -0,0 +1,70 @@ +"""Load and manage engine configuration from engines.json.""" + +from __future__ import annotations + +import json +import os +from typing import Any, Optional + +_CONFIG_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + "config", "engines.json", +) + + +def _load_config(path: Optional[str] = None) -> dict[str, Any]: + config_path = path or _CONFIG_PATH + with open(config_path, "r", encoding="utf-8") as f: + return json.load(f) + + +def get_search_engines(path: Optional[str] = None) -> list[dict[str, Any]]: + return _load_config(path).get("search_engines", []) + + +def get_news_sources(path: Optional[str] = None) -> list[dict[str, Any]]: + return _load_config(path).get("news_sources", []) + + +def get_academic_sources(path: Optional[str] = None) -> list[dict[str, Any]]: + return _load_config(path).get("academic_sources", []) + + +def get_wechat_sources(path: Optional[str] = None) -> list[dict[str, Any]]: + return _load_config(path).get("wechat_sources", []) + + +def get_social_sources(path: Optional[str] = None) -> list[dict[str, Any]]: + return _load_config(path).get("social_sources", []) + + +def get_special_engines(path: Optional[str] = None) -> list[dict[str, Any]]: + return _load_config(path).get("special_engines", []) + + +def get_engine(name: str, path: Optional[str] = None): + config = _load_config(path) + for category in ["search_engines", "news_sources", "academic_sources", + "wechat_sources", "social_sources", "special_engines"]: + for engine in config.get(category, []): + if engine["name"].lower() == name.lower(): + return engine + return None + + +def get_all_sources(path: Optional[str] = None) -> list[dict[str, Any]]: + config = _load_config(path) + all_sources = [] + for category in ["search_engines", "news_sources", "academic_sources", + "wechat_sources", "social_sources", "special_engines"]: + all_sources.extend(config.get(category, [])) + return all_sources + +def list_source_categories(path: Optional[str] = None) -> dict[str, int]: + """Return a dict mapping category name to count of sources.""" + config = _load_config(path) + categories = [ + "search_engines", "news_sources", "academic_sources", + "wechat_sources", "social_sources", "special_engines", + ] + return {cat: len(config.get(cat, [])) for cat in categories} diff --git a/scripts/engines/__init__.py b/scripts/engines/__init__.py new file mode 100644 index 0000000..2297762 --- /dev/null +++ b/scripts/engines/__init__.py @@ -0,0 +1 @@ +"""Search engine implementations.""" diff --git a/scripts/engines/parser_engines.py b/scripts/engines/parser_engines.py new file mode 100644 index 0000000..dc12287 --- /dev/null +++ b/scripts/engines/parser_engines.py @@ -0,0 +1,80 @@ +""" +Parsing-based search engines (e.g. DuckDuckGo HTML scraper). +""" + +from __future__ import annotations + +import logging +from typing import Optional + +import httpx +from bs4 import BeautifulSoup + +from scripts.core.base import SearchEngine, SearchResult + +logger = logging.getLogger(__name__) + + +class DuckDuckGoEngine(SearchEngine): + """DuckDuckGo HTML search engine (no API key required).""" + + SEARCH_URL = "https://html.duckduckgo.com/html/" + + def __init__(self, config: Optional[dict] = None): + super().__init__("DuckDuckGo", config) + timeout = (config or {}).get("timeout", 15) + self._client = httpx.AsyncClient( + timeout=timeout, + headers={ + "User-Agent": (config or {}).get( + "user_agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36", + ), + }, + follow_redirects=True, + ) + + def build_url(self, query: str, **kwargs) -> str: + """Build DuckDuckGo search URL.""" + from urllib.parse import quote_plus + return f"https://duckduckgo.com/?q={quote_plus(query)}" + + async def search( + self, query: str, max_results: int = 10, **kwargs + ) -> list[SearchResult]: + """Search DuckDuckGo and parse HTML results.""" + params = {"q": query} + try: + response = await self._client.post(self.SEARCH_URL, data=params) + response.raise_for_status() + return self._parse_results(response.text, max_results) + except httpx.HTTPError as e: + logger.error(f"DuckDuckGo search failed: {e}") + return [] + + def _parse_results(self, html: str, max_results: int) -> list[SearchResult]: + """Parse DuckDuckGo HTML search results.""" + soup = BeautifulSoup(html, "lxml") + results = [] + + for i, result in enumerate(soup.select(".result")[:max_results]): + title_el = result.select_one(".result__title a") + snippet_el = result.select_one(".result__snippet") + if title_el: + results.append( + SearchResult( + title=title_el.get_text(strip=True), + url=title_el.get("href", ""), + snippet=snippet_el.get_text(strip=True) + if snippet_el else "", + source=self.name, + rank=i + 1, + category="web", + ) + ) + return results + + async def close(self): + await self._client.aclose() diff --git a/scripts/engines/url_engine.py b/scripts/engines/url_engine.py new file mode 100644 index 0000000..460c5af --- /dev/null +++ b/scripts/engines/url_engine.py @@ -0,0 +1,60 @@ +""" +Generic URL-based search engine. + +Builds search URLs from templates defined in engines.json +and returns them as URL results. +""" + +from __future__ import annotations + +import logging +from typing import Any, Optional +from urllib.parse import quote_plus + +from scripts.core.base import SearchEngine, SearchResult + +logger = logging.getLogger(__name__) + + +class UrlSearchEngine(SearchEngine): + """ + Generic search engine that builds URLs from templates. + + Reads URL templates from config (engines.json) and returns + the constructed search URL as a SearchResult. + """ + + def __init__(self, name: str, engine_def: dict[str, Any]): + super().__init__(name, engine_def.get("config", {})) + self.url_template = engine_def.get("url", "") + self.region = engine_def.get("region", "global") + + def build_url(self, query: str, **kwargs) -> str: + """Build search URL by replacing {keyword} with encoded query.""" + if not self.url_template: + return "" + encoded = quote_plus(query) + return self.url_template.replace("{keyword}", encoded) + + async def search( + self, query: str, max_results: int = 10, **kwargs + ) -> list[SearchResult]: + """Return the search URL as a single result.""" + url = self.build_url(query, **kwargs) + if not url: + return [] + + return [ + SearchResult( + title=f"{self.name}: {query}", + url=url, + snippet=f"Search {self.name} for '{query}'", + source=self.name, + rank=1, + category="web", + extra={"url": url, "region": self.region}, + ) + ] + + async def close(self): + pass diff --git a/scripts/news/__init__.py b/scripts/news/__init__.py new file mode 100644 index 0000000..5a0108e --- /dev/null +++ b/scripts/news/__init__.py @@ -0,0 +1 @@ +"""News search engines.""" diff --git a/scripts/news/cls.py b/scripts/news/cls.py new file mode 100644 index 0000000..07fbf1e --- /dev/null +++ b/scripts/news/cls.py @@ -0,0 +1,115 @@ +""" +财联社 (cls.cn) news search engine. + +Scrapes cls.cn search results for financial news. +""" + +from __future__ import annotations + +import logging +from typing import Optional +from urllib.parse import quote_plus + +import httpx +from bs4 import BeautifulSoup + +from scripts.core.base import SearchEngine, SearchResult + +logger = logging.getLogger(__name__) + + +class CLSNewsEngine(SearchEngine): + """财联社 (cls.cn) news search engine.""" + + SEARCH_URL = "https://www.cls.cn/searchPage" + + def __init__(self, config: Optional[dict] = None): + super().__init__("cls", config) + timeout = (config or {}).get("timeout", 15) + self._client = httpx.AsyncClient( + timeout=timeout, + headers={ + "User-Agent": (config or {}).get( + "user_agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36", + ), + "Referer": "https://www.cls.cn/", + }, + follow_redirects=True, + ) + + def build_url(self, query: str, **kwargs) -> str: + return f"{self.SEARCH_URL}?keyword={quote_plus(query)}&type=all" + + async def search( + self, query: str, max_results: int = 10, **kwargs + ) -> list[SearchResult]: + """Search 财联社 and parse results.""" + url = self.build_url(query) + try: + response = await self._client.get(url) + response.raise_for_status() + return self._parse_results(response.text, query, max_results) + except httpx.HTTPError as e: + logger.error(f"财联社 search failed: {e}") + return [] + + def _parse_results( + self, html: str, query: str, max_results: int + ) -> list[SearchResult]: + """Parse 财联社 search page.""" + soup = BeautifulSoup(html, "lxml") + results = [] + + # cls.cn typically uses dynamic rendering, so parsing may be limited + articles = soup.select( + "a[href*='/detail'], .news-item, .search-result-item, article" + ) + for i, article in enumerate(articles[:max_results]): + title_el = article.select_one("h3, h4, .title, .news-title") + link = article.get("href", "") + snippet_el = article.select_one( + ".summary, .desc, .abstract, p" + ) + + title = title_el.get_text(strip=True) if title_el else "" + snippet = snippet_el.get_text(strip=True) if snippet_el else "" + + if not title and not link: + continue + + full_url = link if link.startswith("http") else f"https://www.cls.cn{link}" + + results.append( + SearchResult( + title=title or f"财联社: {query}", + url=full_url, + snippet=snippet or f"Search result from 财联社", + source=self.name, + rank=i + 1, + category="news", + extra={"region": "cn"}, + ) + ) + + if not results: + # Fallback: return URL for manual browsing + search_url = self.build_url(query) + results.append( + SearchResult( + title=f"财联社: {query}", + url=search_url, + snippet=f"Search 财联社 for '{query}' (site requires JS)", + source=self.name, + rank=1, + category="news", + extra={"search_url": search_url, "note": "JS-rendered page"}, + ) + ) + + return results + + async def close(self): + await self._client.aclose() diff --git a/scripts/news/rss.py b/scripts/news/rss.py new file mode 100644 index 0000000..66f9e87 --- /dev/null +++ b/scripts/news/rss.py @@ -0,0 +1,288 @@ +""" +RSS news aggregation engine. + +Fetches and parses RSS/Atom feeds from trusted news sources. +Inspired by the news-summary skill's RSS-based approach. + +Supported sources: +- BBC (World, Business, Technology, Top Stories) +- Reuters (World News) +- NPR (US news) +- Al Jazeera (Global South perspective) +- 财联社 (RSS if available) +- 华尔街见闻 (RSS if available) +""" + +from __future__ import annotations + +import logging +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from typing import Any, Optional +from urllib.parse import urlparse + +import httpx + +from scripts.core.base import SearchEngine, SearchResult + +logger = logging.getLogger(__name__) + +# RSS feed definitions (from news-summary skill + additional sources) +RSS_FEEDS = { + "BBC World": { + "url": "https://feeds.bbci.co.uk/news/world/rss.xml", + "category": "news", + "lang": "en", + }, + "BBC Top Stories": { + "url": "https://feeds.bbci.co.uk/news/rss.xml", + "category": "news", + "lang": "en", + }, + "BBC Business": { + "url": "https://feeds.bbci.co.uk/news/business/rss.xml", + "category": "business", + "lang": "en", + }, + "BBC Technology": { + "url": "https://feeds.bbci.co.uk/news/technology/rss.xml", + "category": "technology", + "lang": "en", + }, + "Reuters": { + "url": "https://www.reutersagency.com/feed/?best-regions=world&post_type=best", + "category": "news", + "lang": "en", + }, + "NPR": { + "url": "https://feeds.npr.org/1001/rss.xml", + "category": "news", + "lang": "en", + }, + "Al Jazeera": { + "url": "https://www.aljazeera.com/xml/rss/all.xml", + "category": "news", + "lang": "en", + }, +} + + +class RSSNewsEngine(SearchEngine): + """ + RSS-based news aggregation engine. + + Fetches and parses RSS/Atom feeds from international news sources. + Supports filtering by category, keyword, and time range. + """ + + def __init__(self, config: Optional[dict] = None): + super().__init__("RSS News", config) + timeout = (config or {}).get("timeout", 30) + self._client = httpx.AsyncClient( + timeout=timeout, + headers={ + "User-Agent": (config or {}).get( + "user_agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36", + ), + "Accept": "application/rss+xml, application/xml, text/xml, */*", + }, + follow_redirects=True, + ) + + def build_url(self, query: str, **kwargs) -> str: + """Build search URL - for RSS this is not a search URL but a feed URL.""" + feed_name = kwargs.get("feed", "BBC World") + feed = RSS_FEEDS.get(feed_name, RSS_FEEDS["BBC World"]) + return feed["url"] + + async def search(self, query: str, max_results: int = 10, **kwargs) -> list[SearchResult]: + """ + Search news across RSS feeds. If query is empty, returns latest headlines. + + Args: + query: Search keyword to filter articles (case-insensitive). + If empty/blank, returns all headlines. + max_results: Maximum articles per feed. + feeds: Specific feeds to query (list of feed names or "all"). + category: Filter by category (news, business, technology). + hours: Only include articles from the last N hours. + + Returns: + List of SearchResult objects sorted by date (newest first). + """ + feeds_to_query = self._select_feeds( + feed_names=kwargs.get("feeds", "all"), + category=kwargs.get("category"), + ) + + results = [] + for feed_name, feed_info in feeds_to_query.items(): + try: + response = await self._client.get(feed_info["url"]) + response.raise_for_status() + + parsed = self._parse_feed(response.text, feed_name, feed_info) + for article in parsed: + # Filter by keyword if query provided + if query: + q = query.lower() + if q in article.title.lower() or q in article.snippet.lower(): + results.append(article) + else: + results.append(article) + + except httpx.HTTPError as e: + logger.warning(f"RSS feed '{feed_name}' failed: {e}") + continue + + # Apply time filter + hours = kwargs.get("hours") + if hours: + from datetime import timedelta + cutoff = datetime.now(timezone.utc) - timedelta(hours=int(hours)) + results = [r for r in results if r.extra.get("parsed_date", datetime.max.replace(tzinfo=timezone.utc)) >= cutoff] + + # Sort by date (newest first), limit results + results.sort(key=lambda r: r.extra.get("parsed_date", datetime.min.replace(tzinfo=timezone.utc)), reverse=True) + return results[:max_results * max(1, len(feeds_to_query))] + + def _select_feeds(self, feed_names="all", category=None): + """Select feeds based on filters.""" + if feed_names != "all": + if isinstance(feed_names, str): + feed_names = [feed_names] + return {k: v for k, v in RSS_FEEDS.items() if k in feed_names} + + if category: + return {k: v for k, v in RSS_FEEDS.items() if v.get("category") == category} + + return RSS_FEEDS + + def _parse_feed(self, xml_text: str, feed_name: str, feed_info: dict) -> list[SearchResult]: + """Parse RSS/Atom XML into SearchResult objects.""" + results = [] + + try: + root = ET.fromstring(xml_text) + except ET.ParseError as e: + logger.error(f"Failed to parse RSS from {feed_name}: {e}") + return [] + + # Detect RSS vs Atom format + if root.tag == "rss": + return self._parse_rss(root, feed_name, feed_info) + elif root.tag.endswith("feed"): + return self._parse_atom(root, feed_name, feed_info) + else: + logger.warning(f"Unknown feed format for {feed_name}: {root.tag}") + return results + + def _parse_rss(self, root: ET.Element, feed_name: str, feed_info: dict) -> list[SearchResult]: + """Parse RSS 2.0 format.""" + results = [] + namespaces = {"content": "http://purl.org/rss/1.0/modules/content/"} + + for i, item in enumerate(root.findall(".//item")): + title = item.findtext("title", "") + link = item.findtext("link", "") + description = item.findtext("description", "") + pub_date_str = item.findtext("pubDate", "") + creator = item.findtext("dc:creator", "") or item.findtext("author", "") + + # Clean up description (remove HTML tags) + import re + snippet = re.sub(r"<[^>]+>", "", description).strip() + # Truncate + snippet = snippet[:300] + "..." if len(snippet) > 300 else snippet + + # Parse date + parsed_date = self._parse_date(pub_date_str) + + results.append( + SearchResult( + title=title, + url=link, + snippet=snippet, + source=feed_name, + rank=i + 1, + category=feed_info.get("category", "news"), + extra={ + "feed_name": feed_name, + "author": creator, + "published": pub_date_str, + "parsed_date": parsed_date, + "feed_url": feed_info["url"], + }, + ) + ) + + return results + + def _parse_atom(self, root: ET.Element, feed_name: str, feed_info: dict) -> list[SearchResult]: + """Parse Atom format.""" + results = [] + ns = {"a": "http://www.w3.org/2005/Atom"} + + feed_title = root.findtext("a:title", "", ns) + + for i, entry in enumerate(root.findall("a:entry", ns)): + title = entry.findtext("a:title", "", ns) + link_el = entry.find("a:link", ns) + link = link_el.get("href", "") if link_el is not None else "" + summary = entry.findtext("a:summary", "", ns) or entry.findtext("a:content", "", ns) + published = entry.findtext("a:published", "", ns) or entry.findtext("a:updated", "", ns) + author = entry.findtext("a:author/a:name", "", ns) + + import re + snippet = re.sub(r"<[^>]+>", "", summary).strip() + snippet = snippet[:300] + "..." if len(snippet) > 300 else snippet + + parsed_date = self._parse_date(published) + + results.append( + SearchResult( + title=title, + url=link, + snippet=snippet, + source=feed_name, + rank=i + 1, + category=feed_info.get("category", "news"), + extra={ + "feed_name": feed_name, + "author": author, + "published": published, + "parsed_date": parsed_date, + "feed_url": feed_info["url"], + }, + ) + ) + + return results + + def _parse_date(self, date_str: str) -> datetime: + """Parse various date formats to datetime.""" + if not date_str: + return datetime.min.replace(tzinfo=timezone.utc) + + formats = [ + "%a, %d %b %Y %H:%M:%S %z", # RSS: Sat, 01 Jan 2024 12:00:00 +0000 + "%a, %d %b %Y %H:%M:%S %Z", # RSS with timezone name + "%Y-%m-%dT%H:%M:%S%z", # Atom ISO 8601 + "%Y-%m-%dT%H:%M:%S.%f%z", # Atom with microseconds + "%Y-%m-%dT%H:%M:%SZ", # Atom UTC + "%Y-%m-%d", # Simple date + ] + + for fmt in formats: + try: + return datetime.strptime(date_str.strip(), fmt).replace(tzinfo=timezone.utc) + except (ValueError, TypeError): + continue + + return datetime.min.replace(tzinfo=timezone.utc) + + async def close(self): + await self._client.aclose() diff --git a/scripts/news/wallstreetcn.py b/scripts/news/wallstreetcn.py new file mode 100644 index 0000000..2f8b889 --- /dev/null +++ b/scripts/news/wallstreetcn.py @@ -0,0 +1,117 @@ +""" +华尔街见闻 (wallstreetcn.com) news search engine. + +Scrapes wallstreetcn.com search results for financial news. +""" + +from __future__ import annotations + +import logging +from typing import Optional +from urllib.parse import quote_plus + +import httpx +from bs4 import BeautifulSoup + +from scripts.core.base import SearchEngine, SearchResult + +logger = logging.getLogger(__name__) + + +class WallStreetCNEngine(SearchEngine): + """华尔街见闻 (wallstreetcn.com) news search engine.""" + + SEARCH_URL = "https://wallstreetcn.com/search" + + def __init__(self, config: Optional[dict] = None): + super().__init__("wallstreetcn", config) + timeout = (config or {}).get("timeout", 15) + self._client = httpx.AsyncClient( + timeout=timeout, + headers={ + "User-Agent": (config or {}).get( + "user_agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36", + ), + "Referer": "https://wallstreetcn.com/", + }, + follow_redirects=True, + ) + + def build_url(self, query: str, **kwargs) -> str: + return f"{self.SEARCH_URL}?q={quote_plus(query)}" + + async def search( + self, query: str, max_results: int = 10, **kwargs + ) -> list[SearchResult]: + """Search 华尔街见闻 and parse results.""" + url = self.build_url(query) + try: + response = await self._client.get(url) + response.raise_for_status() + return self._parse_results(response.text, query, max_results) + except httpx.HTTPError as e: + logger.error(f"华尔街见闻 search failed: {e}") + return [] + + def _parse_results( + self, html: str, query: str, max_results: int + ) -> list[SearchResult]: + """Parse 华尔街见闻 search page.""" + soup = BeautifulSoup(html, "lxml") + results = [] + + articles = soup.select( + "a[href*='/article/'], .search-result-item, article, .item" + ) + for i, article in enumerate(articles[:max_results]): + title_el = article.select_one("h3, h4, .title, a") + link = article.get("href", "") + snippet_el = article.select_one( + ".summary, .desc, .abstract, p" + ) + + title = title_el.get_text(strip=True) if title_el else "" + snippet = snippet_el.get_text(strip=True) if snippet_el else "" + + if not title and not link: + continue + + full_url = ( + link + if link.startswith("http") + else f"https://wallstreetcn.com{link}" + ) + + results.append( + SearchResult( + title=title or f"华尔街见闻: {query}", + url=full_url, + snippet=snippet or f"Search result from 华尔街见闻", + source=self.name, + rank=i + 1, + category="news", + extra={"region": "cn"}, + ) + ) + + if not results: + search_url = self.build_url(query) + results.append( + SearchResult( + title=f"华尔街见闻: {query}", + url=search_url, + snippet=f"Search 华尔街见闻 for '{query}' (site may require JS)", + source=self.name, + rank=1, + category="news", + extra={"search_url": search_url, "note": "JS-rendered page"}, + ) + ) + + return results + + async def close(self): + await self._client.aclose() diff --git a/scripts/social/__init__.py b/scripts/social/__init__.py new file mode 100644 index 0000000..90f7f10 --- /dev/null +++ b/scripts/social/__init__.py @@ -0,0 +1 @@ +"""Social media search implementations (Twitter/X, etc.).""" diff --git a/scripts/social/twitter.py b/scripts/social/twitter.py new file mode 100644 index 0000000..c72f5ea --- /dev/null +++ b/scripts/social/twitter.py @@ -0,0 +1,208 @@ +""" +Twitter/X search module. + +Generates search URLs for X.com (formerly Twitter) search and +optionally parses search result pages. +""" + +from __future__ import annotations + +import logging +from typing import Optional +from urllib.parse import quote_plus + +import httpx +from bs4 import BeautifulSoup + +from scripts.core.base import SearchEngine, SearchResult + +logger = logging.getLogger(__name__) + + +class TwitterSearchEngine(SearchEngine): + """ + Twitter/X search engine. + + Provides search URL generation and optional result scraping. + Supports basic keyword search, user search, and advanced queries. + """ + + SEARCH_URL = "https://twitter.com/search" + BASE_URL = "https://twitter.com" + + def __init__(self, config: Optional[dict] = None): + super().__init__("Twitter", config) + timeout = (config or {}).get("timeout", 15) + self._client = httpx.AsyncClient( + timeout=timeout, + headers={ + "User-Agent": (config or {}).get( + "user_agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36", + ), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + }, + follow_redirects=True, + ) + + def build_url(self, query: str, search_type: str = "top", **kwargs) -> str: + """ + Build Twitter/X search URL. + + Args: + query: Search keyword. + search_type: "top", "latest", "people", "photos", "videos" + **kwargs: Additional URL parameters. + + Returns: + Search URL string. + """ + encoded_query = quote_plus(query) + url = f"{self.SEARCH_URL}?q={encoded_query}&src=typed_query" + + if search_type == "latest": + url += "&f=live" + elif search_type == "people": + url += "&f=user" + elif search_type == "photos": + url += "&f=image" + elif search_type == "videos": + url += "&f=video" + + if kwargs.get("lang"): + url += f"&l={kwargs['lang']}" + if kwargs.get("until"): + url += f"&until={kwargs['until']}" + if kwargs.get("since"): + url += f"&since={kwargs['since']}" + + return url + + def build_advanced_url(self, **params) -> str: + """ + Build advanced Twitter search URL with operators. + + Supported operators: + from:username - Tweets from user + to:username - Tweets to user + @username - Mentions + #hashtag - Hashtag + filter:links - Tweets with links + filter:images - Tweets with images + min_retweets:N - Minimum retweets + min_faves:N - Minimum likes + lang:en - Language + + Returns: + Advanced search URL. + """ + parts = [] + if params.get("from_user"): + parts.append(f"from:{params['from_user']}") + if params.get("to_user"): + parts.append(f"to:{params['to_user']}") + if params.get("hashtag"): + parts.append(f"#{params['hashtag']}") + if params.get("keyword"): + parts.append(params["keyword"]) + if params.get("min_retweets"): + parts.append(f"min_retweets:{params['min_retweets']}") + if params.get("min_faves"): + parts.append(f"min_faves:{params['min_faves']}") + if params.get("lang"): + parts.append(f"lang:{params['lang']}") + if params.get("filter"): + parts.append(f"filter:{params['filter']}") + + query = " ".join(parts) + return self.build_url(query, search_type=params.get("search_type", "top")) + + async def search(self, query: str, max_results: int = 10, search_type: str = "top", **kwargs) -> list[SearchResult]: + """ + Search Twitter/X and optionally parse page content. + + Args: + query: Search keyword. + max_results: Maximum results (used for URL generation). + search_type: "top", "latest", "people", "photos", "videos" + **kwargs: Additional options. + + Returns: + List of SearchResult objects. + """ + url = self.build_url(query, search_type=search_type, **kwargs) + + # Try to scrape the search page for basic info + try: + response = await self._client.get(url) + if response.status_code == 200: + soup = BeautifulSoup(response.text, "lxml") + results = self._parse_page(soup, max_results) + if results: + return results + except Exception as e: + logger.debug(f"Twitter page scraping failed (expected): {e}") + + # Fall back to URL-based result + result_type_map = { + "top": "Top tweets", + "latest": "Latest tweets", + "people": "People", + "photos": "Photos", + "videos": "Videos", + } + desc = result_type_map.get(search_type, "Tweets") + + return [ + SearchResult( + title=f"Twitter: {query} ({desc})", + url=url, + snippet=f"Search Twitter/X for '{query}'. Note: Twitter may require login.", + source=self.name, + rank=1, + category="social", + extra={"search_type": search_type, "twitter_url": url}, + ) + ] + + def _parse_page(self, soup, max_results: int) -> list[SearchResult]: + """Attempt to extract tweet info from search page.""" + results = [] + # Twitter uses JS rendering, so HTML parsing may yield limited results + tweets = soup.select("article[data-testid='tweet'], div.tweet, .content") + for i, tweet in enumerate(tweets[:max_results]): + text_el = tweet.select_one( + "[data-testid='tweetText'], .tweet-text, p[lang]" + ) + link_el = tweet.select_one("a[href*='/status/'], time") + user_el = tweet.select_one( + "[data-testid='User-Name'], .username, .fullname" + ) + + text = text_el.get_text(strip=True) if text_el else "" + user = user_el.get_text(strip=True) if user_el else "" + + tweet_url = "" + if link_el: + href = link_el.get("href", "") + tweet_url = f"{self.BASE_URL}{href}" if href.startswith("/") else href + + if text: + results.append( + SearchResult( + title=f"@{user}: {text[:80]}..." if user else text[:80], + url=tweet_url or self.build_url(text[:30]), + snippet=text, + source=self.name, + rank=i + 1, + category="social", + extra={"user": user} if user else {}, + ) + ) + return results + + async def close(self): + await self._client.aclose() diff --git a/scripts/unified_search.py b/scripts/unified_search.py new file mode 100644 index 0000000..91b5b2d --- /dev/null +++ b/scripts/unified_search.py @@ -0,0 +1,183 @@ +""" +Unified search entry point. + +Provides a single interface to search across all configured sources. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Any, Optional + +from scripts.core.base import SearchResult +from scripts.core.config_loader import ( + get_search_engines, get_news_sources, get_academic_sources, + get_wechat_sources, get_special_engines, get_social_sources, + get_all_sources, get_engine, +) +from scripts.engines.url_engine import UrlSearchEngine +from scripts.engines.parser_engines import DuckDuckGoEngine +from scripts.news.cls import CLSNewsEngine +from scripts.news.wallstreetcn import WallStreetCNEngine +from scripts.news.rss import RSSNewsEngine +from scripts.academic.arxiv import ArXivEngine +from scripts.wechat.sogou_weixin import SogouWeChatEngine +from scripts.social.twitter import TwitterSearchEngine + +logger = logging.getLogger(__name__) + +SOURCE_CATEGORIES = { + "web": get_search_engines, + "news": get_news_sources, + "academic": get_academic_sources, + "wechat": get_wechat_sources, + "social": get_social_sources, + "special": get_special_engines, +} + + +class UnifiedSearch: + """Unified search client that aggregates results from multiple sources.""" + + def __init__(self, config: Optional[dict[str, Any]] = None): + self.config = config or {} + self._cache: dict[str, list[SearchResult]] = {} + self._engines: dict[str, Any] = {} + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + await self.close() + + async def search( + self, query: str, sources: Optional[list[str]] = None, + max_results: int = 10, force_refresh: bool = False, **kwargs, + ) -> list[SearchResult]: + """Search across selected sources and aggregate results.""" + if not sources or "all" in sources: + sources = list(SOURCE_CATEGORIES.keys()) + + cache_key = f"{sorted(sources) if isinstance(sources, list) else sources}:{query}" + if not force_refresh and cache_key in self._cache: + return self._cache[cache_key] + + tasks = [] + for source_spec in sources: + if source_spec in SOURCE_CATEGORIES: + for engine_def in SOURCE_CATEGORIES[source_spec](): + engine = self._get_engine(engine_def) + if engine: + tasks.append(self._safe_search(engine, query, max_results, **kwargs)) + else: + engine_def = get_engine(source_spec) + if engine_def: + engine = self._get_engine(engine_def) + if engine: + tasks.append(self._safe_search(engine, query, max_results, **kwargs)) + + if not tasks: + return [] + + results_lists = await asyncio.gather(*tasks, return_exceptions=True) + all_results: list[SearchResult] = [] + for results in results_lists: + if isinstance(results, Exception): + logger.error(f"Search task failed: {results}") + continue + all_results.extend(results) + + self._cache[cache_key] = all_results + return all_results + + def _get_engine(self, engine_def: dict[str, Any]): + """Get or create an engine instance by definition.""" + name = engine_def.get("name", "") + engine_type = engine_def.get("type", "url") + + if name in self._engines: + return self._engines[name] + + if name == "DuckDuckGo": + engine = DuckDuckGoEngine(self.config) + elif "cls" in name.lower(): + engine = CLSNewsEngine(self.config) + elif "wallstreet" in name.lower() or "华尔街" in name: + engine = WallStreetCNEngine(self.config) + elif name == "ArXiv" or engine_type == "api": + engine = ArXivEngine(self.config) + elif "微信" in name or "wechat" in name.lower() or "weixin" in name.lower(): + engine = SogouWeChatEngine(self.config) + elif "twitter" in name.lower(): + engine = TwitterSearchEngine(self.config) + elif engine_type == "news" and name not in ("cls", "wallstreetcn"): + engine = RSSNewsEngine(self.config) + elif engine_type == "social": + engine = TwitterSearchEngine(self.config) if "twitter" in name.lower() else UrlSearchEngine(name, engine_def) + else: + engine = UrlSearchEngine(name, engine_def) + + self._engines[name] = engine + return engine + + async def _safe_search(self, engine, query, max_results, **kwargs): + try: + return await engine.search(query, max_results=max_results, **kwargs) + except Exception as e: + logger.error(f"Engine {engine.name} failed: {e}") + return [] + + async def search_engines(self, query, region=None, max_results=10): + """Quick search across web search engines.""" + engines = get_search_engines() + if region: + engines = [e for e in engines if e.get("region") == region] + return await self.search(query, sources=[e["name"] for e in engines], max_results=max_results) + + async def search_news(self, query, max_results=10, **kwargs): + """Quick search news sources (CN news + RSS feeds).""" + return await self.search(query, sources=["news"], max_results=max_results, **kwargs) + + async def search_academic(self, query, max_results=10, **kwargs): + """Quick search academic papers.""" + return await self.search(query, sources=["academic"], max_results=max_results, **kwargs) + + async def search_wechat(self, query, max_results=10): + """Quick search WeChat articles.""" + return await self.search(query, sources=["wechat"], max_results=max_results) + + async def search_social(self, query, max_results=10, **kwargs): + """Quick search social media (Twitter/X).""" + return await self.search(query, sources=["social"], max_results=max_results, **kwargs) + + async def search_rss(self, query="", max_results=10, **kwargs): + """Quick search RSS news feeds.""" + engine = RSSNewsEngine(self.config) + return await engine.search(query, max_results=max_results, **kwargs) + + async def get_search_urls(self, query, sources=None): + """Get search URLs without executing searches.""" + if not sources: + sources = [e["name"] for e in get_search_engines()] + + urls = {} + for name in sources: + engine_def = get_engine(name) + if engine_def: + urls[name] = UrlSearchEngine(name, engine_def).build_url(query) + + return urls + + async def get_twitter_search_url(self, query, search_type="top"): + """Get Twitter/X search URL for browser/web_fetch usage.""" + engine = TwitterSearchEngine(self.config) + return engine.build_url(query, search_type=search_type) + + async def close(self): + """Close all engine resources.""" + for engine in self._engines.values(): + try: + await engine.close() + except Exception: + pass diff --git a/scripts/wechat/__init__.py b/scripts/wechat/__init__.py new file mode 100644 index 0000000..93f7549 --- /dev/null +++ b/scripts/wechat/__init__.py @@ -0,0 +1 @@ +"""WeChat search engines.""" diff --git a/scripts/wechat/sogou_weixin.py b/scripts/wechat/sogou_weixin.py new file mode 100644 index 0000000..31cf1b5 --- /dev/null +++ b/scripts/wechat/sogou_weixin.py @@ -0,0 +1,325 @@ +""" +Sogou WeChat (搜狗微信) search engine. + +Searches WeChat public account articles via weixin.sogou.com. + +Anti-scraping strategy (inspired by Node.js wechat-article-search skill): +- Cookie pre-fetch from v.sogou.com to get SNUID +- User-Agent rotation with a pool of real browser UAs +- Random delays between requests (500-1500ms) +- Retry logic with exponential backoff +""" + +from __future__ import annotations + +import asyncio +import logging +import random +import re +from typing import Optional +from urllib.parse import quote_plus + +import httpx +from bs4 import BeautifulSoup + +from scripts.core.base import SearchEngine, SearchResult + +logger = logging.getLogger(__name__) + +# Real browser User-Agent pool for rotation +USER_AGENTS = [ + # Chrome + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + # Safari + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15", + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1", + # Firefox + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0", + # Edge + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0", +] + + +class SogouWeChatEngine(SearchEngine): + """搜狗微信搜索 - searches WeChat public account articles.""" + + SEARCH_URL = "https://weixin.sogou.com/weixin" + COOKIE_URL = "https://v.sogou.com" + + def __init__(self, config: Optional[dict] = None): + super().__init__("weixin", config) + timeout = (config or {}).get("timeout", 15) + self._ua = random.choice(USER_AGENTS) + self._cookies: dict[str, str] = {} + self._client = httpx.AsyncClient( + timeout=timeout, + headers=self._build_headers(), + follow_redirects=False, # Don't auto-follow to detect anti-scraping + ) + + def _build_headers(self) -> dict[str, str]: + return { + "User-Agent": self._ua, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Accept-Encoding": "gzip, deflate", + "Referer": "https://weixin.sogou.com/", + "DNT": "1", + "Connection": "keep-alive", + } + + def _random_delay(self): + """Random delay between 500-1500ms to mimic human browsing.""" + delay = random.uniform(0.5, 1.5) + return asyncio.sleep(delay) + + def _rotate_ua(self): + """Rotate to a new random User-Agent.""" + self._ua = random.choice(USER_AGENTS) + + async def _get_cookie(self) -> dict[str, str]: + """Pre-fetch cookie from v.sogou.com to get SNUID.""" + try: + response = await self._client.get( + self.COOKIE_URL, + headers={ + "User-Agent": self._ua, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "zh-CN,zh;q=0.9", + }, + ) + cookies = {} + for cookie in response.cookies.jar: + if isinstance(cookie, tuple): + # SimpleCookie format: (key, value) + cookies[cookie[0]] = cookie[1] + else: + cookies[cookie.name] = cookie.value + logger.debug(f"Got {len(cookies)} cookies from v.sogou.com") + return cookies + except Exception as e: + logger.warning(f"Cookie pre-fetch failed: {e}") + return {} + + def build_url(self, query: str, search_type: int = 2, page: int = 1, **kwargs) -> str: + """ + Build Sogou WeChat search URL. + + Args: + query: Search keyword. + search_type: 2=articles, 1=accounts. + page: Page number (starts at 1). + """ + return ( + f"{self.SEARCH_URL}?query={quote_plus(query)}" + f"&s_from=input&_sug_=n&type={search_type}" + f"&page={page}&ie=utf8" + ) + + async def search( + self, query: str, max_results: int = 10, **kwargs + ) -> list[SearchResult]: + """Search WeChat articles via Sogou and parse results.""" + # Step 1: Get cookie first + if not self._cookies: + self._cookies = await self._get_cookie() + await self._random_delay() + + # Step 2: Search with retry + max_pages = min(5, (max_results + 9) // 10) + all_results = [] + + for page in range(1, max_pages + 1): + url = self.build_url(query, page=page, **kwargs) + results = await self._search_page(url, query, max_results - len(all_results)) + all_results.extend(results) + + if len(all_results) >= max_results: + break + + # Delay between pages + await self._random_delay() + + if not all_results: + # Fallback: try legacy wx.sogou.com + return await self._legacy_search(query, max_results, **kwargs) + + return all_results[:max_results] + + async def _search_page( + self, url: str, query: str, max_results: int + ) -> list[SearchResult]: + """Search a single page with retry logic.""" + for attempt in range(3): + try: + # Rotate UA on each retry + if attempt > 0: + self._rotate_ua() + self._client.headers.update({"User-Agent": self._ua}) + if self._cookies: + pass # Keep existing cookies + + response = await self._client.get( + url, + cookies=self._cookies, + follow_redirects=False, + ) + + # Check for anti-scraping page + if response.status_code == 302: + # Got redirected - likely hit anti-scraping + logger.warning(f"Sogou anti-scraping triggered (302), attempt {attempt + 1}") + # Try to get new cookie + self._cookies = await self._get_cookie() + await self._random_delay() + continue + + if response.status_code != 200: + logger.warning(f"Sogou returned {response.status_code}, attempt {attempt + 1}") + await self._random_delay() + continue + + html = response.text + + # Check for captcha/block page + if "请输入验证码" in html or "访问频率过高" in html: + logger.warning(f"Sogou captcha triggered, attempt {attempt + 1}") + await asyncio.sleep(random.uniform(2, 4)) + self._cookies = await self._get_cookie() + continue + + return self._parse_results(html, query, max_results) + + except (httpx.HTTPError, httpx.TimeoutException) as e: + logger.warning(f"Sogou search attempt {attempt + 1} failed: {e}") + await self._random_delay() + continue + + return [] + + async def _legacy_search( + self, query: str, max_results: int, **kwargs + ) -> list[SearchResult]: + """Fallback: try the legacy wx.sogou.com domain.""" + legacy_url = f"https://wx.sogou.com/weixin?type=2&query={quote_plus(query)}" + try: + response = await self._client.get( + legacy_url, + cookies=self._cookies, + follow_redirects=True, + ) + if response.status_code == 200: + return self._parse_results(response.text, query, max_results) + except Exception: + pass + + # Ultimate fallback + search_url = self.build_url(query) + return [ + SearchResult( + title=f"微信搜索: {query}", + url=search_url, + snippet=f"Search WeChat for '{query}' via Sogou", + source=self.name, + rank=1, + category="wechat", + extra={"search_url": search_url}, + ) + ] + + def _parse_results( + self, html: str, query: str, max_results: int + ) -> list[SearchResult]: + """Parse Sogou WeChat search results.""" + soup = BeautifulSoup(html, "lxml") + results = [] + + # Main selector: ul.news-list > li (standard Sogou WeChat structure) + items = soup.select("ul.news-list > li") + + if not items: + # Fallback selectors + items = soup.select( + ".news-list2 li, .wx-rb-item, .results .result, " + ".item, .news-box .news-list li" + ) + + for i, item in enumerate(items[:max_results]): + title_el = item.select_one("h3 a") + if not title_el: + title_el = item.select_one(".tit a, .title a, h3") + + snippet_el = item.select_one("p.txt-info, .txt-info, .summary, .des, p.info") + source_el = item.select_one( + ".all-time-y2 a.account, a.account, " + ".account, .source, .s-p, .from" + ) + + title = title_el.get_text(strip=True) if title_el else "" + link = title_el.get("href", "") if title_el and hasattr(title_el, "get") else "" + snippet = snippet_el.get_text(strip=True) if snippet_el else "" + account = source_el.get_text(strip=True) if source_el else "" + + # Parse time from inline scripts or text + time_text = self._parse_time(item) + + if not title: + continue + + # Make sure URL is absolute + if link and not link.startswith("http"): + link = f"https://weixin.sogou.com{link}" + + results.append( + SearchResult( + title=title, + url=link, + snippet=snippet or f"WeChat article from {account or 'unknown'}", + source=self.name, + rank=(i + 1), + category="wechat", + extra={ + "account": account, + "time": time_text, + "search_url": self.build_url(query), + }, + ) + ) + + return results + + def _parse_time(self, item) -> str: + """Extract publish time from search result item.""" + # Try script tag with Unix timestamp (Sogou encodes dates in scripts) + for script in item.select("script"): + text = script.string or "" + # Look for Unix timestamp pattern + match = re.search(r'\d{10}', text) + if match: + import datetime + ts = int(match.group()) + try: + dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone(datetime.timedelta(hours=8))) + return dt.strftime("%Y-%m-%d %H:%M") + except (OSError, ValueError): + pass + + # Try inline text with relative time like "2小时前", "3天前" + time_text = item.get_text() + relative_match = re.search(r'(\d+)\s*(小时|分钟|天|周|月)前', time_text) + if relative_match: + return f"{relative_match.group(1)}{relative_match.group(2)}前" + + # Try standard date patterns + date_match = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', time_text) + if date_match: + return date_match.group(1) + + return "" + + async def close(self): + await self._client.aclose() diff --git a/skill.md b/skill.md index dafebea..d4e71af 100644 --- a/skill.md +++ b/skill.md @@ -1,35 +1,144 @@ -# Web Search Skill +--- +name: web-search-skills +description: "多源搜索 CLI 工具,支持 20+ 搜索引擎。搜索微信公众号文章、新闻资讯、学术论文、社交媒体的综合搜索技能。当用户需要搜索网络信息、查找微信公众号文章、搜索财经新闻、查询学术论文时使用。" +version: 2.1.0 +license: MIT +compatibility: Requires Python 3.8+, httpx, beautifulsoup4, lxml +metadata: + author: openbot-coder + category: development + tags: [search, web, wechat, news, academic, cli] +--- -## Description +# Web Search Skills -Provides web search capabilities for AI agents, supporting multiple search engines, intelligent result processing, and customizable search parameters. +## 概述 -## Triggers +Multi-source web search CLI and library supporting 20+ search engines, including web search, news (财联社, 华尔街见闻), WeChat articles, academic papers (ArXiv), social media (Twitter/X), and RSS feeds. +## 触发词 Triggers + +中文触发词: +- "搜索微信公众号..." +- "搜一下微信公众号..." +- "查一下新闻..." +- "搜索新闻关于..." +- "搜论文..." +- "查学术资料..." +- "搜索一下..." + +English Triggers: - "search the web for..." - "find information about..." - "look up..." +- "search WeChat for..." +- "search news about..." - Any query requiring real-time web data -## Configuration +## 安装 Installation -```yaml -search: - default_engine: duckduckgo - max_results: 10 - timeout: 15 - user_agent: "WebSearchSkill/1.0" +```bash +cd web-search-skills +uv tool install -e . ``` -## Actions +安装后 `web-search` 命令全局可用。 -- `web_search(query, engine=None, max_results=10)` - Perform a web search -- `search_news(query, max_results=5)` - Search news sources -- `search_images(query, max_results=10)` - Search for images -- `extract_content(url)` - Extract readable content from a URL +## 命令 Commands -## Notes +| 命令 Commands | 说明 Description | +|---------------|------------------| +| `web-search search ` | 全源搜索 Search all sources | +| `web-search web ` | 网页搜索 Web search engines | +| `web-search news ` | 新闻搜索 News (财联社, 华尔街见闻) | +| `web-search wechat ` | 微信公众号搜索 WeChat articles | +| `web-search academic ` | 学术搜索 Academic papers (ArXiv) | +| `web-search social ` | 社交搜索 Social media (Twitter/X) | +| `web-search rss ` | RSS 源搜索 RSS feeds | +| `web-search sources` | 列出所有搜索源 List all sources | +| `web-search urls ` | 只生成搜索链接 Search URLs only | -- Respects robots.txt for web scraping -- Implements rate limiting to avoid API bans -- Caches results for repeated queries within the same session +### 通用选项 Options + +| 选项 | 说明 | +|------|------| +| `-n N` | 每源最大结果数 (默认: 10) | +| `-j` | JSON 格式输出 | +| `-o FILE` | 保存结果到 JSON 文件 | +| `-s NAME` | 指定来源名称 | +| `-r cn/global` | 区域过滤 (web 搜索) | +| `-v` | 调试日志 | + +## 项目结构 Structure + +``` +web-search-skills/ +├── scripts/ # Python 脚本目录 +│ ├── cli.py # CLI 入口 +│ ├── unified_search.py # 统一搜索编排器 +│ ├── core/ +│ │ ├── base.py # SearchResult & SearchEngine 基类 +│ │ └── config_loader.py# 从 JSON 加载搜索引擎配置 +│ ├── engines/ +│ │ ├── url_engine.py # URL 模板搜索引擎 +│ │ └── parser_engines.py# HTML 解析搜索 (DuckDuckGo) +│ ├── news/ +│ │ ├── cls.py # 财联社新闻引擎 +│ │ ├── wallstreetcn.py # 华尔街见闻引擎 +│ │ └── rss.py # RSS 聚合引擎 +│ ├── academic/ +│ │ └── arxiv.py # ArXiv 学术论文引擎 +│ ├── wechat/ +│ │ └── sogou_weixin.py # 搜狗微信搜索引擎 +│ └── social/ +│ └── twitter.py # Twitter/X 搜索引擎 +├── config/ +│ └── engines.json # 搜索引擎定义配置 +├── ws.py # 本地运行入口 +├── pyproject.toml # 包构建配置 +├── SKILL.md # 本技能定义文件 +├── LICENSE # MIT 许可证 +└── README.md # 项目文档 +``` + +## 配置 Configuration + +所有搜索引擎定义在 `config/engines.json` 中,支持 6 类源: + +| 类别 | 数量 | +|------|------| +| Web 搜索引擎 | 14 个 (Baidu, Google, DuckDuckGo 等) | +| 新闻源 | 2 个 (财联社, 华尔街见闻) | +| 学术源 | 1 个 (ArXiv) | +| 微信源 | 1 个 (搜狗微信) | +| 社交源 | 2 个 (Twitter, Twitter Latest) | +| 特殊源 | 1 个 (WolframAlpha) | + +每个引擎定义格式: +```json +{"name": "Baidu", "url": "https://www.baidu.com/s?wd={keyword}", "region": "cn", "type": "url"} +``` + +- `{keyword}` 会被替换为搜索关键词 +- `type: url` 表示只生成 URL,`type: parser` 表示会解析 HTML 提取结果 + +## Python API + +也可在代码中直接调用: + +```python +from scripts.unified_search import UnifiedSearch + +async def main(): + us = UnifiedSearch() + results = await us.search("量子计算", sources=["web", "news"], max_results=5) + for r in results: + print(f"[{r.rank}] {r.title}: {r.url}") + await us.close() +``` + +## 注意事项 Notes + +- 搜狗微信搜索有反爬限制,可能返回空结果 +- 财联社和华尔街见闻需要 JavaScript 渲染,`UnifiedSearch` 返回搜索页面链接 +- RSS 源需要稳定的网络连接 diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index c27d7bb..0000000 --- a/src/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Web Search Skills - A collection of web search tools for AI agents.""" - -__version__ = "0.1.0" -__author__ = "openbot-coder" diff --git a/src/web_search.py b/src/web_search.py deleted file mode 100644 index ebc90fb..0000000 --- a/src/web_search.py +++ /dev/null @@ -1,201 +0,0 @@ -""" -Core web search module providing multi-engine search capabilities. - -Supports DuckDuckGo, Google Custom Search, Bing, and extensible backends. -""" - -from __future__ import annotations - -import asyncio -import logging -from dataclasses import dataclass, field -from typing import Any, Optional - -import httpx -from bs4 import BeautifulSoup - -logger = logging.getLogger(__name__) - - -@dataclass -class SearchResult: - """Represents a single search result.""" - title: str - url: str - snippet: str - source: str = "" - rank: int = 0 - - -@dataclass -class SearchConfig: - """Configuration for web search.""" - engine: str = "duckduckgo" - max_results: int = 10 - timeout: int = 15 - user_agent: str = "WebSearchSkill/1.0" - proxies: dict[str, str] = field(default_factory=dict) - - -class WebSearch: - """ - Main web search class supporting multiple search backends. - - Usage: - searcher = WebSearch() - results = searcher.search("Python async programming") - """ - - def __init__(self, config: Optional[SearchConfig | dict[str, Any]] = None): - if config is None: - self.config = SearchConfig() - elif isinstance(config, dict): - self.config = SearchConfig(**config) - else: - self.config = config - - self._client = httpx.AsyncClient( - timeout=self.config.timeout, - headers={"User-Agent": self.config.user_agent}, - proxies=self.config.proxies if self.config.proxies else None, - ) - self._cache: dict[str, list[SearchResult]] = {} - - async def search( - self, - query: str, - engine: Optional[str] = None, - max_results: Optional[int] = None, - force_refresh: bool = False, - ) -> list[SearchResult]: - """ - Perform a web search with the given query. - - Args: - query: Search query string. - engine: Search engine to use (default: config.engine). - max_results: Maximum number of results (default: config.max_results). - force_refresh: Ignore cached results if True. - - Returns: - List of SearchResult objects. - """ - cache_key = f"{engine or self.config.engine}:{query}" - - if not force_refresh and cache_key in self._cache: - logger.debug(f"Returning cached results for: {query}") - return self._cache[cache_key] - - engine = engine or self.config.engine - max_results = max_results or self.config.max_results - - if engine == "duckduckgo": - results = await self._search_duckduckgo(query, max_results) - elif engine == "google": - results = await self._search_google(query, max_results) - elif engine == "bing": - results = await self._search_bing(query, max_results) - else: - raise ValueError(f"Unsupported search engine: {engine}") - - self._cache[cache_key] = results - return results - - async def _search_duckduckgo( - self, query: str, max_results: int - ) -> list[SearchResult]: - """Search using DuckDuckGo (no API key required).""" - url = "https://html.duckduckgo.com/html/" - params = {"q": query} - - try: - response = await self._client.post(url, data=params) - response.raise_for_status() - soup = BeautifulSoup(response.text, "lxml") - - results = [] - for i, result in enumerate(soup.select(".result")[:max_results]): - title_el = result.select_one(".result__title a") - snippet_el = result.select_one(".result__snippet") - - if title_el: - results.append( - SearchResult( - title=title_el.get_text(strip=True), - url=title_el.get("href", ""), - snippet=snippet_el.get_text(strip=True) - if snippet_el - else "", - source="duckduckgo", - rank=i + 1, - ) - ) - return results - except httpx.HTTPError as e: - logger.error(f"DuckDuckGo search failed: {e}") - return [] - - async def _search_google( - self, query: str, max_results: int - ) -> list[SearchResult]: - """Search using Google Custom Search API (requires API key).""" - logger.warning("Google Custom Search requires an API key.") - return [] - - async def _search_bing( - self, query: str, max_results: int - ) -> list[SearchResult]: - """Search using Bing (requires API key).""" - logger.warning("Bing Search requires an API key.") - return [] - - async def extract_content(self, url: str) -> Optional[str]: - """ - Extract readable text content from a URL. - - Args: - url: Target URL to extract content from. - - Returns: - Extracted text content, or None on failure. - """ - try: - response = await self._client.get(url, follow_redirects=True) - response.raise_for_status() - soup = BeautifulSoup(response.text, "lxml") - - # Remove non-content elements - for tag in soup(["script", "style", "nav", "footer", "header"]): - tag.decompose() - - return soup.get_text(separator="\n", strip=True) - except httpx.HTTPError as e: - logger.error(f"Failed to extract content from {url}: {e}") - return None - - async def close(self): - """Close the underlying HTTP client.""" - await self._client.aclose() - - async def __aenter__(self): - return self - - async def __aexit__(self, *args): - await self.close() - - -async def search_web( - query: str, - engine: str = "duckduckgo", - max_results: int = 10, -) -> list[SearchResult]: - """ - Convenience function for one-off web searches. - - Example: - results = await search_web("Python asyncio") - for r in results: - print(f"{r.title}: {r.url}") - """ - async with WebSearch() as searcher: - return await searcher.search(query, engine=engine, max_results=max_results) diff --git a/ws.py b/ws.py new file mode 100644 index 0000000..93a564f --- /dev/null +++ b/ws.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +""" +Web Search Skills CLI entry point. + +Usage: + python ws.py [options] + +Or install and run: + uv tool install -e . + ws [options] +""" +import sys +import os + +# Ensure scripts package is importable +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +if _SCRIPT_DIR not in sys.path: + sys.path.insert(0, _SCRIPT_DIR) + +from scripts.cli import main + +if __name__ == "__main__": + main()