From 24535ff3f274c64188ef990d2564d1db47ef1b1b Mon Sep 17 00:00:00 2001 From: openbot-coder Date: Thu, 14 May 2026 14:28:07 +0800 Subject: [PATCH] Add project structure: LICENSE, .gitignore, skill definition, and core search module --- .gitignore | 37 +++++++ LICENSE | 21 ++++ examples/__init__.py | 1 + examples/basic_search.py | 36 +++++++ requirements.txt | 15 +++ skill.md | 35 +++++++ src/__init__.py | 4 + src/web_search.py | 201 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 350 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 examples/__init__.py create mode 100644 examples/basic_search.py create mode 100644 requirements.txt create mode 100644 skill.md create mode 100644 src/__init__.py create mode 100644 src/web_search.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9ba2d8b --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +env/ +venv/ +.venv/ +*.egg-info/ +*.egg + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Environment +.env +*.env.local + +# Logs +*.log + +# Build +dist/ +build/ +*.whl + +# Config with secrets +config.local.yaml diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7f0c384 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 openbot-coder + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..88f1d65 --- /dev/null +++ b/examples/__init__.py @@ -0,0 +1 @@ +"""Examples for using Web Search Skills.""" diff --git a/examples/basic_search.py b/examples/basic_search.py new file mode 100644 index 0000000..78c285e --- /dev/null +++ b/examples/basic_search.py @@ -0,0 +1,36 @@ +""" +Basic usage example for Web Search Skills. + +Run: + python examples/basic_search.py +""" + +import asyncio + +from src.web_search import WebSearch + + +async def main(): + async with WebSearch() as searcher: + results = await searcher.search( + "Python web scraping best practices 2026", + max_results=5, + ) + + print(f"Found {len(results)} results:\n") + for result in results: + print(f"[{result.rank}] {result.title}") + print(f" URL: {result.url}") + print(f" {result.snippet[:120]}...") + print() + + if results: + content = await searcher.extract_content(results[0].url) + if content: + print(f"--- Content from {results[0].url} ---") + print(content[:300]) + print("...") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..699ad0b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +# Web Search Skills Dependencies + +# Core +httpx>=0.25.0 +beautifulsoup4>=4.12.0 +lxml>=4.9.0 + +# Optional: Google Custom Search +# google-api-python-client>=2.100.0 + +# Development +pytest>=7.0.0 +pytest-asyncio>=0.21.0 +black>=23.0.0 +ruff>=0.1.0 diff --git a/skill.md b/skill.md new file mode 100644 index 0000000..dafebea --- /dev/null +++ b/skill.md @@ -0,0 +1,35 @@ +# Web Search Skill + +## Description + +Provides web search capabilities for AI agents, supporting multiple search engines, intelligent result processing, and customizable search parameters. + +## Triggers + +- "search the web for..." +- "find information about..." +- "look up..." +- Any query requiring real-time web data + +## Configuration + +```yaml +search: + default_engine: duckduckgo + max_results: 10 + timeout: 15 + user_agent: "WebSearchSkill/1.0" +``` + +## Actions + +- `web_search(query, engine=None, max_results=10)` - Perform a web search +- `search_news(query, max_results=5)` - Search news sources +- `search_images(query, max_results=10)` - Search for images +- `extract_content(url)` - Extract readable content from a URL + +## Notes + +- Respects robots.txt for web scraping +- Implements rate limiting to avoid API bans +- Caches results for repeated queries within the same session diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..c27d7bb --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,4 @@ +"""Web Search Skills - A collection of web search tools for AI agents.""" + +__version__ = "0.1.0" +__author__ = "openbot-coder" diff --git a/src/web_search.py b/src/web_search.py new file mode 100644 index 0000000..ebc90fb --- /dev/null +++ b/src/web_search.py @@ -0,0 +1,201 @@ +""" +Core web search module providing multi-engine search capabilities. + +Supports DuckDuckGo, Google Custom Search, Bing, and extensible backends. +""" + +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass, field +from typing import Any, Optional + +import httpx +from bs4 import BeautifulSoup + +logger = logging.getLogger(__name__) + + +@dataclass +class SearchResult: + """Represents a single search result.""" + title: str + url: str + snippet: str + source: str = "" + rank: int = 0 + + +@dataclass +class SearchConfig: + """Configuration for web search.""" + engine: str = "duckduckgo" + max_results: int = 10 + timeout: int = 15 + user_agent: str = "WebSearchSkill/1.0" + proxies: dict[str, str] = field(default_factory=dict) + + +class WebSearch: + """ + Main web search class supporting multiple search backends. + + Usage: + searcher = WebSearch() + results = searcher.search("Python async programming") + """ + + def __init__(self, config: Optional[SearchConfig | dict[str, Any]] = None): + if config is None: + self.config = SearchConfig() + elif isinstance(config, dict): + self.config = SearchConfig(**config) + else: + self.config = config + + self._client = httpx.AsyncClient( + timeout=self.config.timeout, + headers={"User-Agent": self.config.user_agent}, + proxies=self.config.proxies if self.config.proxies else None, + ) + self._cache: dict[str, list[SearchResult]] = {} + + async def search( + self, + query: str, + engine: Optional[str] = None, + max_results: Optional[int] = None, + force_refresh: bool = False, + ) -> list[SearchResult]: + """ + Perform a web search with the given query. + + Args: + query: Search query string. + engine: Search engine to use (default: config.engine). + max_results: Maximum number of results (default: config.max_results). + force_refresh: Ignore cached results if True. + + Returns: + List of SearchResult objects. + """ + cache_key = f"{engine or self.config.engine}:{query}" + + if not force_refresh and cache_key in self._cache: + logger.debug(f"Returning cached results for: {query}") + return self._cache[cache_key] + + engine = engine or self.config.engine + max_results = max_results or self.config.max_results + + if engine == "duckduckgo": + results = await self._search_duckduckgo(query, max_results) + elif engine == "google": + results = await self._search_google(query, max_results) + elif engine == "bing": + results = await self._search_bing(query, max_results) + else: + raise ValueError(f"Unsupported search engine: {engine}") + + self._cache[cache_key] = results + return results + + async def _search_duckduckgo( + self, query: str, max_results: int + ) -> list[SearchResult]: + """Search using DuckDuckGo (no API key required).""" + url = "https://html.duckduckgo.com/html/" + params = {"q": query} + + try: + response = await self._client.post(url, data=params) + response.raise_for_status() + soup = BeautifulSoup(response.text, "lxml") + + results = [] + for i, result in enumerate(soup.select(".result")[:max_results]): + title_el = result.select_one(".result__title a") + snippet_el = result.select_one(".result__snippet") + + if title_el: + results.append( + SearchResult( + title=title_el.get_text(strip=True), + url=title_el.get("href", ""), + snippet=snippet_el.get_text(strip=True) + if snippet_el + else "", + source="duckduckgo", + rank=i + 1, + ) + ) + return results + except httpx.HTTPError as e: + logger.error(f"DuckDuckGo search failed: {e}") + return [] + + async def _search_google( + self, query: str, max_results: int + ) -> list[SearchResult]: + """Search using Google Custom Search API (requires API key).""" + logger.warning("Google Custom Search requires an API key.") + return [] + + async def _search_bing( + self, query: str, max_results: int + ) -> list[SearchResult]: + """Search using Bing (requires API key).""" + logger.warning("Bing Search requires an API key.") + return [] + + async def extract_content(self, url: str) -> Optional[str]: + """ + Extract readable text content from a URL. + + Args: + url: Target URL to extract content from. + + Returns: + Extracted text content, or None on failure. + """ + try: + response = await self._client.get(url, follow_redirects=True) + response.raise_for_status() + soup = BeautifulSoup(response.text, "lxml") + + # Remove non-content elements + for tag in soup(["script", "style", "nav", "footer", "header"]): + tag.decompose() + + return soup.get_text(separator="\n", strip=True) + except httpx.HTTPError as e: + logger.error(f"Failed to extract content from {url}: {e}") + return None + + async def close(self): + """Close the underlying HTTP client.""" + await self._client.aclose() + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + await self.close() + + +async def search_web( + query: str, + engine: str = "duckduckgo", + max_results: int = 10, +) -> list[SearchResult]: + """ + Convenience function for one-off web searches. + + Example: + results = await search_web("Python asyncio") + for r in results: + print(f"{r.title}: {r.url}") + """ + async with WebSearch() as searcher: + return await searcher.search(query, engine=engine, max_results=max_results)