Add project structure: LICENSE, .gitignore, skill definition, and core search module
This commit is contained in:
1 parent
8678b256db
commit
24535ff3f2
8 files changed
+350
No files matched your search
+37
@@ -0,0 +1,37 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
env/
|
||||
venv/
|
||||
.venv/
|
||||
*.egg-info/
|
||||
*.egg
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Environment
|
||||
.env
|
||||
*.env.local
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Build
|
||||
dist/
|
||||
build/
|
||||
*.whl
|
||||
|
||||
# Config with secrets
|
||||
config.local.yaml
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 openbot-coder
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1 @@
|
||||
"""Examples for using Web Search Skills."""
|
||||
@@ -0,0 +1,36 @@
|
||||
"""
|
||||
Basic usage example for Web Search Skills.
|
||||
|
||||
Run:
|
||||
python examples/basic_search.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from src.web_search import WebSearch
|
||||
|
||||
|
||||
async def main():
|
||||
async with WebSearch() as searcher:
|
||||
results = await searcher.search(
|
||||
"Python web scraping best practices 2026",
|
||||
max_results=5,
|
||||
)
|
||||
|
||||
print(f"Found {len(results)} results:\n")
|
||||
for result in results:
|
||||
print(f"[{result.rank}] {result.title}")
|
||||
print(f" URL: {result.url}")
|
||||
print(f" {result.snippet[:120]}...")
|
||||
print()
|
||||
|
||||
if results:
|
||||
content = await searcher.extract_content(results[0].url)
|
||||
if content:
|
||||
print(f"--- Content from {results[0].url} ---")
|
||||
print(content[:300])
|
||||
print("...")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,15 @@
|
||||
# Web Search Skills Dependencies
|
||||
|
||||
# Core
|
||||
httpx>=0.25.0
|
||||
beautifulsoup4>=4.12.0
|
||||
lxml>=4.9.0
|
||||
|
||||
# Optional: Google Custom Search
|
||||
# google-api-python-client>=2.100.0
|
||||
|
||||
# Development
|
||||
pytest>=7.0.0
|
||||
pytest-asyncio>=0.21.0
|
||||
black>=23.0.0
|
||||
ruff>=0.1.0
|
||||
@@ -0,0 +1,35 @@
|
||||
# Web Search Skill
|
||||
|
||||
## Description
|
||||
|
||||
Provides web search capabilities for AI agents, supporting multiple search engines, intelligent result processing, and customizable search parameters.
|
||||
|
||||
## Triggers
|
||||
|
||||
- "search the web for..."
|
||||
- "find information about..."
|
||||
- "look up..."
|
||||
- Any query requiring real-time web data
|
||||
|
||||
## Configuration
|
||||
|
||||
```yaml
|
||||
search:
|
||||
default_engine: duckduckgo
|
||||
max_results: 10
|
||||
timeout: 15
|
||||
user_agent: "WebSearchSkill/1.0"
|
||||
```
|
||||
|
||||
## Actions
|
||||
|
||||
- `web_search(query, engine=None, max_results=10)` - Perform a web search
|
||||
- `search_news(query, max_results=5)` - Search news sources
|
||||
- `search_images(query, max_results=10)` - Search for images
|
||||
- `extract_content(url)` - Extract readable content from a URL
|
||||
|
||||
## Notes
|
||||
|
||||
- Respects robots.txt for web scraping
|
||||
- Implements rate limiting to avoid API bans
|
||||
- Caches results for repeated queries within the same session
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Web Search Skills - A collection of web search tools for AI agents."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
__author__ = "openbot-coder"
|
||||
@@ -0,0 +1,201 @@
|
||||
"""
|
||||
Core web search module providing multi-engine search capabilities.
|
||||
|
||||
Supports DuckDuckGo, Google Custom Search, Bing, and extensible backends.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""Represents a single search result."""
|
||||
title: str
|
||||
url: str
|
||||
snippet: str
|
||||
source: str = ""
|
||||
rank: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchConfig:
|
||||
"""Configuration for web search."""
|
||||
engine: str = "duckduckgo"
|
||||
max_results: int = 10
|
||||
timeout: int = 15
|
||||
user_agent: str = "WebSearchSkill/1.0"
|
||||
proxies: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
class WebSearch:
|
||||
"""
|
||||
Main web search class supporting multiple search backends.
|
||||
|
||||
Usage:
|
||||
searcher = WebSearch()
|
||||
results = searcher.search("Python async programming")
|
||||
"""
|
||||
|
||||
def __init__(self, config: Optional[SearchConfig | dict[str, Any]] = None):
|
||||
if config is None:
|
||||
self.config = SearchConfig()
|
||||
elif isinstance(config, dict):
|
||||
self.config = SearchConfig(**config)
|
||||
else:
|
||||
self.config = config
|
||||
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=self.config.timeout,
|
||||
headers={"User-Agent": self.config.user_agent},
|
||||
proxies=self.config.proxies if self.config.proxies else None,
|
||||
)
|
||||
self._cache: dict[str, list[SearchResult]] = {}
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
engine: Optional[str] = None,
|
||||
max_results: Optional[int] = None,
|
||||
force_refresh: bool = False,
|
||||
) -> list[SearchResult]:
|
||||
"""
|
||||
Perform a web search with the given query.
|
||||
|
||||
Args:
|
||||
query: Search query string.
|
||||
engine: Search engine to use (default: config.engine).
|
||||
max_results: Maximum number of results (default: config.max_results).
|
||||
force_refresh: Ignore cached results if True.
|
||||
|
||||
Returns:
|
||||
List of SearchResult objects.
|
||||
"""
|
||||
cache_key = f"{engine or self.config.engine}:{query}"
|
||||
|
||||
if not force_refresh and cache_key in self._cache:
|
||||
logger.debug(f"Returning cached results for: {query}")
|
||||
return self._cache[cache_key]
|
||||
|
||||
engine = engine or self.config.engine
|
||||
max_results = max_results or self.config.max_results
|
||||
|
||||
if engine == "duckduckgo":
|
||||
results = await self._search_duckduckgo(query, max_results)
|
||||
elif engine == "google":
|
||||
results = await self._search_google(query, max_results)
|
||||
elif engine == "bing":
|
||||
results = await self._search_bing(query, max_results)
|
||||
else:
|
||||
raise ValueError(f"Unsupported search engine: {engine}")
|
||||
|
||||
self._cache[cache_key] = results
|
||||
return results
|
||||
|
||||
async def _search_duckduckgo(
|
||||
self, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Search using DuckDuckGo (no API key required)."""
|
||||
url = "https://html.duckduckgo.com/html/"
|
||||
params = {"q": query}
|
||||
|
||||
try:
|
||||
response = await self._client.post(url, data=params)
|
||||
response.raise_for_status()
|
||||
soup = BeautifulSoup(response.text, "lxml")
|
||||
|
||||
results = []
|
||||
for i, result in enumerate(soup.select(".result")[:max_results]):
|
||||
title_el = result.select_one(".result__title a")
|
||||
snippet_el = result.select_one(".result__snippet")
|
||||
|
||||
if title_el:
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=title_el.get_text(strip=True),
|
||||
url=title_el.get("href", ""),
|
||||
snippet=snippet_el.get_text(strip=True)
|
||||
if snippet_el
|
||||
else "",
|
||||
source="duckduckgo",
|
||||
rank=i + 1,
|
||||
)
|
||||
)
|
||||
return results
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"DuckDuckGo search failed: {e}")
|
||||
return []
|
||||
|
||||
async def _search_google(
|
||||
self, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Search using Google Custom Search API (requires API key)."""
|
||||
logger.warning("Google Custom Search requires an API key.")
|
||||
return []
|
||||
|
||||
async def _search_bing(
|
||||
self, query: str, max_results: int
|
||||
) -> list[SearchResult]:
|
||||
"""Search using Bing (requires API key)."""
|
||||
logger.warning("Bing Search requires an API key.")
|
||||
return []
|
||||
|
||||
async def extract_content(self, url: str) -> Optional[str]:
|
||||
"""
|
||||
Extract readable text content from a URL.
|
||||
|
||||
Args:
|
||||
url: Target URL to extract content from.
|
||||
|
||||
Returns:
|
||||
Extracted text content, or None on failure.
|
||||
"""
|
||||
try:
|
||||
response = await self._client.get(url, follow_redirects=True)
|
||||
response.raise_for_status()
|
||||
soup = BeautifulSoup(response.text, "lxml")
|
||||
|
||||
# Remove non-content elements
|
||||
for tag in soup(["script", "style", "nav", "footer", "header"]):
|
||||
tag.decompose()
|
||||
|
||||
return soup.get_text(separator="\n", strip=True)
|
||||
except httpx.HTTPError as e:
|
||||
logger.error(f"Failed to extract content from {url}: {e}")
|
||||
return None
|
||||
|
||||
async def close(self):
|
||||
"""Close the underlying HTTP client."""
|
||||
await self._client.aclose()
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
await self.close()
|
||||
|
||||
|
||||
async def search_web(
|
||||
query: str,
|
||||
engine: str = "duckduckgo",
|
||||
max_results: int = 10,
|
||||
) -> list[SearchResult]:
|
||||
"""
|
||||
Convenience function for one-off web searches.
|
||||
|
||||
Example:
|
||||
results = await search_web("Python asyncio")
|
||||
for r in results:
|
||||
print(f"{r.title}: {r.url}")
|
||||
"""
|
||||
async with WebSearch() as searcher:
|
||||
return await searcher.search(query, engine=engine, max_results=max_results)
|
||||
Reference in new issue
Block a user