Add project structure: LICENSE, .gitignore, skill definition, and core search module
This commit is contained in:
1 parent
8678b256db
commit
24535ff3f2
8 files changed
+350
No files matched your search
+37
@@ -0,0 +1,37 @@
|
|||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
*.so
|
||||||
|
.Python
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
.venv/
|
||||||
|
*.egg-info/
|
||||||
|
*.egg
|
||||||
|
|
||||||
|
# IDE
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
*~
|
||||||
|
|
||||||
|
# OS
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# Environment
|
||||||
|
.env
|
||||||
|
*.env.local
|
||||||
|
|
||||||
|
# Logs
|
||||||
|
*.log
|
||||||
|
|
||||||
|
# Build
|
||||||
|
dist/
|
||||||
|
build/
|
||||||
|
*.whl
|
||||||
|
|
||||||
|
# Config with secrets
|
||||||
|
config.local.yaml
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2026 openbot-coder
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Examples for using Web Search Skills."""
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
"""
|
||||||
|
Basic usage example for Web Search Skills.
|
||||||
|
|
||||||
|
Run:
|
||||||
|
python examples/basic_search.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from src.web_search import WebSearch
|
||||||
|
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
async with WebSearch() as searcher:
|
||||||
|
results = await searcher.search(
|
||||||
|
"Python web scraping best practices 2026",
|
||||||
|
max_results=5,
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"Found {len(results)} results:\n")
|
||||||
|
for result in results:
|
||||||
|
print(f"[{result.rank}] {result.title}")
|
||||||
|
print(f" URL: {result.url}")
|
||||||
|
print(f" {result.snippet[:120]}...")
|
||||||
|
print()
|
||||||
|
|
||||||
|
if results:
|
||||||
|
content = await searcher.extract_content(results[0].url)
|
||||||
|
if content:
|
||||||
|
print(f"--- Content from {results[0].url} ---")
|
||||||
|
print(content[:300])
|
||||||
|
print("...")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(main())
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
# Web Search Skills Dependencies
|
||||||
|
|
||||||
|
# Core
|
||||||
|
httpx>=0.25.0
|
||||||
|
beautifulsoup4>=4.12.0
|
||||||
|
lxml>=4.9.0
|
||||||
|
|
||||||
|
# Optional: Google Custom Search
|
||||||
|
# google-api-python-client>=2.100.0
|
||||||
|
|
||||||
|
# Development
|
||||||
|
pytest>=7.0.0
|
||||||
|
pytest-asyncio>=0.21.0
|
||||||
|
black>=23.0.0
|
||||||
|
ruff>=0.1.0
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
# Web Search Skill
|
||||||
|
|
||||||
|
## Description
|
||||||
|
|
||||||
|
Provides web search capabilities for AI agents, supporting multiple search engines, intelligent result processing, and customizable search parameters.
|
||||||
|
|
||||||
|
## Triggers
|
||||||
|
|
||||||
|
- "search the web for..."
|
||||||
|
- "find information about..."
|
||||||
|
- "look up..."
|
||||||
|
- Any query requiring real-time web data
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
search:
|
||||||
|
default_engine: duckduckgo
|
||||||
|
max_results: 10
|
||||||
|
timeout: 15
|
||||||
|
user_agent: "WebSearchSkill/1.0"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Actions
|
||||||
|
|
||||||
|
- `web_search(query, engine=None, max_results=10)` - Perform a web search
|
||||||
|
- `search_news(query, max_results=5)` - Search news sources
|
||||||
|
- `search_images(query, max_results=10)` - Search for images
|
||||||
|
- `extract_content(url)` - Extract readable content from a URL
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- Respects robots.txt for web scraping
|
||||||
|
- Implements rate limiting to avoid API bans
|
||||||
|
- Caches results for repeated queries within the same session
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
"""Web Search Skills - A collection of web search tools for AI agents."""
|
||||||
|
|
||||||
|
__version__ = "0.1.0"
|
||||||
|
__author__ = "openbot-coder"
|
||||||
@@ -0,0 +1,201 @@
|
|||||||
|
"""
|
||||||
|
Core web search module providing multi-engine search capabilities.
|
||||||
|
|
||||||
|
Supports DuckDuckGo, Google Custom Search, Bing, and extensible backends.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class SearchResult:
|
||||||
|
"""Represents a single search result."""
|
||||||
|
title: str
|
||||||
|
url: str
|
||||||
|
snippet: str
|
||||||
|
source: str = ""
|
||||||
|
rank: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class SearchConfig:
|
||||||
|
"""Configuration for web search."""
|
||||||
|
engine: str = "duckduckgo"
|
||||||
|
max_results: int = 10
|
||||||
|
timeout: int = 15
|
||||||
|
user_agent: str = "WebSearchSkill/1.0"
|
||||||
|
proxies: dict[str, str] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class WebSearch:
|
||||||
|
"""
|
||||||
|
Main web search class supporting multiple search backends.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
searcher = WebSearch()
|
||||||
|
results = searcher.search("Python async programming")
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, config: Optional[SearchConfig | dict[str, Any]] = None):
|
||||||
|
if config is None:
|
||||||
|
self.config = SearchConfig()
|
||||||
|
elif isinstance(config, dict):
|
||||||
|
self.config = SearchConfig(**config)
|
||||||
|
else:
|
||||||
|
self.config = config
|
||||||
|
|
||||||
|
self._client = httpx.AsyncClient(
|
||||||
|
timeout=self.config.timeout,
|
||||||
|
headers={"User-Agent": self.config.user_agent},
|
||||||
|
proxies=self.config.proxies if self.config.proxies else None,
|
||||||
|
)
|
||||||
|
self._cache: dict[str, list[SearchResult]] = {}
|
||||||
|
|
||||||
|
async def search(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
engine: Optional[str] = None,
|
||||||
|
max_results: Optional[int] = None,
|
||||||
|
force_refresh: bool = False,
|
||||||
|
) -> list[SearchResult]:
|
||||||
|
"""
|
||||||
|
Perform a web search with the given query.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query string.
|
||||||
|
engine: Search engine to use (default: config.engine).
|
||||||
|
max_results: Maximum number of results (default: config.max_results).
|
||||||
|
force_refresh: Ignore cached results if True.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of SearchResult objects.
|
||||||
|
"""
|
||||||
|
cache_key = f"{engine or self.config.engine}:{query}"
|
||||||
|
|
||||||
|
if not force_refresh and cache_key in self._cache:
|
||||||
|
logger.debug(f"Returning cached results for: {query}")
|
||||||
|
return self._cache[cache_key]
|
||||||
|
|
||||||
|
engine = engine or self.config.engine
|
||||||
|
max_results = max_results or self.config.max_results
|
||||||
|
|
||||||
|
if engine == "duckduckgo":
|
||||||
|
results = await self._search_duckduckgo(query, max_results)
|
||||||
|
elif engine == "google":
|
||||||
|
results = await self._search_google(query, max_results)
|
||||||
|
elif engine == "bing":
|
||||||
|
results = await self._search_bing(query, max_results)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unsupported search engine: {engine}")
|
||||||
|
|
||||||
|
self._cache[cache_key] = results
|
||||||
|
return results
|
||||||
|
|
||||||
|
async def _search_duckduckgo(
|
||||||
|
self, query: str, max_results: int
|
||||||
|
) -> list[SearchResult]:
|
||||||
|
"""Search using DuckDuckGo (no API key required)."""
|
||||||
|
url = "https://html.duckduckgo.com/html/"
|
||||||
|
params = {"q": query}
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = await self._client.post(url, data=params)
|
||||||
|
response.raise_for_status()
|
||||||
|
soup = BeautifulSoup(response.text, "lxml")
|
||||||
|
|
||||||
|
results = []
|
||||||
|
for i, result in enumerate(soup.select(".result")[:max_results]):
|
||||||
|
title_el = result.select_one(".result__title a")
|
||||||
|
snippet_el = result.select_one(".result__snippet")
|
||||||
|
|
||||||
|
if title_el:
|
||||||
|
results.append(
|
||||||
|
SearchResult(
|
||||||
|
title=title_el.get_text(strip=True),
|
||||||
|
url=title_el.get("href", ""),
|
||||||
|
snippet=snippet_el.get_text(strip=True)
|
||||||
|
if snippet_el
|
||||||
|
else "",
|
||||||
|
source="duckduckgo",
|
||||||
|
rank=i + 1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return results
|
||||||
|
except httpx.HTTPError as e:
|
||||||
|
logger.error(f"DuckDuckGo search failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def _search_google(
|
||||||
|
self, query: str, max_results: int
|
||||||
|
) -> list[SearchResult]:
|
||||||
|
"""Search using Google Custom Search API (requires API key)."""
|
||||||
|
logger.warning("Google Custom Search requires an API key.")
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def _search_bing(
|
||||||
|
self, query: str, max_results: int
|
||||||
|
) -> list[SearchResult]:
|
||||||
|
"""Search using Bing (requires API key)."""
|
||||||
|
logger.warning("Bing Search requires an API key.")
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def extract_content(self, url: str) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Extract readable text content from a URL.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: Target URL to extract content from.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Extracted text content, or None on failure.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
response = await self._client.get(url, follow_redirects=True)
|
||||||
|
response.raise_for_status()
|
||||||
|
soup = BeautifulSoup(response.text, "lxml")
|
||||||
|
|
||||||
|
# Remove non-content elements
|
||||||
|
for tag in soup(["script", "style", "nav", "footer", "header"]):
|
||||||
|
tag.decompose()
|
||||||
|
|
||||||
|
return soup.get_text(separator="\n", strip=True)
|
||||||
|
except httpx.HTTPError as e:
|
||||||
|
logger.error(f"Failed to extract content from {url}: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
"""Close the underlying HTTP client."""
|
||||||
|
await self._client.aclose()
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *args):
|
||||||
|
await self.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def search_web(
|
||||||
|
query: str,
|
||||||
|
engine: str = "duckduckgo",
|
||||||
|
max_results: int = 10,
|
||||||
|
) -> list[SearchResult]:
|
||||||
|
"""
|
||||||
|
Convenience function for one-off web searches.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
results = await search_web("Python asyncio")
|
||||||
|
for r in results:
|
||||||
|
print(f"{r.title}: {r.url}")
|
||||||
|
"""
|
||||||
|
async with WebSearch() as searcher:
|
||||||
|
return await searcher.search(query, engine=engine, max_results=max_results)
|
||||||
Reference in new issue
Block a user