refactor: restructure project with CLI tool, multi-engine support, and skill-builder compliant SKILL.md
- Replace setup.py with pyproject.toml (uv-compatible) - Add CLI entry point (web-search) with 9 subcommands - Add multi-engine architecture (14 web + 7 special sources) - Add WeChat/CLS/WallStreetCN/ArXiv/Twitter search engines - Remove old src/ and examples/ directories - Remove redundant requirements.txt and setup.py - Update README.md and SKILL.md with full documentation - SKILL.md now compliant with skill-builder standards (frontmatter, triggers, structure)
This commit is contained in:
1 parent
80ac14491e
commit
87935b59ce
29 files changed
+2364
-354
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Social media search implementations (Twitter/X, etc.)."""
|
||||
@@ -0,0 +1,208 @@
|
||||
"""
|
||||
Twitter/X search module.
|
||||
|
||||
Generates search URLs for X.com (formerly Twitter) search and
|
||||
optionally parses search result pages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from scripts.core.base import SearchEngine, SearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TwitterSearchEngine(SearchEngine):
|
||||
"""
|
||||
Twitter/X search engine.
|
||||
|
||||
Provides search URL generation and optional result scraping.
|
||||
Supports basic keyword search, user search, and advanced queries.
|
||||
"""
|
||||
|
||||
SEARCH_URL = "https://twitter.com/search"
|
||||
BASE_URL = "https://twitter.com"
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
super().__init__("Twitter", config)
|
||||
timeout = (config or {}).get("timeout", 15)
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={
|
||||
"User-Agent": (config or {}).get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
def build_url(self, query: str, search_type: str = "top", **kwargs) -> str:
|
||||
"""
|
||||
Build Twitter/X search URL.
|
||||
|
||||
Args:
|
||||
query: Search keyword.
|
||||
search_type: "top", "latest", "people", "photos", "videos"
|
||||
**kwargs: Additional URL parameters.
|
||||
|
||||
Returns:
|
||||
Search URL string.
|
||||
"""
|
||||
encoded_query = quote_plus(query)
|
||||
url = f"{self.SEARCH_URL}?q={encoded_query}&src=typed_query"
|
||||
|
||||
if search_type == "latest":
|
||||
url += "&f=live"
|
||||
elif search_type == "people":
|
||||
url += "&f=user"
|
||||
elif search_type == "photos":
|
||||
url += "&f=image"
|
||||
elif search_type == "videos":
|
||||
url += "&f=video"
|
||||
|
||||
if kwargs.get("lang"):
|
||||
url += f"&l={kwargs['lang']}"
|
||||
if kwargs.get("until"):
|
||||
url += f"&until={kwargs['until']}"
|
||||
if kwargs.get("since"):
|
||||
url += f"&since={kwargs['since']}"
|
||||
|
||||
return url
|
||||
|
||||
def build_advanced_url(self, **params) -> str:
|
||||
"""
|
||||
Build advanced Twitter search URL with operators.
|
||||
|
||||
Supported operators:
|
||||
from:username - Tweets from user
|
||||
to:username - Tweets to user
|
||||
@username - Mentions
|
||||
#hashtag - Hashtag
|
||||
filter:links - Tweets with links
|
||||
filter:images - Tweets with images
|
||||
min_retweets:N - Minimum retweets
|
||||
min_faves:N - Minimum likes
|
||||
lang:en - Language
|
||||
|
||||
Returns:
|
||||
Advanced search URL.
|
||||
"""
|
||||
parts = []
|
||||
if params.get("from_user"):
|
||||
parts.append(f"from:{params['from_user']}")
|
||||
if params.get("to_user"):
|
||||
parts.append(f"to:{params['to_user']}")
|
||||
if params.get("hashtag"):
|
||||
parts.append(f"#{params['hashtag']}")
|
||||
if params.get("keyword"):
|
||||
parts.append(params["keyword"])
|
||||
if params.get("min_retweets"):
|
||||
parts.append(f"min_retweets:{params['min_retweets']}")
|
||||
if params.get("min_faves"):
|
||||
parts.append(f"min_faves:{params['min_faves']}")
|
||||
if params.get("lang"):
|
||||
parts.append(f"lang:{params['lang']}")
|
||||
if params.get("filter"):
|
||||
parts.append(f"filter:{params['filter']}")
|
||||
|
||||
query = " ".join(parts)
|
||||
return self.build_url(query, search_type=params.get("search_type", "top"))
|
||||
|
||||
async def search(self, query: str, max_results: int = 10, search_type: str = "top", **kwargs) -> list[SearchResult]:
|
||||
"""
|
||||
Search Twitter/X and optionally parse page content.
|
||||
|
||||
Args:
|
||||
query: Search keyword.
|
||||
max_results: Maximum results (used for URL generation).
|
||||
search_type: "top", "latest", "people", "photos", "videos"
|
||||
**kwargs: Additional options.
|
||||
|
||||
Returns:
|
||||
List of SearchResult objects.
|
||||
"""
|
||||
url = self.build_url(query, search_type=search_type, **kwargs)
|
||||
|
||||
# Try to scrape the search page for basic info
|
||||
try:
|
||||
response = await self._client.get(url)
|
||||
if response.status_code == 200:
|
||||
soup = BeautifulSoup(response.text, "lxml")
|
||||
results = self._parse_page(soup, max_results)
|
||||
if results:
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.debug(f"Twitter page scraping failed (expected): {e}")
|
||||
|
||||
# Fall back to URL-based result
|
||||
result_type_map = {
|
||||
"top": "Top tweets",
|
||||
"latest": "Latest tweets",
|
||||
"people": "People",
|
||||
"photos": "Photos",
|
||||
"videos": "Videos",
|
||||
}
|
||||
desc = result_type_map.get(search_type, "Tweets")
|
||||
|
||||
return [
|
||||
SearchResult(
|
||||
title=f"Twitter: {query} ({desc})",
|
||||
url=url,
|
||||
snippet=f"Search Twitter/X for '{query}'. Note: Twitter may require login.",
|
||||
source=self.name,
|
||||
rank=1,
|
||||
category="social",
|
||||
extra={"search_type": search_type, "twitter_url": url},
|
||||
)
|
||||
]
|
||||
|
||||
def _parse_page(self, soup, max_results: int) -> list[SearchResult]:
|
||||
"""Attempt to extract tweet info from search page."""
|
||||
results = []
|
||||
# Twitter uses JS rendering, so HTML parsing may yield limited results
|
||||
tweets = soup.select("article[data-testid='tweet'], div.tweet, .content")
|
||||
for i, tweet in enumerate(tweets[:max_results]):
|
||||
text_el = tweet.select_one(
|
||||
"[data-testid='tweetText'], .tweet-text, p[lang]"
|
||||
)
|
||||
link_el = tweet.select_one("a[href*='/status/'], time")
|
||||
user_el = tweet.select_one(
|
||||
"[data-testid='User-Name'], .username, .fullname"
|
||||
)
|
||||
|
||||
text = text_el.get_text(strip=True) if text_el else ""
|
||||
user = user_el.get_text(strip=True) if user_el else ""
|
||||
|
||||
tweet_url = ""
|
||||
if link_el:
|
||||
href = link_el.get("href", "")
|
||||
tweet_url = f"{self.BASE_URL}{href}" if href.startswith("/") else href
|
||||
|
||||
if text:
|
||||
results.append(
|
||||
SearchResult(
|
||||
title=f"@{user}: {text[:80]}..." if user else text[:80],
|
||||
url=tweet_url or self.build_url(text[:30]),
|
||||
snippet=text,
|
||||
source=self.name,
|
||||
rank=i + 1,
|
||||
category="social",
|
||||
extra={"user": user} if user else {},
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
async def close(self):
|
||||
await self._client.aclose()
|
||||
Reference in new issue
Block a user