- Replace setup.py with pyproject.toml (uv-compatible) - Add CLI entry point (web-search) with 9 subcommands - Add multi-engine architecture (14 web + 7 special sources) - Add WeChat/CLS/WallStreetCN/ArXiv/Twitter search engines - Remove old src/ and examples/ directories - Remove redundant requirements.txt and setup.py - Update README.md and SKILL.md with full documentation - SKILL.md now compliant with skill-builder standards (frontmatter, triggers, structure)
209 lines
7.0 KiB
Python
209 lines
7.0 KiB
Python
"""
|
|
Twitter/X search module.
|
|
|
|
Generates search URLs for X.com (formerly Twitter) search and
|
|
optionally parses search result pages.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Optional
|
|
from urllib.parse import quote_plus
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
from scripts.core.base import SearchEngine, SearchResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class TwitterSearchEngine(SearchEngine):
|
|
"""
|
|
Twitter/X search engine.
|
|
|
|
Provides search URL generation and optional result scraping.
|
|
Supports basic keyword search, user search, and advanced queries.
|
|
"""
|
|
|
|
SEARCH_URL = "https://twitter.com/search"
|
|
BASE_URL = "https://twitter.com"
|
|
|
|
def __init__(self, config: Optional[dict] = None):
|
|
super().__init__("Twitter", config)
|
|
timeout = (config or {}).get("timeout", 15)
|
|
self._client = httpx.AsyncClient(
|
|
timeout=timeout,
|
|
headers={
|
|
"User-Agent": (config or {}).get(
|
|
"user_agent",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/120.0.0.0 Safari/537.36",
|
|
),
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.5",
|
|
},
|
|
follow_redirects=True,
|
|
)
|
|
|
|
def build_url(self, query: str, search_type: str = "top", **kwargs) -> str:
|
|
"""
|
|
Build Twitter/X search URL.
|
|
|
|
Args:
|
|
query: Search keyword.
|
|
search_type: "top", "latest", "people", "photos", "videos"
|
|
**kwargs: Additional URL parameters.
|
|
|
|
Returns:
|
|
Search URL string.
|
|
"""
|
|
encoded_query = quote_plus(query)
|
|
url = f"{self.SEARCH_URL}?q={encoded_query}&src=typed_query"
|
|
|
|
if search_type == "latest":
|
|
url += "&f=live"
|
|
elif search_type == "people":
|
|
url += "&f=user"
|
|
elif search_type == "photos":
|
|
url += "&f=image"
|
|
elif search_type == "videos":
|
|
url += "&f=video"
|
|
|
|
if kwargs.get("lang"):
|
|
url += f"&l={kwargs['lang']}"
|
|
if kwargs.get("until"):
|
|
url += f"&until={kwargs['until']}"
|
|
if kwargs.get("since"):
|
|
url += f"&since={kwargs['since']}"
|
|
|
|
return url
|
|
|
|
def build_advanced_url(self, **params) -> str:
|
|
"""
|
|
Build advanced Twitter search URL with operators.
|
|
|
|
Supported operators:
|
|
from:username - Tweets from user
|
|
to:username - Tweets to user
|
|
@username - Mentions
|
|
#hashtag - Hashtag
|
|
filter:links - Tweets with links
|
|
filter:images - Tweets with images
|
|
min_retweets:N - Minimum retweets
|
|
min_faves:N - Minimum likes
|
|
lang:en - Language
|
|
|
|
Returns:
|
|
Advanced search URL.
|
|
"""
|
|
parts = []
|
|
if params.get("from_user"):
|
|
parts.append(f"from:{params['from_user']}")
|
|
if params.get("to_user"):
|
|
parts.append(f"to:{params['to_user']}")
|
|
if params.get("hashtag"):
|
|
parts.append(f"#{params['hashtag']}")
|
|
if params.get("keyword"):
|
|
parts.append(params["keyword"])
|
|
if params.get("min_retweets"):
|
|
parts.append(f"min_retweets:{params['min_retweets']}")
|
|
if params.get("min_faves"):
|
|
parts.append(f"min_faves:{params['min_faves']}")
|
|
if params.get("lang"):
|
|
parts.append(f"lang:{params['lang']}")
|
|
if params.get("filter"):
|
|
parts.append(f"filter:{params['filter']}")
|
|
|
|
query = " ".join(parts)
|
|
return self.build_url(query, search_type=params.get("search_type", "top"))
|
|
|
|
async def search(self, query: str, max_results: int = 10, search_type: str = "top", **kwargs) -> list[SearchResult]:
|
|
"""
|
|
Search Twitter/X and optionally parse page content.
|
|
|
|
Args:
|
|
query: Search keyword.
|
|
max_results: Maximum results (used for URL generation).
|
|
search_type: "top", "latest", "people", "photos", "videos"
|
|
**kwargs: Additional options.
|
|
|
|
Returns:
|
|
List of SearchResult objects.
|
|
"""
|
|
url = self.build_url(query, search_type=search_type, **kwargs)
|
|
|
|
# Try to scrape the search page for basic info
|
|
try:
|
|
response = await self._client.get(url)
|
|
if response.status_code == 200:
|
|
soup = BeautifulSoup(response.text, "lxml")
|
|
results = self._parse_page(soup, max_results)
|
|
if results:
|
|
return results
|
|
except Exception as e:
|
|
logger.debug(f"Twitter page scraping failed (expected): {e}")
|
|
|
|
# Fall back to URL-based result
|
|
result_type_map = {
|
|
"top": "Top tweets",
|
|
"latest": "Latest tweets",
|
|
"people": "People",
|
|
"photos": "Photos",
|
|
"videos": "Videos",
|
|
}
|
|
desc = result_type_map.get(search_type, "Tweets")
|
|
|
|
return [
|
|
SearchResult(
|
|
title=f"Twitter: {query} ({desc})",
|
|
url=url,
|
|
snippet=f"Search Twitter/X for '{query}'. Note: Twitter may require login.",
|
|
source=self.name,
|
|
rank=1,
|
|
category="social",
|
|
extra={"search_type": search_type, "twitter_url": url},
|
|
)
|
|
]
|
|
|
|
def _parse_page(self, soup, max_results: int) -> list[SearchResult]:
|
|
"""Attempt to extract tweet info from search page."""
|
|
results = []
|
|
# Twitter uses JS rendering, so HTML parsing may yield limited results
|
|
tweets = soup.select("article[data-testid='tweet'], div.tweet, .content")
|
|
for i, tweet in enumerate(tweets[:max_results]):
|
|
text_el = tweet.select_one(
|
|
"[data-testid='tweetText'], .tweet-text, p[lang]"
|
|
)
|
|
link_el = tweet.select_one("a[href*='/status/'], time")
|
|
user_el = tweet.select_one(
|
|
"[data-testid='User-Name'], .username, .fullname"
|
|
)
|
|
|
|
text = text_el.get_text(strip=True) if text_el else ""
|
|
user = user_el.get_text(strip=True) if user_el else ""
|
|
|
|
tweet_url = ""
|
|
if link_el:
|
|
href = link_el.get("href", "")
|
|
tweet_url = f"{self.BASE_URL}{href}" if href.startswith("/") else href
|
|
|
|
if text:
|
|
results.append(
|
|
SearchResult(
|
|
title=f"@{user}: {text[:80]}..." if user else text[:80],
|
|
url=tweet_url or self.build_url(text[:30]),
|
|
snippet=text,
|
|
source=self.name,
|
|
rank=i + 1,
|
|
category="social",
|
|
extra={"user": user} if user else {},
|
|
)
|
|
)
|
|
return results
|
|
|
|
async def close(self):
|
|
await self._client.aclose()
|