Files
vex1023 87935b59ce refactor: restructure project with CLI tool, multi-engine support, and skill-builder compliant SKILL.md
- Replace setup.py with pyproject.toml (uv-compatible)
- Add CLI entry point (web-search) with 9 subcommands
- Add multi-engine architecture (14 web + 7 special sources)
- Add WeChat/CLS/WallStreetCN/ArXiv/Twitter search engines
- Remove old src/ and examples/ directories
- Remove redundant requirements.txt and setup.py
- Update README.md and SKILL.md with full documentation
- SKILL.md now compliant with skill-builder standards (frontmatter, triggers, structure)
2026-05-14 16:31:54 +08:00

209 lines
7.0 KiB
Python

"""
Twitter/X search module.
Generates search URLs for X.com (formerly Twitter) search and
optionally parses search result pages.
"""
from __future__ import annotations
import logging
from typing import Optional
from urllib.parse import quote_plus
import httpx
from bs4 import BeautifulSoup
from scripts.core.base import SearchEngine, SearchResult
logger = logging.getLogger(__name__)
class TwitterSearchEngine(SearchEngine):
"""
Twitter/X search engine.
Provides search URL generation and optional result scraping.
Supports basic keyword search, user search, and advanced queries.
"""
SEARCH_URL = "https://twitter.com/search"
BASE_URL = "https://twitter.com"
def __init__(self, config: Optional[dict] = None):
super().__init__("Twitter", config)
timeout = (config or {}).get("timeout", 15)
self._client = httpx.AsyncClient(
timeout=timeout,
headers={
"User-Agent": (config or {}).get(
"user_agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36",
),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
},
follow_redirects=True,
)
def build_url(self, query: str, search_type: str = "top", **kwargs) -> str:
"""
Build Twitter/X search URL.
Args:
query: Search keyword.
search_type: "top", "latest", "people", "photos", "videos"
**kwargs: Additional URL parameters.
Returns:
Search URL string.
"""
encoded_query = quote_plus(query)
url = f"{self.SEARCH_URL}?q={encoded_query}&src=typed_query"
if search_type == "latest":
url += "&f=live"
elif search_type == "people":
url += "&f=user"
elif search_type == "photos":
url += "&f=image"
elif search_type == "videos":
url += "&f=video"
if kwargs.get("lang"):
url += f"&l={kwargs['lang']}"
if kwargs.get("until"):
url += f"&until={kwargs['until']}"
if kwargs.get("since"):
url += f"&since={kwargs['since']}"
return url
def build_advanced_url(self, **params) -> str:
"""
Build advanced Twitter search URL with operators.
Supported operators:
from:username - Tweets from user
to:username - Tweets to user
@username - Mentions
#hashtag - Hashtag
filter:links - Tweets with links
filter:images - Tweets with images
min_retweets:N - Minimum retweets
min_faves:N - Minimum likes
lang:en - Language
Returns:
Advanced search URL.
"""
parts = []
if params.get("from_user"):
parts.append(f"from:{params['from_user']}")
if params.get("to_user"):
parts.append(f"to:{params['to_user']}")
if params.get("hashtag"):
parts.append(f"#{params['hashtag']}")
if params.get("keyword"):
parts.append(params["keyword"])
if params.get("min_retweets"):
parts.append(f"min_retweets:{params['min_retweets']}")
if params.get("min_faves"):
parts.append(f"min_faves:{params['min_faves']}")
if params.get("lang"):
parts.append(f"lang:{params['lang']}")
if params.get("filter"):
parts.append(f"filter:{params['filter']}")
query = " ".join(parts)
return self.build_url(query, search_type=params.get("search_type", "top"))
async def search(self, query: str, max_results: int = 10, search_type: str = "top", **kwargs) -> list[SearchResult]:
"""
Search Twitter/X and optionally parse page content.
Args:
query: Search keyword.
max_results: Maximum results (used for URL generation).
search_type: "top", "latest", "people", "photos", "videos"
**kwargs: Additional options.
Returns:
List of SearchResult objects.
"""
url = self.build_url(query, search_type=search_type, **kwargs)
# Try to scrape the search page for basic info
try:
response = await self._client.get(url)
if response.status_code == 200:
soup = BeautifulSoup(response.text, "lxml")
results = self._parse_page(soup, max_results)
if results:
return results
except Exception as e:
logger.debug(f"Twitter page scraping failed (expected): {e}")
# Fall back to URL-based result
result_type_map = {
"top": "Top tweets",
"latest": "Latest tweets",
"people": "People",
"photos": "Photos",
"videos": "Videos",
}
desc = result_type_map.get(search_type, "Tweets")
return [
SearchResult(
title=f"Twitter: {query} ({desc})",
url=url,
snippet=f"Search Twitter/X for '{query}'. Note: Twitter may require login.",
source=self.name,
rank=1,
category="social",
extra={"search_type": search_type, "twitter_url": url},
)
]
def _parse_page(self, soup, max_results: int) -> list[SearchResult]:
"""Attempt to extract tweet info from search page."""
results = []
# Twitter uses JS rendering, so HTML parsing may yield limited results
tweets = soup.select("article[data-testid='tweet'], div.tweet, .content")
for i, tweet in enumerate(tweets[:max_results]):
text_el = tweet.select_one(
"[data-testid='tweetText'], .tweet-text, p[lang]"
)
link_el = tweet.select_one("a[href*='/status/'], time")
user_el = tweet.select_one(
"[data-testid='User-Name'], .username, .fullname"
)
text = text_el.get_text(strip=True) if text_el else ""
user = user_el.get_text(strip=True) if user_el else ""
tweet_url = ""
if link_el:
href = link_el.get("href", "")
tweet_url = f"{self.BASE_URL}{href}" if href.startswith("/") else href
if text:
results.append(
SearchResult(
title=f"@{user}: {text[:80]}..." if user else text[:80],
url=tweet_url or self.build_url(text[:30]),
snippet=text,
source=self.name,
rank=i + 1,
category="social",
extra={"user": user} if user else {},
)
)
return results
async def close(self):
await self._client.aclose()