"""Baidu HTML scraping engine — free, fastest in China, best Chinese results.""" from __future__ import annotations import time import logging from urllib.parse import quote_plus import httpx from bs4 import BeautifulSoup from scripts.engines.base import BaseEngine, EngineResult logger = logging.getLogger(__name__) HEADERS = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.9", } class BaiduScraper(BaseEngine): """Baidu web scraping — fastest in China, best Chinese content.""" name = "baidu" async def search(self, query: str, max_results: int = 10, **kwargs) -> list[EngineResult]: t0 = time.time() url = f"https://www.baidu.com/s?wd={quote_plus(query)}&rn={max_results}" try: async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client: resp = await client.get(url, headers=HEADERS) resp.raise_for_status() elapsed = time.time() - t0 results = self._parse(resp.text, max_results) logger.info(f"[baidu] {len(results)} results in {elapsed:.2f}s") return results except Exception as e: logger.warning(f"[baidu] failed: {e}") return [] def _parse(self, html: str, max_results: int) -> list[EngineResult]: soup = BeautifulSoup(html, "html.parser") results = [] for item in soup.select("div.result, div.c-container"): title_el = item.select_one("h3 a") if not title_el: continue title = title_el.get_text(strip=True) href = title_el.get("href", "") snippet_el = item.select_one( "span.content-right_8Zs40, div.c-abstract, " "span[class*='content'], div[class*='abstract']" ) snippet = snippet_el.get_text(strip=True) if snippet_el else "" if title and href: results.append(EngineResult( title=title, url=href, snippet=snippet, source="baidu", )) if len(results) >= max_results: break return results