diff --git a/src/pytdxdata/api.py b/src/pytdxdata/api.py index 16f8119..3d9a084 100644 --- a/src/pytdxdata/api.py +++ b/src/pytdxdata/api.py @@ -233,11 +233,13 @@ class TdxData: pool_max: int = 12, mac_pool_cap: int = 4, std_pool_cap: int = 4, + ex_pool_cap: int = 2, ) -> None: self.port = port self.timeout = timeout self.default_adjust = default_adjust self.mac_pool_cap = mac_pool_cap + self.ex_pool_cap = ex_pool_cap self._std_servers = list(standard_servers or get_standard_hosts()) self._mac_servers = list(mac_servers or get_mac_hosts()) self._ex_servers = list(ex_servers or get_ex_hosts()) @@ -262,8 +264,11 @@ class TdxData: probe_factory=lambda: MacKlineCmd(1, "600000", KlinePeriod.DAY, Adjust.NONE, 0, 100), ) # EX扩展市场池(港股/期货/美股,端口7727;MAC EX login + pytdx EX setup 混合) + # 单服务器场景(测速缓存收缩后仅 1 台)下,有效并发 = 服务器数 × ex_pool_cap, + # 默认 cap=2 保证至少 2 并发;acquire 有 Condition 背压等待,不再快速失败。 self._ex_pool = ExPool(self._ex_servers, timeout=timeout, min_size=min(pool_min, 2), max_size=min(pool_max, 6), + per_server_cap=ex_pool_cap, handshake_map=EX_HANDSHAKES) disk = None if self._cache_dir is not None: diff --git a/src/pytdxdata/pool/ex_pool.py b/src/pytdxdata/pool/ex_pool.py index 8d81049..2626e18 100644 --- a/src/pytdxdata/pool/ex_pool.py +++ b/src/pytdxdata/pool/ex_pool.py @@ -1,4 +1,11 @@ -"""EX扩展市场连接池(简化版:固定连接数,借还+坏连接重建+login自动)""" +"""EX扩展市场连接池(固定连接数,借还+坏连接重建+login自动)。 + +与标准池 pool.py 的背压语义对齐:acquire 在无空闲连接且无法扩容时, +基于 asyncio.Condition 排队等待 release 唤醒(默认 acquire_timeout=30s), +而不是快速失败。per_server_cap 限制单台服务器连接数;单服务器场景下 +有效并发 = servers × per_server_cap(默认 1),通过 TdxData(ex_pool_cap=N) +可对同一 EX 主机开多连接。 +""" from __future__ import annotations import asyncio @@ -9,6 +16,23 @@ from .ex_connection import ExTdxConnection log = logging.getLogger(__name__) +DEFAULT_ACQUIRE_TIMEOUT = 30.0 + + +class ExPoolTimeoutError(ConnectionError): + """acquire 等待超时(携带池状态快照,便于排障)。""" + + def __init__(self, pool: "ExPool", timeout: float) -> None: + idle = len(pool._idle) + total = pool._total + servers = len(pool.servers) + cap = pool.per_server_cap + super().__init__( + f"EX连接池获取超时(等了 {timeout:.1f}s):" + f"idle={idle} total={total} servers={servers} per_server_cap={cap} " + f"有效并发={servers * cap}" + ) + class ExPool: """EX通道连接池(港股/期货,端口7727,连接时自动login)。""" @@ -16,6 +40,7 @@ class ExPool: def __init__(self, servers: list[str], *, timeout: float = 10.0, min_size: int = 2, max_size: int = 6, per_server_cap: int = 1, + acquire_timeout: float | None = None, handshake: str = "login", handshake_map: dict | None = None) -> None: self.servers = servers @@ -25,14 +50,17 @@ class ExPool: self.min_size = min_size self.max_size = max_size self.per_server_cap = max(1, per_server_cap) # 一台服务器最多几个连接(默认1=单连接) + self.acquire_timeout = ( + DEFAULT_ACQUIRE_TIMEOUT if acquire_timeout is None else acquire_timeout + ) self._idle: list[tuple[str, ExTdxConnection]] = [] self._server_count: dict[str, int] = {} self._total = 0 self._closed = False - self._lock = asyncio.Lock() + self._cond = asyncio.Condition() async def start(self) -> None: - async with self._lock: + async with self._cond: for host in self.servers[: self.min_size]: if self._server_count.get(host, 0) >= self.per_server_cap: continue @@ -47,7 +75,7 @@ class ExPool: async def close(self) -> None: self._closed = True - async with self._lock: + async with self._cond: for _, conn in self._idle: try: await conn.close() @@ -56,48 +84,61 @@ class ExPool: self._idle.clear() self._total = 0 self._server_count.clear() + self._cond.notify_all() - async def acquire(self) -> ExTdxConnection: - for _ in range(3): - stale_conn = None - new_conn = None - new_host = None - async with self._lock: + async def acquire(self, timeout: float | None = None) -> ExTdxConnection: + """借出一条连接;无空闲且无法扩容时排队等待 release 唤醒。 + + timeout 默认取 self.acquire_timeout(30s);传 None 表示无限等待。 + """ + if timeout is None: + timeout = self.acquire_timeout + deadline = None if timeout is None else time.monotonic() + timeout + async with self._cond: + while True: + # 1. 优先借用空闲连接(过期则丢弃,槽位在锁内回收) if self._idle: _, conn = self._idle.pop() fresh = time.monotonic() - conn.last_used < 30.0 if conn.alive and fresh: return conn - # 连接空闲过久(login会话可能失效)→ 丢弃重连 + # 连接空闲过久(login会话可能失效)→ 丢弃,槽位让给扩容 self._total = max(0, self._total - 1) self._server_count[conn.host] = max(0, self._server_count.get(conn.host, 0) - 1) - stale_conn = conn - # 扩容:在锁内预留槽位,锁外执行网络IO - elif self._total < self.max_size and self.servers: + self._schedule_close(conn) + + # 2. 可扩容 → 锁内预留槽位,锁外建连 + new_conn = None + new_host = None + if self._total < self.max_size and self.servers: host = self._next_available_server() - if host: + if host is not None: self._total += 1 self._server_count[host] = self._server_count.get(host, 0) + 1 new_conn = ExTdxConnection(host, 7727, self.timeout, self.handshake_map.get(host, self.handshake)) new_host = host - # 锁外:关闭过期连接、建立新连接 - if stale_conn is not None: + if new_conn is not None: + # 锁外网络 IO:连接失败回滚槽位后继续等待 + try: + await new_conn.connect() + return new_conn + except Exception as e: + log.warning("EX扩容失败 %s: %s", new_host, str(e)[:60]) + async with self._cond: + self._total = max(0, self._total - 1) + self._server_count[new_host] = max(0, self._server_count.get(new_host, 0) - 1) + + # 3. 池满:排队等待 release 唤醒(或超时) + now = time.monotonic() + if deadline is not None and now >= deadline: + raise ExPoolTimeoutError(self, timeout) try: - await stale_conn.close() - except Exception: - pass - if new_conn is not None: - try: - await new_conn.connect() - return new_conn - except Exception as e: - log.warning("EX扩容失败 %s: %s", new_host, str(e)[:60]) - # 连接失败,回滚预留槽位 - async with self._lock: - self._total = max(0, self._total - 1) - self._server_count[new_host] = max(0, self._server_count.get(new_host, 0) - 1) - await asyncio.sleep(0.05) - raise ConnectionError("EX连接池无可用连接") + if deadline is None: + await self._cond.wait() + else: + await asyncio.wait_for(self._cond.wait(), timeout=deadline - now) + except asyncio.TimeoutError: + raise ExPoolTimeoutError(self, timeout) def _next_available_server(self) -> str | None: for h in self.servers: @@ -105,15 +146,23 @@ class ExPool: return h return None - async def release(self, conn: ExTdxConnection, *, broken: bool = False) -> None: - if broken or not conn.alive: - async with self._lock: - self._total = max(0, self._total - 1) - self._server_count[conn.host] = max(0, self._server_count.get(conn.host, 0) - 1) + def _schedule_close(self, conn: ExTdxConnection) -> None: + async def _close(): try: await conn.close() except Exception: pass - return - async with self._lock: - self._idle.append((conn.host, conn)) + try: + asyncio.get_running_loop().create_task(_close()) + except RuntimeError: + pass + + async def release(self, conn: ExTdxConnection, *, broken: bool = False) -> None: + async with self._cond: + if broken or not conn.alive: + self._total = max(0, self._total - 1) + self._server_count[conn.host] = max(0, self._server_count.get(conn.host, 0) - 1) + self._schedule_close(conn) + else: + self._idle.append((conn.host, conn)) + self._cond.notify_all() diff --git a/tests/test_ex_pool_backpressure.py b/tests/test_ex_pool_backpressure.py new file mode 100644 index 0000000..71513b8 --- /dev/null +++ b/tests/test_ex_pool_backpressure.py @@ -0,0 +1,116 @@ +"""ExPool 背压/并发语义回归(Issue #1:并发超限 0.15s 秒失败)。 + +用 mock 连接模拟:单服务器 + per_server_cap=1 时, +- 并发 acquire 超过有效并发 → 排队等待而不是快速失败; +- release 唤醒等待者; +- acquire 超时异常带池状态; +- ex_pool_cap 提升同主机并发。 +""" +from __future__ import annotations + +import asyncio +import time + +import pytest + +from pytdxdata.pool.ex_pool import ExPool, ExPoolTimeoutError + + +class MockConn: + """Minimal stand-in for ExTdxConnection.""" + + last_created = 0 + + def __init__(self, host: str, port: int, timeout: float, handshake: str = "login"): + self.host = host + self.port = port + self.alive = True + self.last_used = time.monotonic() + self.closed = False + MockConn.last_created += 1 + + async def connect(self) -> None: + await asyncio.sleep(0) # instant "connect" + + async def close(self) -> None: + self.closed = True + self.alive = False + + +@pytest.fixture +def patch_conn(monkeypatch): + monkeypatch.setattr("pytdxdata.pool.ex_pool.ExTdxConnection", MockConn) + MockConn.last_created = 0 + + +async def test_acquire_queues_instead_of_fast_fail(patch_conn): + """并发 > 有效并发时,后续 acquire 等待 release,而非 0.15s 抛错。""" + pool = ExPool(["e1"], min_size=1, max_size=6, per_server_cap=1) + await pool.start() + c1 = await pool.acquire() # 唯一连接被占用 + t0 = time.monotonic() + + async def holder(): + await asyncio.sleep(0.5) # 持有 0.5s 后归还 + await pool.release(c1) + + asyncio.create_task(holder()) + c2 = await pool.acquire() # 应在 ~0.5s 后拿到,而不是 0.15s 抛错 + waited = time.monotonic() - t0 + assert waited >= 0.4, f"acquire 没有等待释放(仅 {waited:.2f}s)" + assert c2 is c1 # 归还的是同一条连接 + + +async def test_acquire_timeout_carries_pool_state(patch_conn): + """超时异常信息带池状态(idle/total/servers/cap)。""" + pool = ExPool(["e1"], min_size=1, max_size=6, per_server_cap=1, acquire_timeout=0.3) + await pool.start() + c1 = await pool.acquire() + with pytest.raises(ExPoolTimeoutError) as ei: + await pool.acquire() + msg = str(ei.value) + assert "total=1" in msg and "per_server_cap=1" in msg and "servers=1" in msg + await pool.release(c1) + + +async def test_ex_pool_cap_allows_parallel_conns_same_host(patch_conn): + """per_server_cap=2 时单主机可并发持有 2 条连接。""" + pool = ExPool(["e1"], min_size=1, max_size=6, per_server_cap=2) + await pool.start() + got = await asyncio.gather(pool.acquire(), pool.acquire()) + assert len(got) == 2 and got[0] is not got[1] + assert MockConn.last_created == 2 + for c in got: + await pool.release(c) + + +async def test_concurrent_acquire_all_succeed(patch_conn): + """Issue 场景:单服务器 + cap=2,6 个并发任务应全部拿到连接(排队)而非部分失败。""" + pool = ExPool(["e1"], min_size=2, max_size=6, per_server_cap=2) + await pool.start() + + async def task(i): + c = await pool.acquire() + await asyncio.sleep(0.1) # 模拟一次 1m 分页拉取 + await pool.release(c) + return i + + results = await asyncio.gather(*[task(i) for i in range(6)]) + assert sorted(results) == list(range(6)) # 全部成功,零失败 + + +async def test_release_notifies_waiters(patch_conn): + """broken 归还(连接死亡)也要唤醒等待者(槽位回收后可扩容)。""" + pool = ExPool(["e1"], min_size=1, max_size=1, per_server_cap=1) + await pool.start() + c1 = await pool.acquire() + c1.alive = False + + async def broken_release(): + await asyncio.sleep(0.2) + await pool.release(c1, broken=True) + + asyncio.create_task(broken_release()) + c2 = await pool.acquire() # 槽位被回收→扩容新建 + assert c2 is not c1 + await pool.release(c2)