Compare commits
1
Commits
main
...
743aa33dd3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
743aa33dd3 |
No files matched your search
@@ -233,11 +233,13 @@ class TdxData:
|
|||||||
pool_max: int = 12,
|
pool_max: int = 12,
|
||||||
mac_pool_cap: int = 4,
|
mac_pool_cap: int = 4,
|
||||||
std_pool_cap: int = 4,
|
std_pool_cap: int = 4,
|
||||||
|
ex_pool_cap: int = 2,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.port = port
|
self.port = port
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
self.default_adjust = default_adjust
|
self.default_adjust = default_adjust
|
||||||
self.mac_pool_cap = mac_pool_cap
|
self.mac_pool_cap = mac_pool_cap
|
||||||
|
self.ex_pool_cap = ex_pool_cap
|
||||||
self._std_servers = list(standard_servers or get_standard_hosts())
|
self._std_servers = list(standard_servers or get_standard_hosts())
|
||||||
self._mac_servers = list(mac_servers or get_mac_hosts())
|
self._mac_servers = list(mac_servers or get_mac_hosts())
|
||||||
self._ex_servers = list(ex_servers or get_ex_hosts())
|
self._ex_servers = list(ex_servers or get_ex_hosts())
|
||||||
@@ -262,8 +264,11 @@ class TdxData:
|
|||||||
probe_factory=lambda: MacKlineCmd(1, "600000", KlinePeriod.DAY, Adjust.NONE, 0, 100),
|
probe_factory=lambda: MacKlineCmd(1, "600000", KlinePeriod.DAY, Adjust.NONE, 0, 100),
|
||||||
)
|
)
|
||||||
# EX扩展市场池(港股/期货/美股,端口7727;MAC EX login + pytdx EX setup 混合)
|
# EX扩展市场池(港股/期货/美股,端口7727;MAC EX login + pytdx EX setup 混合)
|
||||||
|
# 单服务器场景(测速缓存收缩后仅 1 台)下,有效并发 = 服务器数 × ex_pool_cap,
|
||||||
|
# 默认 cap=2 保证至少 2 并发;acquire 有 Condition 背压等待,不再快速失败。
|
||||||
self._ex_pool = ExPool(self._ex_servers, timeout=timeout,
|
self._ex_pool = ExPool(self._ex_servers, timeout=timeout,
|
||||||
min_size=min(pool_min, 2), max_size=min(pool_max, 6),
|
min_size=min(pool_min, 2), max_size=min(pool_max, 6),
|
||||||
|
per_server_cap=ex_pool_cap,
|
||||||
handshake_map=EX_HANDSHAKES)
|
handshake_map=EX_HANDSHAKES)
|
||||||
disk = None
|
disk = None
|
||||||
if self._cache_dir is not None:
|
if self._cache_dir is not None:
|
||||||
|
|||||||
@@ -1,4 +1,11 @@
|
|||||||
"""EX扩展市场连接池(简化版:固定连接数,借还+坏连接重建+login自动)"""
|
"""EX扩展市场连接池(固定连接数,借还+坏连接重建+login自动)。
|
||||||
|
|
||||||
|
与标准池 pool.py 的背压语义对齐:acquire 在无空闲连接且无法扩容时,
|
||||||
|
基于 asyncio.Condition 排队等待 release 唤醒(默认 acquire_timeout=30s),
|
||||||
|
而不是快速失败。per_server_cap 限制单台服务器连接数;单服务器场景下
|
||||||
|
有效并发 = servers × per_server_cap(默认 1),通过 TdxData(ex_pool_cap=N)
|
||||||
|
可对同一 EX 主机开多连接。
|
||||||
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
@@ -9,6 +16,23 @@ from .ex_connection import ExTdxConnection
|
|||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DEFAULT_ACQUIRE_TIMEOUT = 30.0
|
||||||
|
|
||||||
|
|
||||||
|
class ExPoolTimeoutError(ConnectionError):
|
||||||
|
"""acquire 等待超时(携带池状态快照,便于排障)。"""
|
||||||
|
|
||||||
|
def __init__(self, pool: "ExPool", timeout: float) -> None:
|
||||||
|
idle = len(pool._idle)
|
||||||
|
total = pool._total
|
||||||
|
servers = len(pool.servers)
|
||||||
|
cap = pool.per_server_cap
|
||||||
|
super().__init__(
|
||||||
|
f"EX连接池获取超时(等了 {timeout:.1f}s):"
|
||||||
|
f"idle={idle} total={total} servers={servers} per_server_cap={cap} "
|
||||||
|
f"有效并发={servers * cap}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ExPool:
|
class ExPool:
|
||||||
"""EX通道连接池(港股/期货,端口7727,连接时自动login)。"""
|
"""EX通道连接池(港股/期货,端口7727,连接时自动login)。"""
|
||||||
@@ -16,6 +40,7 @@ class ExPool:
|
|||||||
def __init__(self, servers: list[str], *, timeout: float = 10.0,
|
def __init__(self, servers: list[str], *, timeout: float = 10.0,
|
||||||
min_size: int = 2, max_size: int = 6,
|
min_size: int = 2, max_size: int = 6,
|
||||||
per_server_cap: int = 1,
|
per_server_cap: int = 1,
|
||||||
|
acquire_timeout: float | None = None,
|
||||||
handshake: str = "login",
|
handshake: str = "login",
|
||||||
handshake_map: dict | None = None) -> None:
|
handshake_map: dict | None = None) -> None:
|
||||||
self.servers = servers
|
self.servers = servers
|
||||||
@@ -25,14 +50,17 @@ class ExPool:
|
|||||||
self.min_size = min_size
|
self.min_size = min_size
|
||||||
self.max_size = max_size
|
self.max_size = max_size
|
||||||
self.per_server_cap = max(1, per_server_cap) # 一台服务器最多几个连接(默认1=单连接)
|
self.per_server_cap = max(1, per_server_cap) # 一台服务器最多几个连接(默认1=单连接)
|
||||||
|
self.acquire_timeout = (
|
||||||
|
DEFAULT_ACQUIRE_TIMEOUT if acquire_timeout is None else acquire_timeout
|
||||||
|
)
|
||||||
self._idle: list[tuple[str, ExTdxConnection]] = []
|
self._idle: list[tuple[str, ExTdxConnection]] = []
|
||||||
self._server_count: dict[str, int] = {}
|
self._server_count: dict[str, int] = {}
|
||||||
self._total = 0
|
self._total = 0
|
||||||
self._closed = False
|
self._closed = False
|
||||||
self._lock = asyncio.Lock()
|
self._cond = asyncio.Condition()
|
||||||
|
|
||||||
async def start(self) -> None:
|
async def start(self) -> None:
|
||||||
async with self._lock:
|
async with self._cond:
|
||||||
for host in self.servers[: self.min_size]:
|
for host in self.servers[: self.min_size]:
|
||||||
if self._server_count.get(host, 0) >= self.per_server_cap:
|
if self._server_count.get(host, 0) >= self.per_server_cap:
|
||||||
continue
|
continue
|
||||||
@@ -47,7 +75,7 @@ class ExPool:
|
|||||||
|
|
||||||
async def close(self) -> None:
|
async def close(self) -> None:
|
||||||
self._closed = True
|
self._closed = True
|
||||||
async with self._lock:
|
async with self._cond:
|
||||||
for _, conn in self._idle:
|
for _, conn in self._idle:
|
||||||
try:
|
try:
|
||||||
await conn.close()
|
await conn.close()
|
||||||
@@ -56,48 +84,61 @@ class ExPool:
|
|||||||
self._idle.clear()
|
self._idle.clear()
|
||||||
self._total = 0
|
self._total = 0
|
||||||
self._server_count.clear()
|
self._server_count.clear()
|
||||||
|
self._cond.notify_all()
|
||||||
|
|
||||||
async def acquire(self) -> ExTdxConnection:
|
async def acquire(self, timeout: float | None = None) -> ExTdxConnection:
|
||||||
for _ in range(3):
|
"""借出一条连接;无空闲且无法扩容时排队等待 release 唤醒。
|
||||||
stale_conn = None
|
|
||||||
new_conn = None
|
timeout 默认取 self.acquire_timeout(30s);传 None 表示无限等待。
|
||||||
new_host = None
|
"""
|
||||||
async with self._lock:
|
if timeout is None:
|
||||||
|
timeout = self.acquire_timeout
|
||||||
|
deadline = None if timeout is None else time.monotonic() + timeout
|
||||||
|
async with self._cond:
|
||||||
|
while True:
|
||||||
|
# 1. 优先借用空闲连接(过期则丢弃,槽位在锁内回收)
|
||||||
if self._idle:
|
if self._idle:
|
||||||
_, conn = self._idle.pop()
|
_, conn = self._idle.pop()
|
||||||
fresh = time.monotonic() - conn.last_used < 30.0
|
fresh = time.monotonic() - conn.last_used < 30.0
|
||||||
if conn.alive and fresh:
|
if conn.alive and fresh:
|
||||||
return conn
|
return conn
|
||||||
# 连接空闲过久(login会话可能失效)→ 丢弃重连
|
# 连接空闲过久(login会话可能失效)→ 丢弃,槽位让给扩容
|
||||||
self._total = max(0, self._total - 1)
|
self._total = max(0, self._total - 1)
|
||||||
self._server_count[conn.host] = max(0, self._server_count.get(conn.host, 0) - 1)
|
self._server_count[conn.host] = max(0, self._server_count.get(conn.host, 0) - 1)
|
||||||
stale_conn = conn
|
self._schedule_close(conn)
|
||||||
# 扩容:在锁内预留槽位,锁外执行网络IO
|
|
||||||
elif self._total < self.max_size and self.servers:
|
# 2. 可扩容 → 锁内预留槽位,锁外建连
|
||||||
|
new_conn = None
|
||||||
|
new_host = None
|
||||||
|
if self._total < self.max_size and self.servers:
|
||||||
host = self._next_available_server()
|
host = self._next_available_server()
|
||||||
if host:
|
if host is not None:
|
||||||
self._total += 1
|
self._total += 1
|
||||||
self._server_count[host] = self._server_count.get(host, 0) + 1
|
self._server_count[host] = self._server_count.get(host, 0) + 1
|
||||||
new_conn = ExTdxConnection(host, 7727, self.timeout, self.handshake_map.get(host, self.handshake))
|
new_conn = ExTdxConnection(host, 7727, self.timeout, self.handshake_map.get(host, self.handshake))
|
||||||
new_host = host
|
new_host = host
|
||||||
# 锁外:关闭过期连接、建立新连接
|
|
||||||
if stale_conn is not None:
|
|
||||||
try:
|
|
||||||
await stale_conn.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
if new_conn is not None:
|
if new_conn is not None:
|
||||||
|
# 锁外网络 IO:连接失败回滚槽位后继续等待
|
||||||
try:
|
try:
|
||||||
await new_conn.connect()
|
await new_conn.connect()
|
||||||
return new_conn
|
return new_conn
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning("EX扩容失败 %s: %s", new_host, str(e)[:60])
|
log.warning("EX扩容失败 %s: %s", new_host, str(e)[:60])
|
||||||
# 连接失败,回滚预留槽位
|
async with self._cond:
|
||||||
async with self._lock:
|
|
||||||
self._total = max(0, self._total - 1)
|
self._total = max(0, self._total - 1)
|
||||||
self._server_count[new_host] = max(0, self._server_count.get(new_host, 0) - 1)
|
self._server_count[new_host] = max(0, self._server_count.get(new_host, 0) - 1)
|
||||||
await asyncio.sleep(0.05)
|
|
||||||
raise ConnectionError("EX连接池无可用连接")
|
# 3. 池满:排队等待 release 唤醒(或超时)
|
||||||
|
now = time.monotonic()
|
||||||
|
if deadline is not None and now >= deadline:
|
||||||
|
raise ExPoolTimeoutError(self, timeout)
|
||||||
|
try:
|
||||||
|
if deadline is None:
|
||||||
|
await self._cond.wait()
|
||||||
|
else:
|
||||||
|
await asyncio.wait_for(self._cond.wait(), timeout=deadline - now)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
raise ExPoolTimeoutError(self, timeout)
|
||||||
|
|
||||||
def _next_available_server(self) -> str | None:
|
def _next_available_server(self) -> str | None:
|
||||||
for h in self.servers:
|
for h in self.servers:
|
||||||
@@ -105,15 +146,23 @@ class ExPool:
|
|||||||
return h
|
return h
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def release(self, conn: ExTdxConnection, *, broken: bool = False) -> None:
|
def _schedule_close(self, conn: ExTdxConnection) -> None:
|
||||||
if broken or not conn.alive:
|
async def _close():
|
||||||
async with self._lock:
|
|
||||||
self._total = max(0, self._total - 1)
|
|
||||||
self._server_count[conn.host] = max(0, self._server_count.get(conn.host, 0) - 1)
|
|
||||||
try:
|
try:
|
||||||
await conn.close()
|
await conn.close()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
return
|
try:
|
||||||
async with self._lock:
|
asyncio.get_running_loop().create_task(_close())
|
||||||
|
except RuntimeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def release(self, conn: ExTdxConnection, *, broken: bool = False) -> None:
|
||||||
|
async with self._cond:
|
||||||
|
if broken or not conn.alive:
|
||||||
|
self._total = max(0, self._total - 1)
|
||||||
|
self._server_count[conn.host] = max(0, self._server_count.get(conn.host, 0) - 1)
|
||||||
|
self._schedule_close(conn)
|
||||||
|
else:
|
||||||
self._idle.append((conn.host, conn))
|
self._idle.append((conn.host, conn))
|
||||||
|
self._cond.notify_all()
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
"""ExPool 背压/并发语义回归(Issue #1:并发超限 0.15s 秒失败)。
|
||||||
|
|
||||||
|
用 mock 连接模拟:单服务器 + per_server_cap=1 时,
|
||||||
|
- 并发 acquire 超过有效并发 → 排队等待而不是快速失败;
|
||||||
|
- release 唤醒等待者;
|
||||||
|
- acquire 超时异常带池状态;
|
||||||
|
- ex_pool_cap 提升同主机并发。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from pytdxdata.pool.ex_pool import ExPool, ExPoolTimeoutError
|
||||||
|
|
||||||
|
|
||||||
|
class MockConn:
|
||||||
|
"""Minimal stand-in for ExTdxConnection."""
|
||||||
|
|
||||||
|
last_created = 0
|
||||||
|
|
||||||
|
def __init__(self, host: str, port: int, timeout: float, handshake: str = "login"):
|
||||||
|
self.host = host
|
||||||
|
self.port = port
|
||||||
|
self.alive = True
|
||||||
|
self.last_used = time.monotonic()
|
||||||
|
self.closed = False
|
||||||
|
MockConn.last_created += 1
|
||||||
|
|
||||||
|
async def connect(self) -> None:
|
||||||
|
await asyncio.sleep(0) # instant "connect"
|
||||||
|
|
||||||
|
async def close(self) -> None:
|
||||||
|
self.closed = True
|
||||||
|
self.alive = False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def patch_conn(monkeypatch):
|
||||||
|
monkeypatch.setattr("pytdxdata.pool.ex_pool.ExTdxConnection", MockConn)
|
||||||
|
MockConn.last_created = 0
|
||||||
|
|
||||||
|
|
||||||
|
async def test_acquire_queues_instead_of_fast_fail(patch_conn):
|
||||||
|
"""并发 > 有效并发时,后续 acquire 等待 release,而非 0.15s 抛错。"""
|
||||||
|
pool = ExPool(["e1"], min_size=1, max_size=6, per_server_cap=1)
|
||||||
|
await pool.start()
|
||||||
|
c1 = await pool.acquire() # 唯一连接被占用
|
||||||
|
t0 = time.monotonic()
|
||||||
|
|
||||||
|
async def holder():
|
||||||
|
await asyncio.sleep(0.5) # 持有 0.5s 后归还
|
||||||
|
await pool.release(c1)
|
||||||
|
|
||||||
|
asyncio.create_task(holder())
|
||||||
|
c2 = await pool.acquire() # 应在 ~0.5s 后拿到,而不是 0.15s 抛错
|
||||||
|
waited = time.monotonic() - t0
|
||||||
|
assert waited >= 0.4, f"acquire 没有等待释放(仅 {waited:.2f}s)"
|
||||||
|
assert c2 is c1 # 归还的是同一条连接
|
||||||
|
|
||||||
|
|
||||||
|
async def test_acquire_timeout_carries_pool_state(patch_conn):
|
||||||
|
"""超时异常信息带池状态(idle/total/servers/cap)。"""
|
||||||
|
pool = ExPool(["e1"], min_size=1, max_size=6, per_server_cap=1, acquire_timeout=0.3)
|
||||||
|
await pool.start()
|
||||||
|
c1 = await pool.acquire()
|
||||||
|
with pytest.raises(ExPoolTimeoutError) as ei:
|
||||||
|
await pool.acquire()
|
||||||
|
msg = str(ei.value)
|
||||||
|
assert "total=1" in msg and "per_server_cap=1" in msg and "servers=1" in msg
|
||||||
|
await pool.release(c1)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_ex_pool_cap_allows_parallel_conns_same_host(patch_conn):
|
||||||
|
"""per_server_cap=2 时单主机可并发持有 2 条连接。"""
|
||||||
|
pool = ExPool(["e1"], min_size=1, max_size=6, per_server_cap=2)
|
||||||
|
await pool.start()
|
||||||
|
got = await asyncio.gather(pool.acquire(), pool.acquire())
|
||||||
|
assert len(got) == 2 and got[0] is not got[1]
|
||||||
|
assert MockConn.last_created == 2
|
||||||
|
for c in got:
|
||||||
|
await pool.release(c)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_concurrent_acquire_all_succeed(patch_conn):
|
||||||
|
"""Issue 场景:单服务器 + cap=2,6 个并发任务应全部拿到连接(排队)而非部分失败。"""
|
||||||
|
pool = ExPool(["e1"], min_size=2, max_size=6, per_server_cap=2)
|
||||||
|
await pool.start()
|
||||||
|
|
||||||
|
async def task(i):
|
||||||
|
c = await pool.acquire()
|
||||||
|
await asyncio.sleep(0.1) # 模拟一次 1m 分页拉取
|
||||||
|
await pool.release(c)
|
||||||
|
return i
|
||||||
|
|
||||||
|
results = await asyncio.gather(*[task(i) for i in range(6)])
|
||||||
|
assert sorted(results) == list(range(6)) # 全部成功,零失败
|
||||||
|
|
||||||
|
|
||||||
|
async def test_release_notifies_waiters(patch_conn):
|
||||||
|
"""broken 归还(连接死亡)也要唤醒等待者(槽位回收后可扩容)。"""
|
||||||
|
pool = ExPool(["e1"], min_size=1, max_size=1, per_server_cap=1)
|
||||||
|
await pool.start()
|
||||||
|
c1 = await pool.acquire()
|
||||||
|
c1.alive = False
|
||||||
|
|
||||||
|
async def broken_release():
|
||||||
|
await asyncio.sleep(0.2)
|
||||||
|
await pool.release(c1, broken=True)
|
||||||
|
|
||||||
|
asyncio.create_task(broken_release())
|
||||||
|
c2 = await pool.acquire() # 槽位被回收→扩容新建
|
||||||
|
assert c2 is not c1
|
||||||
|
await pool.release(c2)
|
||||||
Reference in new issue
Block a user