fix(pool): 服务器中途掉线不再拖垮整批任务 + 发布 0.7.2 (#16)
* fix(pool): 服务器中途掉线不再拖垮整批任务 三种调度细节会把「一台机器坏了」放大成「整批卡住」: 1. acquire() 在池锁内做 TCP 握手 + 探针 —— 一台主机握手卡满 timeout, 所有并发协程的 acquire/release 一起被挡。改为锁内只预留槽位、 握手在锁外做,失败时回滚计数。 2. 扩容失败分支 continue 且无截止检查 —— 全节点失联时无限重试, 永远不抛 PoolTimeoutError。改为同一次 acquire 内每台主机只试一遍, 试完立刻抛错;再加池级熔断(BLACKOUT_COOLDOWN=2s),期内直接失败, 任一连接成功即解除。否则上层「按标的逐只重试」会把 16×6000 次握手 放大成 60s+ 的空转。 3. 批量 K 线失败批次攒到 gather 结束再串行补 —— 每只重付一次 RTT。 改为 extendleft 回队首由健康连接立刻接手(每只最多 2 次), 尾部兜底也改成信号量并发。分页器同理:worker 退出时把剩余页放回 共享 leftover,活着的 worker 接手,全部退出后仍有剩页则借新连接补完。 EX 池同步改为锁外建连,并给握手失败主机加 30s 拉黑。 实测(16 台替身节点、6000 只 × 240 根 1 分钟 K 线、0.9s 后变黑洞): 无故障 1.79s → 1.80s 2/16 死 2.28s → 1.91s 6/16 死 2.97s → 2.30s 16/16 死 >60s 未返回 → 2.37s 新增 tests/test_failover.py(7 项,含熔断期不再握手)与 scripts/bench_failover.py(对比压测)。 * chore(release): 0.7.2 - server_time 解码修正(0.7.1 及之前按「小时 + 百万分之一小时」解析是错的) - 服务器中途掉线的四项隔离机制(锁外建连 / 本轮不重试 / 池级熔断 / 批次回队) - 版本号四处对齐:pyproject.toml、__init__.py、uv.lock、CHANGELOG --------- Co-authored-by: openbot-coder <openbot-coder@users.noreply.github.com>
This commit is contained in:
1 parent
dea39db441
commit
1cd8f692a1
12 files changed
+838
-104
No files matched your search
@@ -0,0 +1,329 @@
|
||||
"""坏节点隔离回归:**一台服务器中途死掉,不能拖慢整批任务**。
|
||||
|
||||
三件事必须成立(缺一个就会把「单机故障」放大成「整批卡死」):
|
||||
|
||||
1. **建连不能持池锁** —— 死节点握手要等满 ``timeout``,锁内做会把所有并发
|
||||
协程一起冻住。
|
||||
2. **全节点失联要能退出** —— 扩容失败分支 ``continue`` 会无限重试,
|
||||
没有截止检查就永远不抛 ``PoolTimeoutError``。
|
||||
3. **失败批次立刻回队 / 分页份额交给活着的 worker** —— 否则要么串行重试
|
||||
把长尾放大,要么那一整份数据没人做(静默缺页)。
|
||||
|
||||
替身全部在进程内,不连真实网络。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from pytdxdata import TdxData
|
||||
from pytdxdata.models import KlinePeriod, SecurityBar
|
||||
from pytdxdata.pool.pool import DynamicPool, PoolTimeoutError
|
||||
from pytdxdata.scheduler.paginator import PageSpec, Paginator
|
||||
|
||||
NEWEST = datetime.datetime.now().replace(microsecond=0) # noqa: DTZ001
|
||||
|
||||
|
||||
# ====================================================================== #
|
||||
# 一、建连不占池锁 + 全灭可退出
|
||||
# ====================================================================== #
|
||||
|
||||
class _HostConn:
|
||||
"""按 host 决定握手行为的替身连接。
|
||||
|
||||
``slow_hosts`` 里的主机握手会「卡住」``delay`` 秒后失败(模拟 SYN 被丢弃);
|
||||
``dead`` 为真时连握手都不做,立刻拒绝。
|
||||
"""
|
||||
|
||||
slow_hosts: dict[str, float] = {}
|
||||
dead: bool = False
|
||||
connects: list[str] = []
|
||||
|
||||
def __init__(self, host, port=7709, timeout=10.0):
|
||||
self.host = host
|
||||
self._alive = True
|
||||
type(self).connects.append(host)
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
return self._alive
|
||||
|
||||
async def connect(self) -> None:
|
||||
delay = type(self).slow_hosts.get(self.host)
|
||||
if type(self).dead:
|
||||
raise ConnectionRefusedError(f"{self.host} 拒绝连接")
|
||||
if delay is not None:
|
||||
await asyncio.sleep(delay)
|
||||
raise ConnectionRefusedError(f"{self.host} 握手超时")
|
||||
|
||||
async def execute(self, cmd):
|
||||
return "ok"
|
||||
|
||||
async def close(self) -> None:
|
||||
self._alive = False
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def conns(monkeypatch):
|
||||
_HostConn.slow_hosts = {}
|
||||
_HostConn.dead = False
|
||||
_HostConn.connects = []
|
||||
monkeypatch.setattr("pytdxdata.pool.pool.TdxConnection", _HostConn)
|
||||
return _HostConn
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_slow_connect_does_not_freeze_other_borrowers(conns):
|
||||
"""一台主机握手卡 0.5s 时,另一个协程借连接不能跟着等。
|
||||
|
||||
老实现把 ``_new_conn`` 放在池锁里:所有 acquire/release 都被这 0.5s 挡住,
|
||||
几台坏节点叠加就是十几秒的全局停顿。
|
||||
"""
|
||||
conns.slow_hosts = {"dead": 0.5}
|
||||
pool = DynamicPool(["dead", "live"], min_size=0, max_size=2,
|
||||
per_server_cap=1, auto_probe=False)
|
||||
await pool.start()
|
||||
|
||||
slow = asyncio.create_task(pool.acquire()) # 抢到 "dead",开始卡握手
|
||||
await asyncio.sleep(0.05) # 确保它已进入 _new_conn
|
||||
|
||||
t0 = time.monotonic()
|
||||
fast_pc = await pool.acquire() # 此时应立刻拿到 "live"
|
||||
elapsed = time.monotonic() - t0
|
||||
assert elapsed < 0.25, f"被坏节点的握手阻塞了 {elapsed:.2f}s"
|
||||
assert fast_pc.host == "live"
|
||||
|
||||
slow_pc = await slow # 换到 "live" 后成功
|
||||
assert slow_pc.host == "live"
|
||||
for pc in (fast_pc, slow_pc):
|
||||
await pool.release(pc)
|
||||
await pool.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acquire_gives_up_when_every_host_is_dead(conns):
|
||||
"""全节点失联时必须抛 PoolTimeoutError,而不是无限重试。"""
|
||||
conns.dead = True
|
||||
pool = DynamicPool(["d1", "d2"], min_size=0, max_size=4,
|
||||
per_server_cap=1, auto_probe=False)
|
||||
await pool.start()
|
||||
|
||||
t0 = time.monotonic()
|
||||
with pytest.raises(PoolTimeoutError):
|
||||
# 外层 wait_for 只是防止老实现挂死把测试卡住
|
||||
await asyncio.wait_for(pool.acquire(timeout=0.3), timeout=5.0)
|
||||
assert time.monotonic() - t0 < 2.0
|
||||
await pool.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_blackout_stops_retrying_every_host_per_call(conns):
|
||||
"""全灭后进入熔断期:后续 acquire 立刻失败,不再把每台主机握手一遍。
|
||||
|
||||
没有熔断时,上层「按标的逐只重试」的循环会把 16 台 × 6000 只放大成
|
||||
9 万次握手(实测 6000 只跑 60s 都跑不完)。
|
||||
"""
|
||||
conns.dead = True
|
||||
pool = DynamicPool(["d1", "d2", "d3"], min_size=0, max_size=8,
|
||||
per_server_cap=1, auto_probe=False)
|
||||
await pool.start()
|
||||
tried = len(conns.connects) # start() 的 min_size 预热(可能已试过 1 台)
|
||||
|
||||
with pytest.raises(PoolTimeoutError):
|
||||
await asyncio.wait_for(pool.acquire(timeout=1.0), timeout=5.0)
|
||||
round1 = len(conns.connects) - tried
|
||||
assert round1 == 3, f"首轮应恰好试完 3 台,实际 {round1}"
|
||||
tried = len(conns.connects)
|
||||
|
||||
# 熔断期内:不再产生任何建连尝试,且立即抛错
|
||||
t0 = time.monotonic()
|
||||
for _ in range(50):
|
||||
with pytest.raises(PoolTimeoutError):
|
||||
await pool.acquire()
|
||||
assert time.monotonic() - t0 < 0.2
|
||||
assert len(conns.connects) == tried, "熔断期内不该再握手"
|
||||
|
||||
# 熔断到期后恢复探测
|
||||
pool._blackout_until = 0.0
|
||||
with pytest.raises(PoolTimeoutError):
|
||||
await pool.acquire()
|
||||
assert len(conns.connects) > tried
|
||||
await pool.close()
|
||||
|
||||
|
||||
# ====================================================================== #
|
||||
# 二、K 线批量:死节点只损失在途那一批,且不重复打扰
|
||||
# ====================================================================== #
|
||||
|
||||
class _FleetConn:
|
||||
"""MAC 通道替身:``m1`` 在成功服务 ``m1_life`` 次后「宕机」。"""
|
||||
|
||||
m1_life = 2
|
||||
m1_calls = 0
|
||||
m2_calls = 0
|
||||
|
||||
def __init__(self, host="m1", port=7709, timeout=10.0):
|
||||
self.host = host
|
||||
self._alive = True
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
return self._alive
|
||||
|
||||
async def connect(self) -> None:
|
||||
pass
|
||||
|
||||
async def close(self) -> None:
|
||||
self._alive = False
|
||||
|
||||
def _dead(self) -> bool:
|
||||
cls = type(self)
|
||||
if self.host != "m1":
|
||||
cls.m2_calls += 1
|
||||
return False
|
||||
cls.m1_calls += 1
|
||||
return cls.m1_calls > cls.m1_life
|
||||
|
||||
def _rows(self, cmd):
|
||||
if "MacKlineCmd" not in type(cmd).__name__:
|
||||
return []
|
||||
out = []
|
||||
for off in range(cmd.start, cmd.start + cmd.count):
|
||||
when = NEWEST - datetime.timedelta(minutes=off)
|
||||
out.append(SecurityBar(
|
||||
market=1, code="600000", open=10.0, high=10.1, low=9.9,
|
||||
close=10.0, vol=1000, amount=10000, year=when.year,
|
||||
month=when.month, day=when.day, hour=when.hour,
|
||||
minute=when.minute))
|
||||
return out
|
||||
|
||||
async def execute(self, cmd):
|
||||
if self._dead():
|
||||
raise ConnectionError("m1 已宕机")
|
||||
return self._rows(cmd)
|
||||
|
||||
async def execute_pipeline(self, cmds, **kwargs):
|
||||
if self._dead():
|
||||
raise ConnectionError("m1 已宕机")
|
||||
return [self._rows(c) for c in cmds]
|
||||
|
||||
|
||||
def _syms(n: int) -> list[str]:
|
||||
return [f"sh60{i:04d}" for i in range(n)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_bars_recovers_from_midrun_host_death(monkeypatch):
|
||||
"""30 只K线,m1 服务 2 次后宕机:数据一根不少,且不反复打扰死节点。"""
|
||||
_FleetConn.m1_life = 2
|
||||
_FleetConn.m1_calls = 0
|
||||
_FleetConn.m2_calls = 0
|
||||
monkeypatch.setattr("pytdxdata.api.get_mac_hosts", lambda: ["m1", "m2"])
|
||||
monkeypatch.setattr("pytdxdata.pool.pool.TdxConnection",
|
||||
lambda h, p=7709, t=3.0: _FleetConn(h))
|
||||
|
||||
td = TdxData(cache_dir=None, pool_min=1, pool_max=4)
|
||||
await td.start()
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
bars = await td.get_bars(_syms(30), KlinePeriod.MIN_1, count=240)
|
||||
finally:
|
||||
elapsed = time.monotonic() - t0
|
||||
await td.close()
|
||||
|
||||
assert len(bars) == 30 * 240, "死节点不能造成缺数据"
|
||||
# 死节点只该被「在途那一批」碰到,之后靠冷却隔离,不该被反复重试
|
||||
assert _FleetConn.m1_calls <= 2 * 4, f"死节点被打了 {_FleetConn.m1_calls} 次"
|
||||
assert _FleetConn.m2_calls > 0
|
||||
assert elapsed < 5.0, f"单台服务器宕机把整批拖到 {elapsed:.2f}s"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_hosts_dead_fails_fast(monkeypatch):
|
||||
"""全节点失联:**快速报错**,不能挂死,也不能每只都付满一次 30s 池超时。
|
||||
|
||||
老实现里扩容失败分支 ``continue`` 且没有截止检查 → ``acquire`` 无限重试,
|
||||
十万次建连把事件循环饿死(协程连取消都投递不进去)。
|
||||
"""
|
||||
_FleetConn.m1_life = 0
|
||||
_FleetConn.m1_calls = 0
|
||||
_FleetConn.m2_calls = 0
|
||||
monkeypatch.setattr("pytdxdata.api.get_mac_hosts", lambda: ["m1"])
|
||||
monkeypatch.setattr("pytdxdata.pool.pool.TdxConnection",
|
||||
lambda h, p=7709, t=3.0: _FleetConn(h))
|
||||
|
||||
td = TdxData(cache_dir=None, pool_min=1, pool_max=2)
|
||||
await td.start()
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
with pytest.raises(PoolTimeoutError):
|
||||
await asyncio.wait_for(
|
||||
td.get_bars(_syms(10), KlinePeriod.MIN_1, count=240), timeout=15.0)
|
||||
finally:
|
||||
elapsed = time.monotonic() - t0
|
||||
await td.close()
|
||||
assert elapsed < 5.0, f"全灭场景花了 {elapsed:.2f}s 才报错"
|
||||
# 每台主机只该试一次,不该疯狂重连
|
||||
assert _FleetConn.m1_calls < 40, f"重复建连 {_FleetConn.m1_calls} 次"
|
||||
|
||||
|
||||
# ====================================================================== #
|
||||
# 三、分页器的份额再分配
|
||||
# ====================================================================== #
|
||||
|
||||
class _PageConn:
|
||||
def __init__(self, die_after: int | None = None, page_size: int = 100,
|
||||
total: int = 600):
|
||||
self.die_after = die_after
|
||||
self.served = 0
|
||||
self.page_size = page_size
|
||||
self.total = total
|
||||
|
||||
async def execute(self, cmd):
|
||||
start, count = cmd
|
||||
if self.die_after is not None and self.served >= self.die_after:
|
||||
raise ConnectionError("连接已断")
|
||||
self.served += 1
|
||||
return list(range(start, min(start + count, self.total)))
|
||||
|
||||
|
||||
class _PagePool:
|
||||
def __init__(self, conns):
|
||||
self.conns = conns
|
||||
self._i = 0
|
||||
self.released = []
|
||||
|
||||
async def acquire_many(self, n, *, timeout=None):
|
||||
return self.conns[:n]
|
||||
|
||||
async def acquire(self, *, timeout=None):
|
||||
pc = self.conns[self._i % len(self.conns)]
|
||||
self._i += 1
|
||||
return pc
|
||||
|
||||
async def release(self, pc, *, broken=False):
|
||||
self.released.append(broken)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_paginator_redistributes_pages_of_dead_worker():
|
||||
"""1 号连接拉到第 2 页就断:它剩下的页要由其它连接补完,不能整份丢掉。"""
|
||||
conns = [_PageConn(die_after=2), _PageConn(), _PageConn()]
|
||||
pag = Paginator(_PagePool(conns), concurrency=3) # type: ignore[arg-type]
|
||||
spec = PageSpec(page_size=100, total=600) # 6 页,每 worker 2 页
|
||||
got = await pag.fetch_all(lambda s, c: (s, c), spec)
|
||||
assert len(got) == 600, f"丢了 {600 - len(got)} 条"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_paginator_still_distributes_work_across_conns():
|
||||
"""份额再分配不能退化成「一条连接干完所有页」。"""
|
||||
conns = [_PageConn() for _ in range(3)]
|
||||
pag = Paginator(_PagePool(conns), concurrency=3) # type: ignore[arg-type]
|
||||
spec = PageSpec(page_size=100, total=600)
|
||||
got = await pag.fetch_all(lambda s, c: (s, c), spec)
|
||||
assert len(got) == 600
|
||||
assert all(c.served >= 1 for c in conns), "每台连接都该分到活"
|
||||
Reference in new issue
Block a user