From 1cd8f692a189c153b8e7df91e990272fe4bb9c46 Mon Sep 17 00:00:00 2001 From: openbot-coder Date: Fri, 30 Oct 2026 16:57:29 +0800 Subject: [PATCH] =?UTF-8?q?fix(pool):=20=E6=9C=8D=E5=8A=A1=E5=99=A8?= =?UTF-8?q?=E4=B8=AD=E9=80=94=E6=8E=89=E7=BA=BF=E4=B8=8D=E5=86=8D=E6=8B=96?= =?UTF-8?q?=E5=9E=AE=E6=95=B4=E6=89=B9=E4=BB=BB=E5=8A=A1=20+=20=E5=8F=91?= =?UTF-8?q?=E5=B8=83=200.7.2=20(#16)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(pool): 服务器中途掉线不再拖垮整批任务 三种调度细节会把「一台机器坏了」放大成「整批卡住」: 1. acquire() 在池锁内做 TCP 握手 + 探针 —— 一台主机握手卡满 timeout, 所有并发协程的 acquire/release 一起被挡。改为锁内只预留槽位、 握手在锁外做,失败时回滚计数。 2. 扩容失败分支 continue 且无截止检查 —— 全节点失联时无限重试, 永远不抛 PoolTimeoutError。改为同一次 acquire 内每台主机只试一遍, 试完立刻抛错;再加池级熔断(BLACKOUT_COOLDOWN=2s),期内直接失败, 任一连接成功即解除。否则上层「按标的逐只重试」会把 16×6000 次握手 放大成 60s+ 的空转。 3. 批量 K 线失败批次攒到 gather 结束再串行补 —— 每只重付一次 RTT。 改为 extendleft 回队首由健康连接立刻接手(每只最多 2 次), 尾部兜底也改成信号量并发。分页器同理:worker 退出时把剩余页放回 共享 leftover,活着的 worker 接手,全部退出后仍有剩页则借新连接补完。 EX 池同步改为锁外建连,并给握手失败主机加 30s 拉黑。 实测(16 台替身节点、6000 只 × 240 根 1 分钟 K 线、0.9s 后变黑洞): 无故障 1.79s → 1.80s 2/16 死 2.28s → 1.91s 6/16 死 2.97s → 2.30s 16/16 死 >60s 未返回 → 2.37s 新增 tests/test_failover.py(7 项,含熔断期不再握手)与 scripts/bench_failover.py(对比压测)。 * chore(release): 0.7.2 - server_time 解码修正(0.7.1 及之前按「小时 + 百万分之一小时」解析是错的) - 服务器中途掉线的四项隔离机制(锁外建连 / 本轮不重试 / 池级熔断 / 批次回队) - 版本号四处对齐:pyproject.toml、__init__.py、uv.lock、CHANGELOG --------- Co-authored-by: openbot-coder --- CHANGELOG.md | 41 +++- docs/API_REFERENCE.md | 2 +- docs/internals/cache-pool.md | 24 ++ pyproject.toml | 2 +- scripts/bench_failover.py | 147 ++++++++++++ src/pytdxdata/__init__.py | 2 +- src/pytdxdata/api.py | 46 +++- src/pytdxdata/pool/ex_pool.py | 91 +++++--- src/pytdxdata/pool/pool.py | 170 ++++++++++---- src/pytdxdata/scheduler/paginator.py | 86 +++++-- tests/test_failover.py | 329 +++++++++++++++++++++++++++ uv.lock | 2 +- 12 files changed, 838 insertions(+), 104 deletions(-) create mode 100644 scripts/bench_failover.py create mode 100644 tests/test_failover.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bfa387..9153f29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,43 @@ 本项目遵循 [语义化版本](https://semver.org/lang/zh-CN/)。 -## [未发布] +## [0.7.2] - 2026-10-30 + +### 可靠性 + +- **服务器中途掉线不再拖垮整批任务**。 + + 此前三种调度细节会把「一台机器坏了」放大成「整批卡住」: + + | 问题 | 后果 | 现在 | + |---|---|---| + | `acquire()` 在**池锁内**做 TCP 握手 + 探针 | 一台坏机握手卡满 `timeout`,所有并发协程一起停住 | 锁内只预留槽位,握手在锁外做 | + | 扩容失败分支 `continue` 且无截止检查 | 全节点失联时无限重试,永远不抛 `PoolTimeoutError` | 每台主机同一次 `acquire` 内只试一遍,试完立刻抛错 | + | 全灭后上层逐只重试 | 6,000 只 × 16 台 × 握手的空转 | 池级熔断 2 秒,熔断期内直接失败;有连接成功即解除 | + + 实测(16 台替身节点、6,000 只 × 240 根 1 分钟 K 线、跑起来 0.9 秒后 6 台变黑洞): + + | 场景 | 0.7.1 | 0.7.2 | + |---|---|---| + | 无节点故障 | 1.79 s | 1.80 s | + | 2/16 中途失联 | 2.28 s | **1.91 s** | + | 6/16 中途失联 | 2.97 s | **2.30 s** | + | **16/16 中途失联** | **>60 s 未返回** | **2.37 s** | + +- **批量 K 线失败批次从「事后串行补」改为「立刻回队」**。 + + 流水线中途失败时,原来把整批攒到 `gather` 结束再逐只重试 —— 每只都要重付一次 RTT, + 几百只就是十几秒长尾。现在失败批次直接 `extendleft` 回队首,由**其它健康连接**马上接手 + (每只最多回队 2 次,超限后落进尾部并发兜底)。尾部兜底也从串行改成信号量并发。 + +- **分页器不再因为一条连接坏掉而丢掉它那份页**。 + + 老实现按 `jobs[wid::k]` 静态切片,一个 worker 退出,它那一整份页就没人做了。现在 + worker 退出时把剩余页放回共享 `leftover`,还活着的 worker 会接着做;全部 worker 退出后 + 还有剩页,再借一条新连接补完(最多两轮)。 + +- **EX 池同样改为锁外建连**,并给握手失败的主机加 30 秒拉黑(`ExPool.FAIL_COOLDOWN`), + 避免每次 `acquire` 都重付一遍握手超时。 ### 修复 @@ -885,7 +921,8 @@ pip install -U "pytdxdata[polars]" # 想直出 DataFrame - 初始版本:动态连接池 + 双层 TTL 缓存 + 3 连接并发分页 + 双通道(标准 / MAC)+ 扩展市场(港股 / 美股 / 期货 / 期权) -[Unreleased]: https://github.com/openbot-coder/pytdxdata/compare/v0.7.1...HEAD +[Unreleased]: https://github.com/openbot-coder/pytdxdata/compare/v0.7.2...HEAD +[0.7.2]: https://github.com/openbot-coder/pytdxdata/compare/v0.7.1...v0.7.2 [0.7.1]: https://github.com/openbot-coder/pytdxdata/compare/v0.7.0...v0.7.1 [0.7.0]: https://github.com/openbot-coder/pytdxdata/compare/v0.6.0...v0.7.0 [0.6.0]: https://github.com/openbot-coder/pytdxdata/compare/v0.5.0...v0.6.0 diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md index 7ee834a..c1a41f0 100644 --- a/docs/API_REFERENCE.md +++ b/docs/API_REFERENCE.md @@ -1,6 +1,6 @@ # pytdxdata 对外接口清单 -> 以 `src/pytdxdata` 源码为准整理(v0.7.1)。分四类:Python API / 数据模型 / 辅助模块 / CLI。 +> 以 `src/pytdxdata` 源码为准整理(v0.7.2)。分四类:Python API / 数据模型 / 辅助模块 / CLI。 > > 方法与代码的一致性由 `scripts/check_docs.py` 在 CI 中校验。 diff --git a/docs/internals/cache-pool.md b/docs/internals/cache-pool.md index f6ebeaf..998483d 100644 --- a/docs/internals/cache-pool.md +++ b/docs/internals/cache-pool.md @@ -81,6 +81,30 @@ results = await asyncio.gather(*[ EWMA(指数加权移动平均)延迟让选路对「最近变慢」敏感,而不是被历史均值拖住。 +### 单台服务器中途掉线 + +「掉线」和「慢」是两回事:慢只是多等一会儿,掉线会把整批任务卡住。0.7.2 起有四道闸: + +| 机制 | 作用 | +|---|---| +| **锁外建连** | `acquire` 锁内只预留槽位,TCP 握手 + 探针在锁外做。否则一台主机握手卡满 `timeout` 期间,所有并发协程的 `acquire` / `release` 全被池锁挡住 | +| **本轮不重试同一台** | 一次 `acquire` 里握手失败的主机记进 `avoid`,不再重复试;每台都试完立刻抛 `PoolTimeoutError` | +| **池级熔断** | 全主机失败后进入 `BLACKOUT_COOLDOWN`(2 秒)黑洞期,期内 `acquire` 直接失败、不再握手;任一连接成功即解除 | +| **批次回队 / 份额再分配** | 批量 K 线流水线失败的那一批立刻回队首由健康连接接手;分页器让活着的 worker 接手坏 worker 剩下的页 | + +实测 16 台替身节点、6,000 只 × 240 根 1 分钟 K 线、跑起来 0.9 秒后集体黑洞: + +| 场景 | 0.7.1 | 0.7.2 | +|---|---|---| +| 无节点故障 | 1.79 s | 1.80 s | +| 2/16 中途失联 | 2.28 s | 1.91 s | +| 6/16 中途失联 | 2.97 s | 2.30 s | +| 16/16 中途失联 | **> 60 s 未返回** | **2.37 s** | + +全灭时的行为分两种:**一开始就没有可用连接** → `get_bars` 抛 `PoolTimeoutError`(不再静默 +返回空列表);**跑到一半全灭** → 返回已经取到的部分(不抛错、不挂死)。熔断只在**全部主机** +都握手失败时触发,池子满了排队等待不受影响。 + ### MAC 通道的单台连接上限 `mac_pool_cap` 默认 `4`,与标准池的 `std_pool_cap` 一致——这是**对端单台连接限制**, diff --git a/pyproject.toml b/pyproject.toml index 3791cc8..bd89af9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pytdxdata" -version = "0.7.1" +version = "0.7.2" description = "通达信(TongDaXin)行情数据客户端:动态连接池 + 双层TTL缓存 + 3连接并发分页,纯asyncio零运行时依赖,支持A股/港股/美股/期货/期权" readme = "README.md" requires-python = ">=3.11" diff --git a/scripts/bench_failover.py b/scripts/bench_failover.py new file mode 100644 index 0000000..3595580 --- /dev/null +++ b/scripts/bench_failover.py @@ -0,0 +1,147 @@ +"""坏节点隔离压测:N 台服务器,其中几台**跑到一半突然死掉**。 + +关注两个指标: +1. **总耗时** —— 死节点不能把整批任务拖长(老实现会因为持锁建连 + 串行补 + 重试把长尾放大成倍)。 +2. **数据完整度** —— 死节点不能造成缺数据。 + +用法:: + + python scripts/bench_failover.py --syms 3000 --hosts 16 --dead 5,9 + +输出一行 JSON,便于对比改前/改后。 +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import time + +import pytdxdata.api as api +import pytdxdata.pool.pool as poolmod +from pytdxdata import TdxData +from pytdxdata.models import KlinePeriod, SecurityBar + +NOW = time.monotonic() +PIPE_LATENCY = 0.012 # 一次流水线往返(模拟公网 12ms) +CONNECT_LATENCY = 0.05 # 正常主机 TCP 握手 + 握手命令 +DEAD_HANG = 0.8 # 死节点:请求发出去没响应,最终超时 +CONNECT_HANG = 0.0 # 黑洞握手:连上了但握手永不完(0=直接拒绝) + +DEAD_AT = 0.0 # 死节点从第几秒开始不响应 + + +class FakeConn: + dead: set[int] = set() + calls = 0 + pipes = 0 + dead_pipe_hits = 0 # 打到死节点的流水线次数(含"半开等待") + + def __init__(self, host, port=7709, timeout=3.0): + self.host = host + self.idx = int(host[1:]) + self._alive = True + + @property + def alive(self) -> bool: + return self._alive + + async def connect(self) -> None: + if self._is_dead() and CONNECT_HANG: + # 黑洞节点:TCP 连上但握手包永远不回 —— 真实宕机最常见的样子 + await asyncio.sleep(CONNECT_HANG) + raise TimeoutError(f"{self.host} 握手超时") + await asyncio.sleep(CONNECT_LATENCY) + if self._is_dead(): + raise ConnectionRefusedError(f"{self.host} 已宕机") + + def _is_dead(self) -> bool: + return self.idx in type(self).dead and time.monotonic() - NOW >= DEAD_AT + + async def execute(self, cmd): + await asyncio.sleep(PIPE_LATENCY) + if self._is_dead(): + raise ConnectionError(f"{self.host} 无响应") + type(self).calls += 1 + return self._rows(cmd) + + async def execute_pipeline(self, cmds, **kwargs): + await asyncio.sleep(PIPE_LATENCY) + if self._is_dead(): + type(self).dead_pipe_hits += 1 + await asyncio.sleep(DEAD_HANG) # 半开连接:等到超时才报错 + raise ConnectionError(f"{self.host} 无响应") + type(self).pipes += 1 + return [self._rows(c) for c in cmds] + + async def close(self) -> None: + self._alive = False + + @staticmethod + def _rows(cmd): + if "MacKlineCmd" not in type(cmd).__name__: + return [] + bar = SecurityBar( + market=1, code="600000", open=10.0, high=10.1, low=9.9, close=10.0, + vol=1000, amount=10000, year=2026, month=1, day=1, hour=9, minute=30) + return [bar] * int(getattr(cmd, "count", 1)) # 同一个对象复用,便于控内存 + + +async def run(n_syms: int, n_hosts: int, dead: set[int]) -> dict: + global NOW + NOW = time.monotonic() + FakeConn.dead = dead + FakeConn.calls = 0 + FakeConn.pipes = 0 + FakeConn.dead_pipe_hits = 0 + hosts = [f"h{i}" for i in range(n_hosts)] + api.get_mac_hosts = lambda: hosts + api.get_standard_hosts = lambda: hosts # 别让标准池去碰真实公网主机 + api.get_ex_hosts = lambda: [] + poolmod.schedule_probe = lambda: None # 关掉后台全量测速 + poolmod.TdxConnection = FakeConn # 池内部按名字取,必须在建池之前替身 + # cap=1:一台主机一条连接,坏一台就少一条链路(最坏情况) + td = TdxData(cache_dir=None, pool_min=4, pool_max=24, mac_pool_cap=1) + await td.start() + syms = [f"sh60{i:04d}" for i in range(n_syms)] + t0 = time.monotonic() + try: + bars = await td.get_bars(syms, KlinePeriod.MIN_1, count=240) + finally: + elapsed = time.monotonic() - t0 + await td.close() + return { + "syms": n_syms, + "hosts": n_hosts, + "dead": sorted(dead), + "elapsed_s": round(elapsed, 3), + "bars": len(bars), + "expect_bars": n_syms * 240, + "complete": len(bars) == n_syms * 240, + "pipeline_calls": FakeConn.pipes, + "dead_pipe_hits": FakeConn.dead_pipe_hits, + } + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--syms", type=int, default=3000) + ap.add_argument("--hosts", type=int, default=16) + ap.add_argument("--dead", default="5,9", + help="中途死掉的主机下标,逗号分隔;空串=全健康") + ap.add_argument("--die-at", type=float, default=0.25, + help="开跑后第几秒这些主机开始不响应(0=一开始就死)") + ap.add_argument("--connect-hang", type=float, default=0.0, + help="死节点的握手卡顿秒数(>0 模拟黑洞节点,0=直接拒绝)") + args = ap.parse_args() + dead = {int(x) for x in args.dead.split(",") if x.strip()} + global DEAD_AT, CONNECT_HANG + DEAD_AT = args.die_at + CONNECT_HANG = args.connect_hang + print(json.dumps(asyncio.run(run(args.syms, args.hosts, dead)), + ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/src/pytdxdata/__init__.py b/src/pytdxdata/__init__.py index 1904696..c824017 100644 --- a/src/pytdxdata/__init__.py +++ b/src/pytdxdata/__init__.py @@ -18,7 +18,7 @@ from .models import ( XdxrRecord, ) -__version__ = "0.7.1" +__version__ = "0.7.2" __all__ = [ "TdxData", "Market", diff --git a/src/pytdxdata/api.py b/src/pytdxdata/api.py index 4ae17a7..8080b52 100644 --- a/src/pytdxdata/api.py +++ b/src/pytdxdata/api.py @@ -187,6 +187,9 @@ _BULK_MAX_COUNT = 700 # MAC 单请求返回上限;超过只能逐只分页 # 最多用几台服务器:池里 24 台全用反而更慢 —— 慢节点会成为长尾(最长的那台 # 拖住整轮),实测 count=2 全市场 24 台 2.95s / 只取最快 12 台 1.45s。 _BULK_MAX_CONNS = 12 +# 流水线失败的一批最多回队几次。回队 = 交给**其它健康连接**立刻接手; +# 上限保证全节点失联时不会无限打转,超限后落到尾部并发兜底。 +_BULK_MAX_RETRY = 2 # 交易时段(分钟):上午 9:30-11:30、下午 13:00-15:00,共 240 分钟 _AM_OPEN, _AM_CLOSE = 9 * 60 + 30, 11 * 60 + 30 _PM_OPEN, _PM_CLOSE = 13 * 60, 15 * 60 @@ -973,6 +976,7 @@ class TdxData: by_text = {s.text: s for s in syms} acc: dict[str, list[SecurityBar]] = {} retry: dict[str, Symbol] = {} + attempts: dict[str, int] = {} async def worker(pc) -> None: broken = False @@ -985,10 +989,22 @@ class TdxData: try: res = await pc.execute_pipeline(cmds) except Exception: - # 流水线中途失败 = 响应流已错配,连接作废、整批交回逐只重试 + # 流水线中途失败 = 响应流已错配,连接作废。 + # 这一批**立刻放回队首**:其它健康连接会马上接手, + # 不必等 gather 结束再串行补(老实现每条重试都要重新 + # 付一次 RTT,几百只就是十几秒的长尾)。 broken = True - for s, _, _ in batch: - retry[s.text] = s + again: list[tuple[Symbol, int, int]] = [] + for item in batch: + s = item[0] + n = attempts.get(s.text, 0) + if n < _BULK_MAX_RETRY: + attempts[s.text] = n + 1 + again.append(item) + else: + retry[s.text] = s + if again: + tasks.extendleft(reversed(again)) return for (s, _, _), bars in zip(batch, res): acc.setdefault(s.text, []).extend(bars) @@ -996,19 +1012,27 @@ class TdxData: await pool.release(pc, broken=broken) await asyncio.gather(*(worker(pc) for pc in held)) - # 连接全挂时 worker 会提前 return,队列里剩下的标的没人取 —— 一并交回逐只, + # 连接全挂时 worker 会提前 return,队列里剩下的标的没人取 —— 一并交回重试, # 否则会**静默少数据**(实测 20 只只回 16 只)。 while tasks: s, _, _ = tasks.popleft() retry.setdefault(s.text, s) out = {k: self._finish_bars(v, by_text[k]) for k, v in acc.items()} - for text, s in retry.items(): - try: - out[text] = await self._bars_one( - s, period, start=start, count=count, adjust=adj, - start_date=None, end_date=None) - except Exception: - out.setdefault(text, []) + if retry: + # 尾部兜底也并发做:坏节点把批次吐回来时可能有几十上百只, + # 串行重试等于把「一台机器卡住」放大成「整批卡住」。 + sem = asyncio.Semaphore(max(2, min(len(retry), pool.max_size))) + + async def rescue(text: str, s: Symbol) -> None: + async with sem: + try: + out[text] = await self._bars_one( + s, period, start=start, count=count, adjust=adj, + start_date=None, end_date=None) + except Exception: + out.setdefault(text, []) + + await asyncio.gather(*(rescue(t, s) for t, s in retry.items())) return out async def _bars_one( diff --git a/src/pytdxdata/pool/ex_pool.py b/src/pytdxdata/pool/ex_pool.py index 2626e18..8db82b8 100644 --- a/src/pytdxdata/pool/ex_pool.py +++ b/src/pytdxdata/pool/ex_pool.py @@ -37,6 +37,9 @@ class ExPoolTimeoutError(ConnectionError): class ExPool: """EX通道连接池(港股/期货,端口7727,连接时自动login)。""" + FAIL_COOLDOWN = 30.0 + """握手失败后拉黑该主机的秒数(EX 节点少,冷却比标准池短)。""" + def __init__(self, servers: list[str], *, timeout: float = 10.0, min_size: int = 2, max_size: int = 6, per_server_cap: int = 1, @@ -58,6 +61,9 @@ class ExPool: self._total = 0 self._closed = False self._cond = asyncio.Condition() + # 死节点短期拉黑:一台 EX 主机握手失败后,后续 acquire 直接绕开它, + # 而不是每次都再付一遍 ``self.timeout`` 的握手超时。 + self._fail_until: dict[str, float] = {} async def start(self) -> None: async with self._cond: @@ -72,6 +78,7 @@ class ExPool: self._server_count[host] = self._server_count.get(host, 0) + 1 except Exception as e: log.warning("EX连接失败 %s: %s", host, str(e)[:60]) + self._fail_until[host] = time.monotonic() + self.FAIL_COOLDOWN async def close(self) -> None: self._closed = True @@ -94,8 +101,11 @@ class ExPool: if timeout is None: timeout = self.acquire_timeout deadline = None if timeout is None else time.monotonic() + timeout - async with self._cond: - while True: + avoid: set[str] = set() # 本次 acquire 里已经握手失败、别再试的主机 + while True: + new_conn = None + new_host = None + async with self._cond: # 1. 优先借用空闲连接(过期则丢弃,槽位在锁内回收) if self._idle: _, conn = self._idle.pop() @@ -107,44 +117,65 @@ class ExPool: self._server_count[conn.host] = max(0, self._server_count.get(conn.host, 0) - 1) self._schedule_close(conn) - # 2. 可扩容 → 锁内预留槽位,锁外建连 - new_conn = None - new_host = None + # 2. 可扩容 → 锁内预留槽位,锁外建连(不能持锁做网络 IO) if self._total < self.max_size and self.servers: - host = self._next_available_server() + host = self._next_available_server(avoid) if host is not None: self._total += 1 self._server_count[host] = self._server_count.get(host, 0) + 1 - new_conn = ExTdxConnection(host, 7727, self.timeout, self.handshake_map.get(host, self.handshake)) + new_conn = ExTdxConnection( + host, 7727, self.timeout, + self.handshake_map.get(host, self.handshake)) new_host = host - if new_conn is not None: - # 锁外网络 IO:连接失败回滚槽位后继续等待 + elif avoid and all(h in avoid for h in self.servers): + # 所有主机本次都试过了、全失败:等下去也不会有结果 + raise ExPoolTimeoutError(self, timeout) + + # 3. 没抢到槽位 → 在锁内排队等 release 唤醒(waiter 必须持锁) + if new_conn is None: + now = time.monotonic() + if deadline is not None and now >= deadline: + raise ExPoolTimeoutError(self, timeout) try: - await new_conn.connect() - return new_conn - except Exception as e: - log.warning("EX扩容失败 %s: %s", new_host, str(e)[:60]) - async with self._cond: - self._total = max(0, self._total - 1) - self._server_count[new_host] = max(0, self._server_count.get(new_host, 0) - 1) + if deadline is None: + await self._cond.wait() + else: + await asyncio.wait_for(self._cond.wait(), timeout=deadline - now) + except asyncio.TimeoutError: + raise ExPoolTimeoutError(self, timeout) from None + continue - # 3. 池满:排队等待 release 唤醒(或超时) - now = time.monotonic() - if deadline is not None and now >= deadline: - raise ExPoolTimeoutError(self, timeout) - try: - if deadline is None: - await self._cond.wait() - else: - await asyncio.wait_for(self._cond.wait(), timeout=deadline - now) - except asyncio.TimeoutError: - raise ExPoolTimeoutError(self, timeout) + # ---- 锁外建连 ---- + try: + await new_conn.connect() + async with self._cond: + self._fail_until.pop(new_host, None) + return new_conn + except Exception as e: + log.warning("EX扩容失败 %s: %s", new_host, str(e)[:60]) + avoid.add(new_host) # type: ignore[arg-type] + async with self._cond: + self._total = max(0, self._total - 1) + self._server_count[new_host] = max(0, self._server_count.get(new_host, 0) - 1) + self._fail_until[new_host] = time.monotonic() + self.FAIL_COOLDOWN + self._cond.notify_all() + continue - def _next_available_server(self) -> str | None: + def _next_available_server(self, avoid: set[str] | None = None) -> str | None: + """挑一台**未达并发上限**的主机:优先未冷却的,冷却中的只在别无选择时用。""" + avoid = avoid or set() + fallback: str | None = None + now = time.monotonic() for h in self.servers: - if self._server_count.get(h, 0) < self.per_server_cap: + if h in avoid: + continue + if self._server_count.get(h, 0) >= self.per_server_cap: + continue + if now >= self._fail_until.get(h, 0.0): return h - return None + if fallback is None: + fallback = h + return fallback def _schedule_close(self, conn: ExTdxConnection) -> None: async def _close(): diff --git a/src/pytdxdata/pool/pool.py b/src/pytdxdata/pool/pool.py index 3f36ecf..b0fa4df 100644 --- a/src/pytdxdata/pool/pool.py +++ b/src/pytdxdata/pool/pool.py @@ -75,6 +75,15 @@ class PooledConnection: class DynamicPool: """动态连接池:按服务器维度统计连接数,每台服务器最多 per_server_cap 个连接。""" + BLACKOUT_COOLDOWN = 2.0 + """全主机失败后的熔断时长(秒)。 + + 全部服务器同时不可用时,每次 :meth:`acquire` 都会把每台主机握手一遍再放弃。 + 上层是「按标的逐只重试」的循环,6000 只标的 × 16 台主机 × 握手耗时 = 几分钟 + 的空转。熔断期内直接抛 :class:`PoolTimeoutError`,让调用方快速走完并返回, + 冷却结束再试一次探测全灭是否恢复。 + """ + def __init__(self, servers: list[str], *, port: int = DEFAULT_PORT, timeout: float = 10.0, min_size: int = 4, max_size: int = 12, per_server_cap: int = 1, @@ -104,6 +113,7 @@ class DynamicPool: self._total = 0 self._closed = False self._started = False + self._blackout_until = 0.0 # 全灭熔断截止时刻(monotonic) self._lock = asyncio.Lock() self._cond = asyncio.Condition(self._lock) self._reaper_task: asyncio.Task | None = None @@ -127,10 +137,12 @@ class DynamicPool: break try: pc = await self._new_conn(host) - self._idle.append(pc) - self._total += 1 except Exception as e: log.warning("初始连接失败 %s: %s", host, str(e)[:60]) + continue + async with self._lock: + self._register_locked(pc) + self._idle.append(pc) if self._total == 0: log.warning("连接池初始连接全部失败,将在使用时按需重试") @@ -160,6 +172,12 @@ class DynamicPool: return out async def _new_conn(self, host: str) -> PooledConnection: + """建立一条连接(**纯 IO,不改任何计数**)。 + + 计数由调用方在锁内用 :meth:`_register_locked` / :meth:`_reserve_locked` + 单独完成——建连必须能在池锁之外进行,否则一台坏服务器的握手超时 + 会把所有并发协程一起冻住。 + """ conn = TdxConnection(host, self.port, self.timeout) await conn.connect() if self.probe_factory is not None: @@ -168,12 +186,28 @@ class DynamicPool: except Exception: await conn.close() raise - pc = PooledConnection(conn, host) + return PooledConnection(conn, host) + + def _register_locked(self, pc: PooledConnection) -> None: + """登记一条已建好的连接(调用方需持锁)。""" self._conns.add(pc) - self._server_count[host] = self._server_count.get(host, 0) + 1 + self._server_count[pc.host] = self._server_count.get(pc.host, 0) + 1 + self._total += 1 + self.stats.total = self._total self.stats.created += 1 - self.stats.total = self._total + 1 - return pc + + def _reserve_locked(self, host: str) -> None: + """锁内**预留**一个槽位(真正建连在锁外做)。""" + self._total += 1 + self._server_count[host] = self._server_count.get(host, 0) + 1 + self.stats.total = self._total + + def _unreserve_locked(self, host: str) -> None: + """撤销预留(建连失败时回滚计数)。""" + self._total = max(0, self._total - 1) + if self._server_count.get(host, 0) > 0: + self._server_count[host] -= 1 + self.stats.total = self._total async def close(self) -> None: self._closed = True @@ -208,10 +242,19 @@ class DynamicPool: 单连接约束下,池大小 = 服务器数 × per_server_cap。当并发请求超过池大小时, 超额协程在此**排队等待**(基于 Condition),待其他协程 release 后被唤醒, 而非立即超时放弃。默认 timeout=30s 防止在坏服务器场景下永久挂起。 + + **建连在池锁之外**:锁内只预留槽位,TCP 握手 + 探针放到锁外做。否则 + 一台坏服务器握手超时(最长 ``self.timeout``)期间,所有其它协程的 + acquire/release 全部被这个锁挡住——5 台坏节点能把整批任务拖慢十几秒。 """ deadline = None if timeout is None else (time.monotonic() + timeout) - async with self._cond: - while True: + avoid: set[str] = set() # 本次 acquire 里已握手失败的主机,别再重复试 + while True: + if time.monotonic() < self._blackout_until: + raise PoolTimeoutError( + "所有服务器均不可用(熔断中,稍后自动重试)") + host = "" + async with self._cond: now = time.monotonic() # 1. 优先借用空闲连接(跳过冷却中的服务器,避免反复选中坏节点) while self._idle: @@ -222,35 +265,65 @@ class DynamicPool: return pc self._drop_locked(pc) - # 2. 可扩容(未达上限且有可用服务器)→ 新建连接 - if self._total < self.max_size: - host = self._next_available_server() - if host is None: - # 兜底:全部服务器都在冷却中,而池子未满 —— 此时等 release - # 永远等不到(没人持有),只能挑最健康的一台顶着冷却重试 - host = self._next_available_server(ignore_cooldown=True) - if host: - try: - pc = await self._new_conn(host) - self._total += 1 - self.stats.borrowed += 1 - return pc - except Exception as e: - log.warning("扩容连接失败 %s: %s", host, str(e)[:60]) - self.health.record_failure(host) - # 循环重试:可能换下一台服务器 - continue - - # 3. 池满:排队等待 release 唤醒(或超时) + # 2. 全坏场景下这里必须能退出:扩容失败分支会 continue, + # 没有这一行就会无限重试、永远不会抛 PoolTimeoutError。 if deadline is not None and now >= deadline: raise PoolTimeoutError("连接池获取超时(连接数已达上限且无空闲)") - try: - if deadline is None: - await self._cond.wait() - else: - await asyncio.wait_for(self._cond.wait(), timeout=deadline - now) - except asyncio.TimeoutError: - raise PoolTimeoutError("连接池获取超时(连接数已达上限且无空闲)") + + # 3. 可扩容 → 锁内**只预留槽位** + if self._total < self.max_size: + host = self._next_available_server(avoid=avoid) or "" + if not host: + # 兜底:全部服务器都在冷却中,而池子未满 —— 此时等 release + # 永远等不到(没人持有),只能挑最健康的一台顶着冷却重试 + host = self._next_available_server( + ignore_cooldown=True, avoid=avoid) or "" + if host: + self._reserve_locked(host) + elif avoid and all(h in avoid for h in self.servers): + # 本轮每台主机都握手失败过了 → 再等下去也不会有连接。 + # 同时熔断一小段时间,避免上层「逐只重试」把握手放大成 + # 成千上万次(实测 6000 标的能空转 60s+)。 + self._blackout_until = now + self.BLACKOUT_COOLDOWN + raise PoolTimeoutError( + f"所有服务器均不可用({len(avoid)} 台均已失败)") + + if not host: + # 4. 池满/无可用服务器:排队等 release 唤醒(或超时) + remaining = None if deadline is None else deadline - now + try: + if remaining is None: + await self._cond.wait() + else: + await asyncio.wait_for(self._cond.wait(), timeout=remaining) + except asyncio.TimeoutError: + raise PoolTimeoutError( + "连接池获取超时(连接数已达上限且无空闲)") from None + continue + + # ---- 锁外建连:失败只影响本次请求,不冻结其它协程 ---- + try: + pc = await self._new_conn(host) + except Exception as e: + avoid.add(host) + async with self._cond: + self._unreserve_locked(host) + self._cond.notify_all() # 槽位已回滚,唤醒排队者 + self.health.record_failure(host) + log.warning("扩容连接失败 %s: %s", host, str(e)[:60]) + continue + if self._closed: + await pc.close() + async with self._cond: + self._unreserve_locked(host) + self._cond.notify_all() + raise PoolTimeoutError("连接池已关闭") + # 槽位已在锁内预留,这里只补登记,不重复计数 + self._conns.add(pc) + self.stats.created += 1 + self.stats.borrowed += 1 + self._blackout_until = 0.0 # 有连接成功 → 立刻解除熔断 + return pc async def acquire_many(self, k: int) -> list[PooledConnection]: """批量借出 k 个连接(分页并发用),超额请求排队等待不丢。""" @@ -262,8 +335,11 @@ class DynamicPool: break return out - def _next_available_server(self, *, ignore_cooldown: bool = False) -> str | None: - ranked = self.health.rank(self.servers) + def _next_available_server(self, *, ignore_cooldown: bool = False, + avoid: set[str] | None = None) -> str | None: + """挑一台主机(``avoid`` 里的跳过,用于同一次 acquire 内不重复试错)。""" + avoid = avoid or set() + ranked = self.health.rank([h for h in self.servers if h not in avoid]) # 1. 优先选未冷却且未达并发上限的服务器 for h in ranked: if self.health.in_cooldown(h) and not ignore_cooldown: @@ -311,12 +387,20 @@ class DynamicPool: results = await asyncio.gather( *(self._new_conn(h) for h in hosts), return_exceptions=True) async with self._lock: - for pc in results: - if isinstance(pc, PooledConnection): - self._total += 1 - self.stats.total = self._total - if not self._closed and pc.alive: - self._idle.append(pc) + for host, res in zip(hosts, results): + if not isinstance(res, PooledConnection): + # 建连失败 → 记一次失败,后续 acquire 会把它排到队尾 + self.health.record_failure(host) + continue + pc: PooledConnection = res + if self._closed or not pc.alive or self._total >= self.max_size: + # 建好却发现池已满/已关闭:必须关掉,否则连接泄漏, + # 而且 _total 会被虚增(老实现就在这儿漏计数) + asyncio.create_task(self._safe_close(pc)) + continue + self._register_locked(pc) + self._idle.append(pc) + self._cond.notify_all() async def release(self, pc: PooledConnection, *, broken: bool = False) -> None: if self.stats.borrowed > 0: diff --git a/src/pytdxdata/scheduler/paginator.py b/src/pytdxdata/scheduler/paginator.py index c702225..ce47203 100644 --- a/src/pytdxdata/scheduler/paginator.py +++ b/src/pytdxdata/scheduler/paginator.py @@ -10,6 +10,7 @@ from __future__ import annotations import asyncio import logging +from collections import deque from dataclasses import dataclass from typing import Any, Callable @@ -81,15 +82,53 @@ class Paginator: stop = asyncio.Event() out: list[PageResult] = [] out_lock = asyncio.Lock() + # 静态切片保证「每台连接都干活」(并发摊开),另备一个共享的 + # ``leftover``:某个 worker 的连接坏掉退出时,它**没做完的页**会 + # 放回这里,由还活着的 worker 接着做。老实现没有这层——一个 + # worker 挂掉,它那一整份页就没人做了,要么整轮报错、要么静默缺页。 + leftover: deque[tuple[int, int]] = deque() + q_lock = asyncio.Lock() k = len(slots) workers = [ asyncio.create_task( - self._worker(slots[wid], jobs[wid::k], cmd_factory, - stop, out, out_lock, spec) + self._worker(slots[wid], jobs[wid::k], leftover, q_lock, + cmd_factory, stop, out, out_lock, spec) ) for wid in range(k) ] await asyncio.gather(*workers) + # 收尾补页:所有 worker 都退出后 leftover 里可能还剩页(连接全坏时会 + # 发生)。再借一条**新连接**顺序把它们补完,最多两轮——否则 skip 的页 + # 就变成静默缺数据(strict=False 时最危险:不报错、只是少几页)。 + rounds = 0 + while leftover and not stop.is_set() and rounds < 2: + rounds += 1 + try: + pc = await self.pool.acquire(timeout=5.0) + except PoolTimeoutError: + break + slot = _Slot(pc) + progressed = False + try: + while leftover and not stop.is_set(): + start, count = leftover[0] # 成功后才出队,失败可重试 + items, ok = await self._fetch_one( + slot, cmd_factory(start, count), count) + if not ok: + break + leftover.popleft() + progressed = True + async with out_lock: + out.append(PageResult(start=start, count=count, items=items)) + if spec.stop_when_short and len(items) < count: + stop.set() + finally: + if slot.pc is not None: + await self.pool.release(slot.pc) + if not progressed: + break + if leftover: + log.warning("分页调度:%d 页最终未取到(strict=False 留空)", len(leftover)) finally: for s in slots: if s.pc is not None: @@ -117,21 +156,40 @@ class Paginator: return jobs async def _worker( - self, slot: _Slot, jobs: list[tuple[int, int]], + self, slot: _Slot, + jobs: list[tuple[int, int]], + leftover: "deque[tuple[int, int]]", + q_lock: asyncio.Lock, cmd_factory: Callable[[int, int], Any], stop: asyncio.Event, out: list[PageResult], out_lock: asyncio.Lock, spec: PageSpec, ) -> None: - for start, count in jobs: - if stop.is_set() or slot.pc is None: - return - items, ok = await self._fetch_one(slot, cmd_factory(start, count), count) - if not ok: - return # 换连接失败 → 本 worker 退出(该页留空) - async with out_lock: - out.append(PageResult(start=start, count=count, items=items)) - if spec.stop_when_short and len(items) < count: - stop.set() - return + i = 0 + try: + while True: + if i < len(jobs): + start, count = jobs[i] + i += 1 + else: + # 自己的份额做完了 → 看看有没有别人掉下的活 + async with q_lock: + if stop.is_set() or not leftover: + return + start, count = leftover.popleft() + if slot.pc is None or stop.is_set(): + return + items, ok = await self._fetch_one(slot, cmd_factory(start, count), count) + if not ok: + return # 换连接失败 → 本 worker 退出(剩余页在 finally 里交还) + async with out_lock: + out.append(PageResult(start=start, count=count, items=items)) + if spec.stop_when_short and len(items) < count: + stop.set() + return + finally: + rest = jobs[i:] + if rest: + async with q_lock: + leftover.extendleft(reversed(rest)) async def _fetch_one(self, slot: _Slot, cmd, page_size: int): """单页拉取,失败换连接重试。 diff --git a/tests/test_failover.py b/tests/test_failover.py new file mode 100644 index 0000000..81bfd7a --- /dev/null +++ b/tests/test_failover.py @@ -0,0 +1,329 @@ +"""坏节点隔离回归:**一台服务器中途死掉,不能拖慢整批任务**。 + +三件事必须成立(缺一个就会把「单机故障」放大成「整批卡死」): + +1. **建连不能持池锁** —— 死节点握手要等满 ``timeout``,锁内做会把所有并发 + 协程一起冻住。 +2. **全节点失联要能退出** —— 扩容失败分支 ``continue`` 会无限重试, + 没有截止检查就永远不抛 ``PoolTimeoutError``。 +3. **失败批次立刻回队 / 分页份额交给活着的 worker** —— 否则要么串行重试 + 把长尾放大,要么那一整份数据没人做(静默缺页)。 + +替身全部在进程内,不连真实网络。 +""" +from __future__ import annotations + +import asyncio +import datetime +import time + +import pytest + +from pytdxdata import TdxData +from pytdxdata.models import KlinePeriod, SecurityBar +from pytdxdata.pool.pool import DynamicPool, PoolTimeoutError +from pytdxdata.scheduler.paginator import PageSpec, Paginator + +NEWEST = datetime.datetime.now().replace(microsecond=0) # noqa: DTZ001 + + +# ====================================================================== # +# 一、建连不占池锁 + 全灭可退出 +# ====================================================================== # + +class _HostConn: + """按 host 决定握手行为的替身连接。 + + ``slow_hosts`` 里的主机握手会「卡住」``delay`` 秒后失败(模拟 SYN 被丢弃); + ``dead`` 为真时连握手都不做,立刻拒绝。 + """ + + slow_hosts: dict[str, float] = {} + dead: bool = False + connects: list[str] = [] + + def __init__(self, host, port=7709, timeout=10.0): + self.host = host + self._alive = True + type(self).connects.append(host) + + @property + def alive(self) -> bool: + return self._alive + + async def connect(self) -> None: + delay = type(self).slow_hosts.get(self.host) + if type(self).dead: + raise ConnectionRefusedError(f"{self.host} 拒绝连接") + if delay is not None: + await asyncio.sleep(delay) + raise ConnectionRefusedError(f"{self.host} 握手超时") + + async def execute(self, cmd): + return "ok" + + async def close(self) -> None: + self._alive = False + + +@pytest.fixture +def conns(monkeypatch): + _HostConn.slow_hosts = {} + _HostConn.dead = False + _HostConn.connects = [] + monkeypatch.setattr("pytdxdata.pool.pool.TdxConnection", _HostConn) + return _HostConn + + +@pytest.mark.asyncio +async def test_slow_connect_does_not_freeze_other_borrowers(conns): + """一台主机握手卡 0.5s 时,另一个协程借连接不能跟着等。 + + 老实现把 ``_new_conn`` 放在池锁里:所有 acquire/release 都被这 0.5s 挡住, + 几台坏节点叠加就是十几秒的全局停顿。 + """ + conns.slow_hosts = {"dead": 0.5} + pool = DynamicPool(["dead", "live"], min_size=0, max_size=2, + per_server_cap=1, auto_probe=False) + await pool.start() + + slow = asyncio.create_task(pool.acquire()) # 抢到 "dead",开始卡握手 + await asyncio.sleep(0.05) # 确保它已进入 _new_conn + + t0 = time.monotonic() + fast_pc = await pool.acquire() # 此时应立刻拿到 "live" + elapsed = time.monotonic() - t0 + assert elapsed < 0.25, f"被坏节点的握手阻塞了 {elapsed:.2f}s" + assert fast_pc.host == "live" + + slow_pc = await slow # 换到 "live" 后成功 + assert slow_pc.host == "live" + for pc in (fast_pc, slow_pc): + await pool.release(pc) + await pool.close() + + +@pytest.mark.asyncio +async def test_acquire_gives_up_when_every_host_is_dead(conns): + """全节点失联时必须抛 PoolTimeoutError,而不是无限重试。""" + conns.dead = True + pool = DynamicPool(["d1", "d2"], min_size=0, max_size=4, + per_server_cap=1, auto_probe=False) + await pool.start() + + t0 = time.monotonic() + with pytest.raises(PoolTimeoutError): + # 外层 wait_for 只是防止老实现挂死把测试卡住 + await asyncio.wait_for(pool.acquire(timeout=0.3), timeout=5.0) + assert time.monotonic() - t0 < 2.0 + await pool.close() + + +@pytest.mark.asyncio +async def test_blackout_stops_retrying_every_host_per_call(conns): + """全灭后进入熔断期:后续 acquire 立刻失败,不再把每台主机握手一遍。 + + 没有熔断时,上层「按标的逐只重试」的循环会把 16 台 × 6000 只放大成 + 9 万次握手(实测 6000 只跑 60s 都跑不完)。 + """ + conns.dead = True + pool = DynamicPool(["d1", "d2", "d3"], min_size=0, max_size=8, + per_server_cap=1, auto_probe=False) + await pool.start() + tried = len(conns.connects) # start() 的 min_size 预热(可能已试过 1 台) + + with pytest.raises(PoolTimeoutError): + await asyncio.wait_for(pool.acquire(timeout=1.0), timeout=5.0) + round1 = len(conns.connects) - tried + assert round1 == 3, f"首轮应恰好试完 3 台,实际 {round1}" + tried = len(conns.connects) + + # 熔断期内:不再产生任何建连尝试,且立即抛错 + t0 = time.monotonic() + for _ in range(50): + with pytest.raises(PoolTimeoutError): + await pool.acquire() + assert time.monotonic() - t0 < 0.2 + assert len(conns.connects) == tried, "熔断期内不该再握手" + + # 熔断到期后恢复探测 + pool._blackout_until = 0.0 + with pytest.raises(PoolTimeoutError): + await pool.acquire() + assert len(conns.connects) > tried + await pool.close() + + +# ====================================================================== # +# 二、K 线批量:死节点只损失在途那一批,且不重复打扰 +# ====================================================================== # + +class _FleetConn: + """MAC 通道替身:``m1`` 在成功服务 ``m1_life`` 次后「宕机」。""" + + m1_life = 2 + m1_calls = 0 + m2_calls = 0 + + def __init__(self, host="m1", port=7709, timeout=10.0): + self.host = host + self._alive = True + + @property + def alive(self) -> bool: + return self._alive + + async def connect(self) -> None: + pass + + async def close(self) -> None: + self._alive = False + + def _dead(self) -> bool: + cls = type(self) + if self.host != "m1": + cls.m2_calls += 1 + return False + cls.m1_calls += 1 + return cls.m1_calls > cls.m1_life + + def _rows(self, cmd): + if "MacKlineCmd" not in type(cmd).__name__: + return [] + out = [] + for off in range(cmd.start, cmd.start + cmd.count): + when = NEWEST - datetime.timedelta(minutes=off) + out.append(SecurityBar( + market=1, code="600000", open=10.0, high=10.1, low=9.9, + close=10.0, vol=1000, amount=10000, year=when.year, + month=when.month, day=when.day, hour=when.hour, + minute=when.minute)) + return out + + async def execute(self, cmd): + if self._dead(): + raise ConnectionError("m1 已宕机") + return self._rows(cmd) + + async def execute_pipeline(self, cmds, **kwargs): + if self._dead(): + raise ConnectionError("m1 已宕机") + return [self._rows(c) for c in cmds] + + +def _syms(n: int) -> list[str]: + return [f"sh60{i:04d}" for i in range(n)] + + +@pytest.mark.asyncio +async def test_bulk_bars_recovers_from_midrun_host_death(monkeypatch): + """30 只K线,m1 服务 2 次后宕机:数据一根不少,且不反复打扰死节点。""" + _FleetConn.m1_life = 2 + _FleetConn.m1_calls = 0 + _FleetConn.m2_calls = 0 + monkeypatch.setattr("pytdxdata.api.get_mac_hosts", lambda: ["m1", "m2"]) + monkeypatch.setattr("pytdxdata.pool.pool.TdxConnection", + lambda h, p=7709, t=3.0: _FleetConn(h)) + + td = TdxData(cache_dir=None, pool_min=1, pool_max=4) + await td.start() + t0 = time.monotonic() + try: + bars = await td.get_bars(_syms(30), KlinePeriod.MIN_1, count=240) + finally: + elapsed = time.monotonic() - t0 + await td.close() + + assert len(bars) == 30 * 240, "死节点不能造成缺数据" + # 死节点只该被「在途那一批」碰到,之后靠冷却隔离,不该被反复重试 + assert _FleetConn.m1_calls <= 2 * 4, f"死节点被打了 {_FleetConn.m1_calls} 次" + assert _FleetConn.m2_calls > 0 + assert elapsed < 5.0, f"单台服务器宕机把整批拖到 {elapsed:.2f}s" + + +@pytest.mark.asyncio +async def test_all_hosts_dead_fails_fast(monkeypatch): + """全节点失联:**快速报错**,不能挂死,也不能每只都付满一次 30s 池超时。 + + 老实现里扩容失败分支 ``continue`` 且没有截止检查 → ``acquire`` 无限重试, + 十万次建连把事件循环饿死(协程连取消都投递不进去)。 + """ + _FleetConn.m1_life = 0 + _FleetConn.m1_calls = 0 + _FleetConn.m2_calls = 0 + monkeypatch.setattr("pytdxdata.api.get_mac_hosts", lambda: ["m1"]) + monkeypatch.setattr("pytdxdata.pool.pool.TdxConnection", + lambda h, p=7709, t=3.0: _FleetConn(h)) + + td = TdxData(cache_dir=None, pool_min=1, pool_max=2) + await td.start() + t0 = time.monotonic() + try: + with pytest.raises(PoolTimeoutError): + await asyncio.wait_for( + td.get_bars(_syms(10), KlinePeriod.MIN_1, count=240), timeout=15.0) + finally: + elapsed = time.monotonic() - t0 + await td.close() + assert elapsed < 5.0, f"全灭场景花了 {elapsed:.2f}s 才报错" + # 每台主机只该试一次,不该疯狂重连 + assert _FleetConn.m1_calls < 40, f"重复建连 {_FleetConn.m1_calls} 次" + + +# ====================================================================== # +# 三、分页器的份额再分配 +# ====================================================================== # + +class _PageConn: + def __init__(self, die_after: int | None = None, page_size: int = 100, + total: int = 600): + self.die_after = die_after + self.served = 0 + self.page_size = page_size + self.total = total + + async def execute(self, cmd): + start, count = cmd + if self.die_after is not None and self.served >= self.die_after: + raise ConnectionError("连接已断") + self.served += 1 + return list(range(start, min(start + count, self.total))) + + +class _PagePool: + def __init__(self, conns): + self.conns = conns + self._i = 0 + self.released = [] + + async def acquire_many(self, n, *, timeout=None): + return self.conns[:n] + + async def acquire(self, *, timeout=None): + pc = self.conns[self._i % len(self.conns)] + self._i += 1 + return pc + + async def release(self, pc, *, broken=False): + self.released.append(broken) + + +@pytest.mark.asyncio +async def test_paginator_redistributes_pages_of_dead_worker(): + """1 号连接拉到第 2 页就断:它剩下的页要由其它连接补完,不能整份丢掉。""" + conns = [_PageConn(die_after=2), _PageConn(), _PageConn()] + pag = Paginator(_PagePool(conns), concurrency=3) # type: ignore[arg-type] + spec = PageSpec(page_size=100, total=600) # 6 页,每 worker 2 页 + got = await pag.fetch_all(lambda s, c: (s, c), spec) + assert len(got) == 600, f"丢了 {600 - len(got)} 条" + + +@pytest.mark.asyncio +async def test_paginator_still_distributes_work_across_conns(): + """份额再分配不能退化成「一条连接干完所有页」。""" + conns = [_PageConn() for _ in range(3)] + pag = Paginator(_PagePool(conns), concurrency=3) # type: ignore[arg-type] + spec = PageSpec(page_size=100, total=600) + got = await pag.fetch_all(lambda s, c: (s, c), spec) + assert len(got) == 600 + assert all(c.served >= 1 for c in conns), "每台连接都该分到活" diff --git a/uv.lock b/uv.lock index 78d78ec..c6ea031 100644 --- a/uv.lock +++ b/uv.lock @@ -810,7 +810,7 @@ wheels = [ [[package]] name = "pytdxdata" -version = "0.7.1" +version = "0.7.2" source = { editable = "." } [package.optional-dependencies]