perf: 全市场报价改单连接流水线(3.5s → 0.33s)+ 修复基金笔数口径 (#9)
## 性能:get_quotes 全市场 5,227 只 A股 3.5s → 0.33s 协议硬上限 80 只/请求 → 必须切 66 片;老实现是纯串行 for 循环, 每片一次往返(≈60ms),3.5s 全花在等网络上。 发现通达信响应是自描述长度帧且服务器按序应答,因此同一连接可连发 多条请求只付一次往返:实测单连接流水 66 请求 526ms(串行 7,372ms)。 - TdxConnection.execute_pipeline(cmds):一次下发全部 → 按序读回;中途失败强制断连 - _quotes_std 改工作窃取调度(快台多干,避免被最慢节点拖累) - DynamicPool.warm(n):并发预建连接(acquire 在全局锁内一次一条,22 并发首拉会多花 5~6s) - CacheManager.get_or_fetch_obj:回源直接返回对象,省掉冷路径 pickle→unpickle 白烧 - pool_max 默认 12 → 24 调优实测 p50/max:每台 1 连接×深度 6 = 455ms;每台 2 连接×深度 3 = 325ms。 端到端(全默认):首次 783ms、稳态 300~343ms、热缓存 67ms。 正确性:用昨收价交叉比对 30 只零错配,流水线与逐片结果集合完全一致。 ## 修复:指数 / ETF / LOF 的 trade_count 不再伪装成笔数 通达信对指数和多数场内基金不提供笔数,那 4 字节里塞的是「价格 × 1000」 (指数是「均价 × 1000」)。旧版直接当笔数用,tdx transactions sh510310 会打印出「3994 笔」这种假数据。 - 新增 TransactionRecord.trade_count_available - 新增 symbols.has_order_count(market, code) 按品种判定 (沪市 501x/502x/506x 实测有真实笔数,不能按「5 开头=基金」一刀切) - 历史通道 0x013001 无笔数字段,同样标记不可用 ## 文档:可转债量纲口径 核查可转债分时/逐笔后确认数据本身无缺陷(16 只当日 + 8 只历史日, 分时末价/现价、Σvol/日线vol、逐笔 VWAP 比值均 1.000000), 但 vol 单位是「手」(=10 张)导致 amount/vol 是均价的 10 倍,对外文档缺失。 - SecurityBar / TransactionRecord / MinuteBar 量纲注释补全 - has_order_count 实测表补沪可转债 110/111/113/118、沪 EB 132、深可转债 123/127/128 - API_REFERENCE 新增跨品种量纲对照表;llms.txt / SKILL.md 同步 ## 版本 0.6.7 ## 测试 306 → 346 全通过(新增 tests/test_pipeline.py 11 项 + 基金笔数相关回归) Co-authored-by: openbot-coder <openbot-coder@users.noreply.github.com>
This commit is contained in:
1 parent
2c07400a3a
commit
8a78156145
17 files changed
+860
-61
No files matched your search
@@ -0,0 +1,236 @@
|
||||
"""单连接流水线(execute_pipeline)单元测试:不连真实网络。
|
||||
|
||||
关键防线:流水线把多条请求塞进同一条 TCP 流,一旦响应与请求错配就是
|
||||
「价格张冠李戴」级别的灾难,所以这里逐项锁死顺序、长度与失败语义。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
import unittest.mock as mock
|
||||
|
||||
import pytest
|
||||
|
||||
from pytdxdata.pool.connection import TdxConnection
|
||||
from pytdxdata.pool.pool import PooledConnection
|
||||
from pytdxdata.protocol.commands.base import BaseCommand
|
||||
|
||||
|
||||
class EchoCmd(BaseCommand[bytes]):
|
||||
"""请求体即 tag,响应体原样带回,便于断言顺序。"""
|
||||
|
||||
def __init__(self, tag: bytes) -> None:
|
||||
self.tag = tag
|
||||
|
||||
def build_request(self) -> bytes:
|
||||
return b"REQ:" + self.tag
|
||||
|
||||
def parse_response(self, body: bytes) -> bytes:
|
||||
return body
|
||||
|
||||
|
||||
class FakeQuote:
|
||||
"""最小报价替身:stamp() 会写 symbol,需要 market/code 是可 int/str 的真值。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.market = 0
|
||||
self.code = "000001"
|
||||
self.symbol = ""
|
||||
|
||||
|
||||
def frame(body: bytes) -> bytes:
|
||||
"""构造一帧未压缩响应(zipsize == unzipsize)。"""
|
||||
return struct.pack("<IIIHH", 7654321, 0, 0x053E, len(body), len(body)) + body
|
||||
|
||||
|
||||
class FakeReader:
|
||||
def __init__(self, data: bytes) -> None:
|
||||
self.data = data
|
||||
self.pos = 0
|
||||
|
||||
async def read(self, n: int) -> bytes:
|
||||
chunk = self.data[self.pos:self.pos + n]
|
||||
self.pos += len(chunk)
|
||||
return chunk
|
||||
|
||||
|
||||
class FakeWriter:
|
||||
def __init__(self) -> None:
|
||||
self.written = b""
|
||||
self.closed = False
|
||||
self.transport = FakeQuote()
|
||||
|
||||
def write(self, data: bytes) -> None:
|
||||
self.written += data
|
||||
|
||||
async def drain(self) -> None:
|
||||
pass
|
||||
|
||||
def is_closing(self) -> bool:
|
||||
return self.closed
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
async def wait_closed(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def make_conn(payload: bytes, host: str = "1.2.3.4") -> TdxConnection:
|
||||
conn = TdxConnection(host, 7709, 5.0)
|
||||
conn._reader = FakeReader(payload) # type: ignore[assignment]
|
||||
conn._writer = FakeWriter() # type: ignore[assignment]
|
||||
return conn
|
||||
|
||||
|
||||
# ---------- 基本语义 ----------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_cmds_returns_empty():
|
||||
conn = make_conn(b"")
|
||||
assert await conn.execute_pipeline([]) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_paired_in_order():
|
||||
bodies = [b"AAA", b"BBB", b"CCC"]
|
||||
conn = make_conn(b"".join(frame(b) for b in bodies))
|
||||
cmds = [EchoCmd(t) for t in (b"1", b"2", b"3")]
|
||||
got = await conn.execute_pipeline(cmds)
|
||||
assert got == bodies # 顺序必须与 cmds 严格一致
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_requests_sent_at_once():
|
||||
conn = make_conn(b"".join(frame(b"X") for _ in range(3)))
|
||||
cmds = [EchoCmd(t) for t in (b"a", b"b", b"c")]
|
||||
await conn.execute_pipeline(cmds)
|
||||
assert conn._writer.written == b"REQ:aREQ:bREQ:c"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_closed_connection_raises():
|
||||
conn = make_conn(b"")
|
||||
conn._writer = None
|
||||
with pytest.raises(ConnectionError):
|
||||
await conn.execute_pipeline([EchoCmd(b"1")])
|
||||
|
||||
|
||||
# ---------- 失败语义:响应流错配后必须废连接 ----------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_truncated_stream_force_closes():
|
||||
# 只给第一帧完整的、第二帧只有半个头 → 读不满,抛错且连接作废
|
||||
conn = make_conn(frame(b"AAA") + b"\x00" * 5)
|
||||
with pytest.raises(ConnectionError):
|
||||
await conn.execute_pipeline([EchoCmd(b"1"), EchoCmd(b"2")])
|
||||
assert conn._writer is None # _force_close 已清空 writer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_body_length_mismatch_propagates():
|
||||
bad = struct.pack("<IIIHH", 7654321, 0, 0x053E, 99, 99) + b"short"
|
||||
conn = make_conn(bad)
|
||||
# 帧头自称 99 字节但流里没有 → _recv_exact 读空,连接作废
|
||||
with pytest.raises(ConnectionError):
|
||||
await conn.execute_pipeline([EchoCmd(b"1")])
|
||||
assert conn._writer is None
|
||||
|
||||
|
||||
# ---------- 池包装转发 ----------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pooled_connection_forwards_pipeline():
|
||||
inner = mock.AsyncMock()
|
||||
inner.execute_pipeline = mock.AsyncMock(return_value=["r1", "r2"])
|
||||
pc = PooledConnection(inner, "host1")
|
||||
got = await pc.execute_pipeline([EchoCmd(b"1"), EchoCmd(b"2")])
|
||||
assert got == ["r1", "r2"]
|
||||
inner.execute_pipeline.assert_awaited_once()
|
||||
|
||||
|
||||
# ---------- api 层:切批是否走流水线 ----------
|
||||
|
||||
def _api_with_fake_pool():
|
||||
from pytdxdata.api import TdxData
|
||||
from pytdxdata.symbols import CN, Symbol
|
||||
|
||||
td = TdxData.__new__(TdxData) # 跳过 __init__(不建网络池)
|
||||
td._std_pool = FakeQuote()
|
||||
td._std_pool.max_size = 24
|
||||
td._std_pool.servers = [f"s{i}" for i in range(11)]
|
||||
td._std_pool.acquire = mock.AsyncMock()
|
||||
td._std_pool.release = mock.AsyncMock()
|
||||
td._std_pool.warm = mock.AsyncMock() # 预建连接是网络动作,测试里空转
|
||||
return td, Symbol, CN
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quotes_std_uses_pipeline_for_multi_page():
|
||||
td, Symbol, CN = _api_with_fake_pool()
|
||||
syms = [Symbol(0, f"{i:06d}", CN) for i in range(1, 401)] # 5 片
|
||||
chunks = []
|
||||
|
||||
async def fake_pipeline(batch_group):
|
||||
chunks.append(len(batch_group))
|
||||
return [[FakeQuote()] * 80 for _ in batch_group]
|
||||
|
||||
td._execute_std_pipeline = fake_pipeline
|
||||
out = await td._quotes_std(syms, pipeline=True)
|
||||
assert len(out) == 5 * 80
|
||||
assert sum(chunks) == 5 # 每片恰好被处理一次,不多不少
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quotes_std_single_page_skips_pipeline():
|
||||
td, Symbol, CN = _api_with_fake_pool()
|
||||
called = []
|
||||
td._execute_std_pipeline = mock.AsyncMock(return_value=[])
|
||||
|
||||
async def fake_std(cmd):
|
||||
called.append(1)
|
||||
return []
|
||||
|
||||
td._execute_standard = fake_std
|
||||
syms = [Symbol(0, f"{i:06d}", CN) for i in range(1, 51)] # 1 片
|
||||
await td._quotes_std(syms, pipeline=True)
|
||||
assert called == [1]
|
||||
td._execute_std_pipeline.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quotes_std_no_pipeline_flag_uses_plain_path():
|
||||
td, Symbol, CN = _api_with_fake_pool()
|
||||
|
||||
async def fake_std(cmd):
|
||||
return [FakeQuote()] * 80
|
||||
|
||||
td._execute_standard = fake_std
|
||||
td._execute_std_pipeline = mock.AsyncMock(return_value=[])
|
||||
syms = [Symbol(0, f"{i:06d}", CN) for i in range(1, 401)]
|
||||
out = await td._quotes_std(syms, pipeline=False)
|
||||
assert len(out) == 5 * 80
|
||||
td._execute_std_pipeline.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_failure_falls_back_to_per_page():
|
||||
"""节点不支持流水线时,降级逐片,结果条数不变。"""
|
||||
td, Symbol, CN = _api_with_fake_pool()
|
||||
|
||||
pool = td._std_pool
|
||||
pool.acquire = mock.AsyncMock(return_value=mock.Mock(
|
||||
execute_pipeline=mock.AsyncMock(side_effect=ConnectionError("boom"))))
|
||||
pool.release = mock.AsyncMock()
|
||||
|
||||
calls = []
|
||||
|
||||
async def fake_std(cmd):
|
||||
calls.append(1)
|
||||
return [FakeQuote()] * 80
|
||||
|
||||
td._execute_standard = fake_std
|
||||
syms = [Symbol(0, f"{i:06d}", CN) for i in range(1, 241)] # 3 片
|
||||
out = await td._execute_std_pipeline(
|
||||
[syms[i:i + 80] for i in range(0, len(syms), 80)])
|
||||
assert len(out) == 3 and len(calls) == 3
|
||||
assert pool.release.await_args.kwargs.get("broken") is True
|
||||
@@ -206,6 +206,50 @@ def test_mac_transaction_parse_negative_trade_count_auction():
|
||||
assert max(r.trade_count for r in recs) == 3
|
||||
|
||||
|
||||
def test_mac_transaction_etf_trade_count_unavailable():
|
||||
"""ETF/LOF 的 trade_count 是「价格×1000」占位,必须标记为不可用。
|
||||
|
||||
实测(2026-10-22 sh510310 15:27:35):价 3.989、原字段 3994;
|
||||
旧行为把它当笔数打印成「3994笔」,是假的。
|
||||
"""
|
||||
body = bytearray(39 + 2 * 18)
|
||||
struct.pack_into("<H", body, 29, 2)
|
||||
struct.pack_into("<IfIiH", body, 39, 56000, 3.989, 15, 3994, 5)
|
||||
struct.pack_into("<IfIiH", body, 39 + 18, 56003, 3.989, 4, 3994, 5)
|
||||
|
||||
for code, market, expect in (("510310", 1, False), ("159915", 0, False),
|
||||
("600000", 1, True), ("501050", 1, True)):
|
||||
cmd = MacTransactionCmd(market, code, 20261022, 0, 100)
|
||||
recs = cmd.parse_response(bytes(body))
|
||||
assert recs[0].trade_count_available is expect, code
|
||||
if expect:
|
||||
assert recs[0].trade_count == 3994
|
||||
else:
|
||||
assert recs[0].trade_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("market,code,expect", [
|
||||
# 股票 / 可转债 / 回购:有笔数
|
||||
(1, "600000", True), (1, "688111", True), (1, "689009", True),
|
||||
(0, "000001", True), (0, "300750", True), (0, "128136", True),
|
||||
(1, "204001", True),
|
||||
# 沪市 LOF 50x:实测有真实笔数,不能一刀切
|
||||
(1, "501050", True), (1, "501018", True), (1, "502000", True), (1, "506000", True),
|
||||
# 沪市 ETF/LOF 51/52/56/58:无笔数
|
||||
(1, "510310", False), (1, "511990", False), (1, "512880", False),
|
||||
(1, "513050", False), (1, "515000", False), (1, "518880", False),
|
||||
(1, "560010", False), (1, "562500", False), (1, "588000", False),
|
||||
# 深市基金:无笔数
|
||||
(0, "159915", False), (0, "159001", False),
|
||||
(0, "161725", False), (0, "160119", False), (0, "165520", False),
|
||||
# 指数:无笔数
|
||||
(1, "000001", False), (0, "399001", False), (1, "880305", False),
|
||||
])
|
||||
def test_has_order_count(market, code, expect):
|
||||
from pytdxdata.symbols import has_order_count
|
||||
assert has_order_count(market, code) is expect
|
||||
|
||||
|
||||
# ---------- 跨时区时间戳归一化(美股 EX 通道) ----------
|
||||
#
|
||||
# 通达信 EX 通道对美股返回的时间戳把两个时区拼在一起:日期字段是**美东交易日**、
|
||||
|
||||
Reference in new issue
Block a user