v0.4.0: 初始开源发布

通达信(TDX)行情数据客户端 —— 动态连接池 + 双层TTL缓存 + 3连接并发分页,
纯 asyncio、零运行时依赖。一套接口覆盖 A股(沪/深/北) / 港股 / 美股 / 期货 / 期权。

主要能力:
- 四层架构:api → pool(标准/MAC/EX 三池) → protocol(16 命令模块) → cache(内存+SQLite)
- TdxData 提供 51 个公开异步方法;CLI(tdx) 提供 19 个子命令
- 服务器文件下载 + 解析:板块 block_*.dat / 行业 tdxhy.cfg / 历史财报 gpcw*.zip
- 85 个离线测试(协议 / 连接池 / 缓存 / 分页 / API / 文件解析 / CLI)

协议层参照 easy-tdx 的字节级格式独立实现,不依赖 easy-tdx 包。
This commit is contained in:
openbot-coder committed 2026-09-15 11:38:44 +08:00
commit 9183c3294f
68 files changed
+9249

No files matched your search

+331
View File
@@ -0,0 +1,331 @@
"""统一接口 TdxData 测试(mock 连接,不连真实网络)+ 分页调度器测试"""
from __future__ import annotations
import asyncio
import datetime
import unittest.mock as mock
import pytest
from pytdxdata.api import TdxData
from pytdxdata.models import KlinePeriod, Market, MinuteBar, SecurityBar
from pytdxdata.protocol.commands.mac_tick import MacTickChartCmd
from pytdxdata.scheduler.paginator import PageSpec, Paginator
def _make_bar(i: int) -> SecurityBar:
return SecurityBar(market=1, code="600000", open=10.0, high=10.1, low=9.9,
close=10.0 + i * 0.01, vol=1000, amount=10000,
year=2026, month=8, day=1 + i // 240, hour=9, minute=i % 60)
# ---------- Paginator ----------
class FakePoolConn:
"""模拟连接:返回按start切片的数据。"""
def __init__(self, total_bars: int, page_size: int, fail_pages: set[int] | None = None):
self.total = total_bars
self.page_size = page_size
self.fail_pages = fail_pages or set()
self.executed: list[tuple[int, int]] = []
async def execute(self, cmd):
start, count = cmd # cmd 即 (start, count) 元组
self.executed.append((start, count))
if start // self.page_size in self.fail_pages:
raise ConnectionError("mock失败")
return [_make_bar(i) for i in range(start, min(start + count, self.total))]
class FakePool:
def __init__(self, conns: list[FakePoolConn]):
self.conns = conns
self.released: list = []
self._acquire_idx = 0
async def acquire_many(self, n, *, timeout=None):
return self.conns[:n]
async def acquire(self, *, timeout=None):
pc = self.conns[self._acquire_idx % len(self.conns)]
self._acquire_idx += 1
return pc
async def release(self, pc, *, broken=False):
self.released.append((pc, broken))
@pytest.mark.asyncio
async def test_paginator_parallel_assembly():
"""3连接并行拉2400根(3页×800),按序拼接。"""
conns = [FakePoolConn(2400, 800) for _ in range(3)]
pool = FakePool(conns)
pag = Paginator(pool, concurrency=3) # type: ignore[arg-type]
def factory(s, c):
return (s, c)
result = await pag.fetch_all(factory, PageSpec(page_size=800, total=2400))
assert len(result) == 2400
# 按序:前800根是0-799
assert result[0].close == 10.0
assert result[799].close == pytest.approx(10.0 + 799 * 0.01)
assert result[800].close == pytest.approx(10.0 + 800 * 0.01)
# 每个连接都用了(并发分页发生)
assert all(len(c.executed) >= 1 for c in conns)
@pytest.mark.asyncio
async def test_paginator_retry_and_strict():
"""第1页失败→strict=True抛错。"""
conns = [FakePoolConn(2400, 800, fail_pages={0}) for _ in range(3)]
pool = FakePool(conns)
pag = Paginator(pool, concurrency=3, retries=1, strict=True) # type: ignore[arg-type]
def factory(s, c):
return (s, c)
with pytest.raises(ConnectionError):
await pag.fetch_all(factory, PageSpec(page_size=800, total=2400))
@pytest.mark.asyncio
async def test_paginator_strict_false():
conns = [FakePoolConn(2400, 800, fail_pages={0}) for _ in range(3)]
pool = FakePool(conns)
pag = Paginator(pool, concurrency=3, retries=0, strict=False) # type: ignore[arg-type]
def factory(s, c):
return (s, c)
result = await pag.fetch_all(factory, PageSpec(page_size=800, total=2400))
assert len(result) == 1600 # 第0页留空,1-2页成功
# ---------- TdxData 统一接口 ----------
class FakeApiConn:
"""模拟 execute:按命令类型返回。"""
def __init__(self, server="s1"):
self.server = server
self.host = server
self.alive_flag = True
@property
def alive(self):
return self.alive_flag
async def connect(self):
pass
async def execute(self, cmd):
cname = type(cmd).__name__
if "GetSecurityBarsCmd" in cname:
return [_make_bar(i) for i in range(cmd.start, cmd.start + cmd.count)]
if "GetSecurityQuotesCmd" in cname:
return []
if "MacTransactionCmd" in cname:
return []
if "GetSecurityCountCmd" in cname:
return 100
if "GetXdxrInfoCmd" in cname:
return []
if "GetFinanceInfoCmd" in cname:
return None
return []
async def close(self):
self.alive_flag = False
@pytest.mark.asyncio
async def test_tdxdata_get_kline_with_cache():
"""get_kline:第一次回源,第二次缓存命中(不重复请求)。"""
with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \
mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \
mock.patch("pytdxdata.pool.pool.TdxConnection",
side_effect=lambda h, p=7709, t=10.0: FakeApiConn(h)):
td = TdxData(cache_dir=None, pool_min=1, pool_max=2)
await td.start()
try:
bars1 = await td.get_kline(1, "600000", KlinePeriod.DAY, count=100)
bars2 = await td.get_kline(1, "600000", KlinePeriod.DAY, count=100)
assert len(bars1) == 100
assert len(bars2) == 100
assert bars1[0] == bars2[0]
finally:
await td.close()
class MinuteApiConn(FakeApiConn):
"""捕获 get_minute 实际下发的命令(类级,跨实例共享)。"""
last_minute_cmd = None
async def execute(self, cmd):
if "MacTickChartCmd" in type(cmd).__name__:
MinuteApiConn.last_minute_cmd = cmd
return [MinuteBar(price=9.14, vol=553.0, hour=9, minute=30)]
return await super().execute(cmd)
@pytest.mark.asyncio
async def test_get_minute_default_uses_history_cmd():
"""date=None 走 MAC 通道 MacTickChartCmd(今天):0x122D当日分时。"""
MinuteApiConn.last_minute_cmd = None
with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \
mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \
mock.patch("pytdxdata.pool.pool.TdxConnection",
side_effect=lambda h, p=7709, t=10.0: MinuteApiConn(h)):
td = TdxData(cache_dir=None, pool_min=1, pool_max=2, ex_servers=[])
await td.start()
try:
bars = await td.get_minute(0, "000001")
assert len(bars) == 1
assert bars[0].price == 9.14
cmd = MinuteApiConn.last_minute_cmd
assert isinstance(cmd, MacTickChartCmd)
today = datetime.date.today()
assert cmd.date == today.year * 10000 + today.month * 100 + today.day
finally:
await td.close()
@pytest.mark.asyncio
async def test_tdxdata_get_security_count():
with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \
mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \
mock.patch("pytdxdata.pool.pool.TdxConnection",
side_effect=lambda h, p=7709, t=10.0: FakeApiConn(h)):
td = TdxData(cache_dir=None, pool_min=1, pool_max=2)
await td.start()
try:
n = await td.get_security_count(0)
assert n == 100
finally:
await td.close()
# ---------- Paginator start_offset ----------
@pytest.mark.asyncio
async def test_paginator_start_offset():
"""start_offset 让分页从指定偏移开始,跳过前面的数据。"""
conns = [FakePoolConn(5000, 800) for _ in range(3)]
pool = FakePool(conns)
pag = Paginator(pool, concurrency=3) # type: ignore[arg-type]
def factory(s, c):
return (s, c)
# 从偏移 4000 开始拉 800 根(total=4800)
result = await pag.fetch_all(factory, PageSpec(
page_size=800, total=4800, start_offset=4000))
assert len(result) == 800
assert result[0].close == pytest.approx(10.0 + 4000 * 0.01)
# 验证没有拉取 offset < 4000 的数据
starts_used = [s for conn in conns for s, _ in conn.executed]
assert all(s >= 4000 for s in starts_used)
@pytest.mark.asyncio
async def test_paginator_start_offset_default_zero():
"""不传 start_offset 时行为不变(从0开始)。"""
conns = [FakePoolConn(2400, 800) for _ in range(3)]
pool = FakePool(conns)
pag = Paginator(pool, concurrency=3) # type: ignore[arg-type]
def factory(s, c):
return (s, c)
result = await pag.fetch_all(factory, PageSpec(page_size=800, total=2400))
assert len(result) == 2400
assert result[0].close == 10.0 # 从 offset 0 开始
# ---------- get_kline 日期过滤 ----------
class DateBarApiConn(FakeApiConn):
"""支持日期范围探测的模拟连接(按偏移返回带日期的K线)。"""
# 类级可调:总共多少条、每条的日期序列
total_bars: int = 500
async def execute(self, cmd):
cname = type(cmd).__name__
if "GetSecurityBarsCmd" in cname:
start, count = cmd.start, cmd.count
result = []
for i in range(start, min(start + count, self.total_bars)):
# 每条间隔1天:偏移0=2026-01-01, 偏移1=2026-01-02, ...
dt = datetime.datetime(2026, 1, 1) + datetime.timedelta(days=i)
result.append(SecurityBar(
market=1, code="600000", open=10.0, high=10.1, low=9.9,
close=10.0 + i * 0.01, vol=1000, amount=10000,
year=dt.year, month=dt.month, day=dt.day,
hour=9, minute=30))
return result
return await super().execute(cmd)
@pytest.mark.asyncio
async def test_get_kline_start_date_filters():
"""get_kline(start_date=...) 只返回该日期之后的K线。"""
DateBarApiConn.total_bars = 500
with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \
mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \
mock.patch("pytdxdata.pool.pool.TdxConnection",
side_effect=lambda h, p=7709, t=10.0: DateBarApiConn(h)):
td = TdxData(cache_dir=None, pool_min=1, pool_max=2)
await td.start()
try:
bars = await td.get_kline(
1, "600000", KlinePeriod.DAY,
start_date=datetime.datetime(2026, 3, 1))
# 2026-01-01 偏移0 → 2026-03-01 偏移59,所以 >= 偏移59
assert len(bars) > 0
assert bars[0].year == 2026 and bars[0].month == 3 and bars[0].day == 1
finally:
await td.close()
@pytest.mark.asyncio
async def test_get_kline_date_range():
"""get_kline(start_date=..., end_date=...) 返回日期范围内的K线。"""
DateBarApiConn.total_bars = 500
with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \
mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \
mock.patch("pytdxdata.pool.pool.TdxConnection",
side_effect=lambda h, p=7709, t=10.0: DateBarApiConn(h)):
td = TdxData(cache_dir=None, pool_min=1, pool_max=2)
await td.start()
try:
bars = await td.get_kline(
1, "600000", KlinePeriod.DAY,
start_date=datetime.datetime(2026, 2, 1),
end_date=datetime.datetime(2026, 2, 28))
assert len(bars) == 28
assert bars[0].month == 2 and bars[0].day == 1
assert bars[-1].month == 2 and bars[-1].day == 28
finally:
await td.close()
@pytest.mark.asyncio
async def test_get_kline_since():
"""get_kline_since 便捷方法:返回指定日期之后的所有K线。"""
DateBarApiConn.total_bars = 500
with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \
mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \
mock.patch("pytdxdata.pool.pool.TdxConnection",
side_effect=lambda h, p=7709, t=10.0: DateBarApiConn(h)):
td = TdxData(cache_dir=None, pool_min=1, pool_max=2)
await td.start()
try:
bars = await td.get_kline_since(
1, "600000", KlinePeriod.DAY, datetime.datetime(2026, 6, 1))
assert len(bars) > 0
assert bars[0].year == 2026 and bars[0].month == 6 and bars[0].day == 1
# 最后一条应该是偏移499=2027-04-16
assert bars[-1].year == 2027
finally:
await td.close()
+144
View File
@@ -0,0 +1,144 @@
"""缓存单元测试:TTL/LRU/双层/single-flight"""
from __future__ import annotations
import asyncio
import tempfile
from pathlib import Path
import pytest
from pytdxdata.cache.disk import DiskCache
from pytdxdata.cache.key import CacheKey
from pytdxdata.cache.manager import CacheManager
from pytdxdata.cache.memory import MemoryCache
from pytdxdata.cache.ttl import ttl_for
# ---------- CacheKey ----------
def test_key_roundtrip():
k = CacheKey("kline", 1, "600000", "DAY", 0, 2400, "QFQ")
s = k.to_str()
assert CacheKey.from_str(s) == k
def test_quote_key_order_independent():
a = CacheKey.quote_key([(0, "000001"), (1, "600000")])
b = CacheKey.quote_key([(1, "600000"), (0, "000001")])
assert a.to_str() == b.to_str()
# ---------- TTL ----------
def test_ttl_quote():
assert ttl_for("quote") == 5.0
def test_ttl_kline_minute():
assert ttl_for("kline", period="MIN_1") == 60.0
def test_ttl_kline_day():
t = ttl_for("kline", period="DAY")
assert 0 < t <= 12 * 3600
def test_ttl_hist_transaction_permanent():
assert ttl_for("transaction", date=20260101) == 0.0
def test_ttl_today_transaction():
assert ttl_for("transaction") == 60.0
# ---------- MemoryCache ----------
def test_memory_ttl_expiry():
mc = MemoryCache()
mc.set("k", b"v", ttl=0.1)
assert mc.get("k") == b"v"
import time
time.sleep(0.15)
assert mc.get("k") is None
def test_memory_lru_eviction():
mc = MemoryCache(max_entries=2)
mc.set("a", b"1", 10)
mc.set("b", b"2", 10)
mc.set("c", b"3", 10)
assert len(mc) <= 2
assert mc.get("a") is None # 最旧被逐出
def test_memory_permanent():
mc = MemoryCache()
mc.set("k", b"v", ttl=0) # 永久
assert mc.get("k") == b"v"
# ---------- CacheManager + single-flight ----------
@pytest.mark.asyncio
async def test_manager_double_layer():
tmp = Path(tempfile.mkdtemp()) / "cache.db"
mem = MemoryCache()
disk = DiskCache(tmp)
mgr = CacheManager(mem, disk)
await mgr.start()
key = CacheKey("kline", 1, "600000", "DAY", 0, 800)
fetch_count = 0
async def fetcher():
nonlocal fetch_count
fetch_count += 1
return b"payload"
v1 = await mgr.get_or_fetch(key, fetcher)
v2 = await mgr.get_or_fetch(key, fetcher)
assert v1 == v2 == b"payload"
assert fetch_count == 1 # 第二次命中缓存
# 磁盘持久:新管理器读同一磁盘库
mgr2 = CacheManager(MemoryCache(), DiskCache(tmp))
await mgr2.start()
v3 = await mgr2.get(key)
assert v3 == b"payload"
await mgr.close()
await mgr2.close()
@pytest.mark.asyncio
async def test_manager_single_flight():
mem = MemoryCache()
mgr = CacheManager(mem, None)
key = CacheKey("quote", extra="0:000001")
fetch_count = 0
async def fetcher():
nonlocal fetch_count
fetch_count += 1
await asyncio.sleep(0.05)
return b"x"
await asyncio.gather(
mgr.get_or_fetch(key, fetcher),
mgr.get_or_fetch(key, fetcher),
mgr.get_or_fetch(key, fetcher),
)
assert fetch_count == 1 # single-flight 只回源一次
await mgr.close()
@pytest.mark.asyncio
async def test_manager_invalidate():
mem = MemoryCache()
mgr = CacheManager(mem, None)
key = CacheKey("kline", 1, "600000", "DAY", 0, 800)
await mgr.set(key, b"data")
assert await mgr.get(key) == b"data"
n = await mgr.invalidate("kline", market=1, code="600000")
assert n >= 1
assert await mgr.get(key) is None
await mgr.close()
+158
View File
@@ -0,0 +1,158 @@
"""CLI 服务器文件命令测试:block / industry / financial-list / financial。
全部离线:用 FakeTD 提供异步桩,不发任何网络请求。
"""
from __future__ import annotations
from pytdxdata.cli import cmd_block, cmd_financial, cmd_financial_list, cmd_industry
from pytdxdata.models import FinancialFileInfo, FinancialRecord, IndustryInfo, TdxBlock
_BLOCKS = [
TdxBlock(name="人工智能", category=2, count=3, codes=["300308", "002415", "688111"]),
TdxBlock(name="白酒", category=0, count=1, codes=["600519"]),
]
_INDUSTRY = {
"600000": IndustryInfo(code="600000", tdx_industry="880305", sw_industry="801780"),
"000001": IndustryInfo(code="000001", tdx_industry="880471", sw_industry=""),
}
_FILES = [FinancialFileInfo(filename="gpcw20260331.zip", hash="abc123", filesize=1234567)]
_RECORDS = [
FinancialRecord(code="000001", market=0, report_date=20260331, fields=[1.0, 2.5, 3.5, 4.0]),
FinancialRecord(code="600000", market=1, report_date=20260331, fields=[9.0, 8.0]),
]
class FakeTD:
"""最小异步桩:只实现 4 个服务器文件方法。"""
def __init__(self, blocks=None, industry=None, files=None, records=None):
self.blocks = _BLOCKS if blocks is None else blocks
self.industry = _INDUSTRY if industry is None else industry
self.files = _FILES if files is None else files
self.records = _RECORDS if records is None else records
async def get_block_parsed(self, filename):
return self.blocks
async def get_industry_map(self):
return self.industry
async def get_financial_file_infos(self):
return self.files
async def get_financial_records_parsed(self, filename):
return self.records
# --------------------------------------------------------------------------- #
# block
# --------------------------------------------------------------------------- #
async def test_cmd_block_basic(capsys):
await cmd_block(FakeTD(), "block_gn.dat", 30, False)
out = capsys.readouterr().out
assert "block_gn.dat 板块文件 (2 个板块)" in out
assert "概念" in out and "人工智能" in out # category=2 → 概念
assert "行业" in out and "白酒" in out # category=0 → 行业
assert "300308" in out
assert "其余" not in out
async def test_cmd_block_limit_and_codes(capsys):
await cmd_block(FakeTD(), "block_gn.dat", 1, True)
out = capsys.readouterr().out
assert "人工智能" in out
assert "白酒" not in out # 被 --limit 1 截掉
assert "其余 1 个板块省略" in out
assert "300308 002415 688111" in out # --codes 展开完整成分
async def test_cmd_block_unknown_category_and_empty(capsys):
weird = [TdxBlock(name="神秘", category=9, count=0, codes=[])]
await cmd_block(FakeTD(blocks=weird), "block_x.dat", 30, False)
assert "?" in capsys.readouterr().out
await cmd_block(FakeTD(blocks=[]), "block_gn.dat", 30, False)
assert "0 个板块" in capsys.readouterr().out
# --------------------------------------------------------------------------- #
# industry
# --------------------------------------------------------------------------- #
async def test_cmd_industry_all(capsys):
await cmd_industry(FakeTD(), [], 30, None)
out = capsys.readouterr().out
assert "行业分类 tdxhy.cfg (共 2 只)" in out
assert "600000" in out and "880305" in out and "801780" in out
assert "880471" in out and "-" in out # 申万缺失 → "-"
async def test_cmd_industry_codes_and_missing(capsys):
await cmd_industry(FakeTD(), ["600000", "999999"], 30, None)
out = capsys.readouterr().out
assert "600000" in out and "通达信=880305" in out
assert "999999" in out and "未收录" in out
async def test_cmd_industry_search_and_limit(capsys):
await cmd_industry(FakeTD(), [], 30, "880305")
out = capsys.readouterr().out
assert "匹配 '880305' 1 条" in out
assert "600000" in out and "000001" not in out
await cmd_industry(FakeTD(), [], 1, None)
assert "其余 1 条省略" in capsys.readouterr().out
await cmd_industry(FakeTD(industry={}), [], 30, None)
assert "共 0 只" in capsys.readouterr().out
# --------------------------------------------------------------------------- #
# financial-list
# --------------------------------------------------------------------------- #
async def test_cmd_financial_list(capsys):
await cmd_financial_list(FakeTD(), 30)
out = capsys.readouterr().out
assert "gpcw.txt, 共 1 个" in out
assert "gpcw20260331.zip" in out and "abc123" in out and "1234567" in out
assert "tdx financial gpcw20260331.zip" in out
async def test_cmd_financial_list_limit_and_empty(capsys):
extra = _FILES + [FinancialFileInfo(filename="gpcw20251231.zip", hash="def", filesize=1)]
await cmd_financial_list(FakeTD(files=extra), 1)
assert "其余 1 个省略" in capsys.readouterr().out
await cmd_financial_list(FakeTD(files=[]), 30)
assert "共 0 个" in capsys.readouterr().out
# --------------------------------------------------------------------------- #
# financial
# --------------------------------------------------------------------------- #
async def test_cmd_financial_basic(capsys):
await cmd_financial(FakeTD(), "gpcw20260331.zip", None, 5, 4)
out = capsys.readouterr().out
assert "gpcw20260331.zip 财报记录 (2 条)" in out
assert "市场=深" in out and "市场=沪" in out
assert "报告期=20260331" in out
assert "1 2.5 3.5 4" in out
async def test_cmd_financial_filter_and_fields_zero(capsys):
await cmd_financial(FakeTD(), "gpcw20260331.zip", "600000", 5, 0)
out = capsys.readouterr().out
assert "(1 条)" in out
assert "600000" in out and "000001" not in out
assert "字段数=2" in out
async def test_cmd_financial_limit_and_empty(capsys):
await cmd_financial(FakeTD(), "gpcw20260331.zip", None, 1, 4)
assert "其余 1 条省略" in capsys.readouterr().out
await cmd_financial(FakeTD(records=[]), "gpcw20260331.zip", None, 5, 4)
assert "(0 条)" in capsys.readouterr().out
+319
View File
@@ -0,0 +1,319 @@
"""服务器文件解析器测试:板块 .dat / 行业 tdxhy.cfg / 历史专业财报 gpcw。
全部离线构造二进制样本,不连网络。
"""
from __future__ import annotations
import io
import struct
import unittest.mock as mock
import zipfile
import pytest
from pytdxdata.api import TdxData
from pytdxdata.codec.block import infer_block_category, parse_block_dat
from pytdxdata.codec.financial import parse_financial_dat, parse_financial_file_list
from pytdxdata.codec.industry import parse_tdxhy_cfg
from pytdxdata.models import FinancialFileInfo, FinancialRecord, IndustryInfo, TdxBlock
# --------------------------------------------------------------------------- #
# 样本构造工具
# --------------------------------------------------------------------------- #
_HEADER = 384
_RECORD = 2813
def _make_block_dat(blocks: list[tuple[str, list[str], int, int]]) -> bytes:
"""构造板块 .dat:blocks = [(name, codes, declared_count, type), ...]"""
out = bytearray(_HEADER)
out += struct.pack("<H", len(blocks))
for name, codes, declared_count, btype in blocks:
rec = bytearray()
rec += name.encode("gbk")[:9].ljust(9, b"\x00")
rec += struct.pack("<HH", declared_count, btype)
area = bytearray(2800)
for i, code in enumerate(codes):
area[i * 7 : (i + 1) * 7] = code.encode("ascii").ljust(7, b"\x00")
rec += area
assert len(rec) == _RECORD
out += rec
return bytes(out)
def _make_dat_raw(
index_entries: list[tuple[bytes, int, int]],
data_blob: bytes,
*,
header_report_date: int = 20260331,
max_count: int | None = None,
report_size: int = 8,
) -> bytes:
"""构造 gpcw*.dat:index_entries = [(code_bytes, market, offset), ...]"""
count = len(index_entries) if max_count is None else max_count
idx = b"".join(struct.pack("<6sBL", cb, m, off) for cb, m, off in index_entries)
header = struct.pack("<1hI1H3L", 0, header_report_date, count, 0, report_size, 0)
return header + idx + data_blob
def _make_financial_zip(dat_bytes: bytes, inner_name: str = "gpcw.dat") -> bytes:
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zf.writestr(inner_name, dat_bytes)
return buf.getvalue()
# --------------------------------------------------------------------------- #
# parse_block_dat
# --------------------------------------------------------------------------- #
def test_parse_block_dat_basic():
data = _make_block_dat([
("银行", ["000001", "600036"], 2, 1),
("房地产", ["000002"], 1, 1),
])
blocks = parse_block_dat(data, "block_zs.dat")
assert len(blocks) == 2
assert isinstance(blocks[0], TdxBlock)
assert blocks[0].name == "银行"
assert blocks[0].category == 0
assert blocks[0].count == 2
assert blocks[0].codes == ["000001", "600036"]
assert blocks[1].name == "房地产"
assert blocks[1].codes == ["000002"]
@pytest.mark.parametrize(
("filename", "expected"),
[
("block_zs.dat", 0),
("block_gn.dat", 2),
("block_fg.dat", 3),
("mystery.dat", 0),
("BLOCK_GN.DAT", 2),
],
)
def test_infer_block_category(filename, expected):
assert infer_block_category(filename) == expected
def test_parse_block_dat_too_short():
assert parse_block_dat(b"\x00" * 100) == []
def test_parse_block_dat_count_exceeds_body():
"""声明 3 条但只带 1 条记录 -> 只解析出 1 条。"""
data = _make_block_dat([("银行", ["000001"], 1, 1)])
data = data[: _HEADER] + struct.pack("<H", 3) + data[_HEADER + 2 :]
blocks = parse_block_dat(data, "block_zs.dat")
assert len(blocks) == 1
def test_parse_block_dat_declared_count_over_codes():
"""声明的成分数大于实际写入的代码数 -> 空槽位被跳过。"""
data = _make_block_dat([("银行", ["000001", "600036"], 5, 1)])
blocks = parse_block_dat(data, "block_zs.dat")
assert blocks[0].count == 5
assert blocks[0].codes == ["000001", "600036"]
# --------------------------------------------------------------------------- #
# parse_tdxhy_cfg
# --------------------------------------------------------------------------- #
def test_parse_tdxhy_cfg_basic():
content = "0|000001|A01|||S01\n1|600036|A02|||S02\n".encode("gbk")
result = parse_tdxhy_cfg(content)
assert set(result) == {"000001", "600036"}
assert isinstance(result["000001"], IndustryInfo)
assert result["000001"].tdx_industry == "A01"
assert result["000001"].sw_industry == "S01"
def test_parse_tdxhy_cfg_short_line_skipped():
"""不足 3 列的行被跳过。"""
content = "0|000001\nbadline\n".encode("gbk")
assert parse_tdxhy_cfg(content) == {}
def test_parse_tdxhy_cfg_non_six_digit_skipped():
content = "0|SH600036|A01|||S01\n0|60003|A01\n0|abcdef|A01|||S01\n".encode("gbk")
assert parse_tdxhy_cfg(content) == {}
def test_parse_tdxhy_cfg_missing_sw_column():
"""只有 5 列时申万行业为空串。"""
content = "0|000001|A01||\n".encode("gbk")
result = parse_tdxhy_cfg(content)
assert result["000001"].tdx_industry == "A01"
assert result["000001"].sw_industry == ""
def test_parse_tdxhy_cfg_empty():
assert parse_tdxhy_cfg(b"") == {}
# --------------------------------------------------------------------------- #
# parse_financial_file_list
# --------------------------------------------------------------------------- #
def test_parse_financial_file_list_basic():
data = b"gpcw20260331.zip,abcdef012345,1024\ngpcw20251231.zip,deadbeef,2048\n"
infos = parse_financial_file_list(data)
assert len(infos) == 2
assert isinstance(infos[0], FinancialFileInfo)
assert infos[0].filename == "gpcw20260331.zip"
assert infos[0].hash == "abcdef012345"
assert infos[0].filesize == 1024
def test_parse_financial_file_list_skips_bad_lines():
data = b"badline\nx,y,notanint\nok.zip,abc,10\n\n"
infos = parse_financial_file_list(data)
assert [i.filename for i in infos] == ["ok.zip"]
def test_parse_financial_file_list_empty():
assert parse_financial_file_list(b"") == []
# --------------------------------------------------------------------------- #
# parse_financial_dat
# --------------------------------------------------------------------------- #
def _valid_two_record_dat() -> bytes:
# data 起点 = 20(header) + 2*11(index) = 42
blob = struct.pack("<2f", 1.5, 2.5) + struct.pack("<2f", 3.0, 4.0)
return _make_dat_raw(
[(b"600000", 1, 42), (b"000001", 0, 50)], blob
)
def test_parse_financial_dat_explicit_date():
recs = parse_financial_dat(_valid_two_record_dat(), report_date=20260101)
assert len(recs) == 2
assert isinstance(recs[0], FinancialRecord)
assert recs[0].code == "600000"
assert recs[0].market == 1
assert recs[0].report_date == 20260101
assert recs[0].fields == pytest.approx([1.5, 2.5])
assert recs[1].code == "000001"
assert recs[1].market == 0
assert recs[1].fields == pytest.approx([3.0, 4.0])
def test_parse_financial_dat_fallback_date_from_header():
recs = parse_financial_dat(_valid_two_record_dat(), report_date=0)
assert recs[0].report_date == 20260331
def test_parse_financial_dat_too_short():
assert parse_financial_dat(b"\x00" * 10) == []
def test_parse_financial_dat_zero_report_size():
data = _make_dat_raw([(b"600000", 1, 31)], b"", report_size=0)
assert parse_financial_dat(data) == []
def test_parse_financial_dat_skips_zero_offset_and_empty_code():
blob = struct.pack("<2f", 1.0, 2.0)
# 第 1 条 offset=0 跳过;第 2 条 code 全 0 跳过
data = _make_dat_raw([(b"600000", 1, 0), (b"\x00" * 6, 1, 31)], blob)
assert parse_financial_dat(data) == []
def test_parse_financial_dat_offset_out_of_range():
data = _make_dat_raw([(b"600000", 1, 9999)], struct.pack("<2f", 1.0, 2.0))
assert parse_financial_dat(data) == []
def test_parse_financial_dat_max_count_breaks():
"""max_count 大于实际索引 -> 越界即 break。"""
blob = struct.pack("<2f", 1.0, 2.0)
data = _make_dat_raw([(b"600000", 1, 31)], blob, max_count=5)
recs = parse_financial_dat(data)
assert len(recs) == 1
# --------------------------------------------------------------------------- #
# TdxData 解析接口
# --------------------------------------------------------------------------- #
def _new_tdxdata() -> TdxData:
with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \
mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \
mock.patch("pytdxdata.api.get_ex_hosts", return_value=[]):
return TdxData(cache_dir=None, pool_min=1, pool_max=2, ex_servers=[])
@pytest.mark.asyncio
async def test_api_get_block_parsed():
td = _new_tdxdata()
raw = _make_block_dat([("银行", ["000001"], 1, 1)])
with mock.patch.object(td, "get_block_info", new=mock.AsyncMock(return_value=raw)):
blocks = await td.get_block_parsed("block_gn.dat")
assert len(blocks) == 1
assert blocks[0].category == 2 # gn -> 概念
@pytest.mark.asyncio
async def test_api_get_industry_map():
td = _new_tdxdata()
raw = "0|000001|A01|||S01\n".encode("gbk")
patched = mock.AsyncMock(return_value=raw)
with mock.patch.object(td, "get_report_file", new=patched):
result = await td.get_industry_map()
patched.assert_awaited_once_with("tdxhy.cfg")
assert result["000001"].sw_industry == "S01"
@pytest.mark.asyncio
async def test_api_get_financial_file_infos():
td = _new_tdxdata()
raw = b"gpcw20260331.zip,abc,1024\n"
patched = mock.AsyncMock(return_value=raw)
with mock.patch.object(td, "get_report_file", new=patched):
infos = await td.get_financial_file_infos()
patched.assert_awaited_once_with("tdxfin/gpcw.txt")
assert infos[0].filesize == 1024
@pytest.mark.asyncio
async def test_api_get_financial_records_parsed():
td = _new_tdxdata()
zip_bytes = _make_financial_zip(_valid_two_record_dat())
with mock.patch.object(td, "get_report_file",
new=mock.AsyncMock(return_value=zip_bytes)):
recs = await td.get_financial_records_parsed("tdxfin/gpcw20260331.zip")
assert len(recs) == 2
assert recs[0].report_date == 20260331 # 取自文件名
@pytest.mark.asyncio
async def test_api_get_financial_records_parsed_empty_zip_data():
td = _new_tdxdata()
with mock.patch.object(td, "get_report_file",
new=mock.AsyncMock(return_value=b"")):
assert await td.get_financial_records_parsed("tdxfin/gpcw20260331.zip") == []
@pytest.mark.asyncio
async def test_api_get_financial_records_parsed_no_dat_member():
td = _new_tdxdata()
zip_bytes = _make_financial_zip(b"whatever", inner_name="readme.txt")
with mock.patch.object(td, "get_report_file",
new=mock.AsyncMock(return_value=zip_bytes)):
assert await td.get_financial_records_parsed("nodate.zip") == []
@pytest.mark.asyncio
async def test_api_get_financial_records_parsed_no_date_in_name():
"""文件名无 8 位日期 -> 回退用 .dat 头部报告期。"""
td = _new_tdxdata()
zip_bytes = _make_financial_zip(_valid_two_record_dat())
with mock.patch.object(td, "get_report_file",
new=mock.AsyncMock(return_value=zip_bytes)):
recs = await td.get_financial_records_parsed("gpcw.zip")
assert recs[0].report_date == 20260331
+120
View File
@@ -0,0 +1,120 @@
"""连接池/健康分单元测试(mock服务器,不连真实网络)"""
from __future__ import annotations
import asyncio
import sys
import unittest.mock as mock
import pytest
from pytdxdata.pool.health import HealthEngine
from pytdxdata.pool.pool import DynamicPool, PoolTimeoutError
# ---------- HealthEngine ----------
def test_health_default_score():
h = HealthEngine()
assert h.score("server1") == 1.0
def test_health_failure_penalty():
h = HealthEngine()
h.record_failure("s1")
assert h.score("s1") == 0.5
h.record_failure("s1")
h.record_failure("s1")
assert h.in_cooldown("s1") # 连续3次失败进入冷却
def test_health_success_recovery():
h = HealthEngine()
h.record_failure("s1")
h.record_success("s1")
assert h.score("s1") == pytest.approx(0.7)
assert not h.in_cooldown("s1")
def test_health_rank():
h = HealthEngine()
h.record_success("fast", latency=0.05)
h.record_success("slow", latency=0.5)
ranked = h.rank(["slow", "fast"])
assert ranked[0] == "fast"
# ---------- DynamicPool ----------
class FakeConn:
def __init__(self, host):
self.host = host
self.alive_flag = True
@property
def alive(self):
return self.alive_flag
async def connect(self):
pass
async def ping(self):
return 0.01
async def close(self):
self.alive_flag = False
async def execute(self, cmd):
return "ok"
@pytest.fixture
def fake_pool():
with mock.patch("pytdxdata.pool.pool.TdxConnection", side_effect=lambda h, p=7709, t=10.0: FakeConn(h)):
pool = DynamicPool(["s1", "s2"], min_size=1, max_size=3, per_server_cap=2,
idle_ttl=0.2, scale_down_grace=0, reaper_interval=0.05)
yield pool
@pytest.mark.asyncio
async def test_pool_acquire_release(fake_pool):
await fake_pool.start()
pc = await fake_pool.acquire()
assert pc.server in ("s1", "s2")
assert fake_pool.stats.borrowed == 1
await fake_pool.release(pc)
assert fake_pool.stats.borrowed == 0
await fake_pool.close()
@pytest.mark.asyncio
async def test_pool_acquire_many(fake_pool):
await fake_pool.start()
conns = await fake_pool.acquire_many(2)
assert len(conns) == 2
servers = {c.server for c in conns}
for c in conns:
await fake_pool.release(c)
await fake_pool.close()
@pytest.mark.asyncio
async def test_pool_broken_release(fake_pool):
await fake_pool.start()
pc = await fake_pool.acquire()
await fake_pool.release(pc, broken=True)
assert fake_pool.stats.total <= fake_pool.min_size
await fake_pool.close()
@pytest.mark.asyncio
async def test_pool_scale_down(fake_pool):
"""空闲连接超时后缩容到 min_size。"""
await fake_pool.start()
pc1 = await fake_pool.acquire()
await fake_pool.release(pc1)
pc2 = await fake_pool.acquire()
await fake_pool.release(pc2)
total_before = fake_pool.stats.total
await asyncio.sleep(0.3) # 等 reaper 缩容
assert fake_pool.stats.total <= total_before
await fake_pool.close()
+178
View File
@@ -0,0 +1,178 @@
"""协议层单元测试:帧/价格差分/浮点/命令解析(黄金字节向量,对照 easy-tdx 已验证实现)"""
from __future__ import annotations
import struct
import pytest
from pytdxdata.protocol.frame import FrameHeader, decompress_body, parse_header
from pytdxdata.protocol.price import get_price, put_price, decode_ohlc_diffs
from pytdxdata.protocol.volume import decode_volume
from pytdxdata.protocol.commands.ex_proto import GetExInstrumentBarsCmd
from pytdxdata.protocol.commands.kline import GetSecurityBarsCmd
from pytdxdata.protocol.commands.minute import GetHistoryMinuteTimeDataCmd
from pytdxdata.protocol.commands.transaction import MacTransactionCmd
# ---------- 变长有符号整数 ----------
@pytest.mark.parametrize("value", [0, 1, -1, 63, 64, -64, 127, -128, 1000, -100000, 2**20])
def test_price_roundtrip(value: int):
enc = put_price(value)
dec, pos = get_price(enc, 0)
assert dec == value
assert pos == len(enc)
def test_price_negative_flag():
# 首字节 bit6=符号
enc = put_price(-5)
assert enc[0] & 0x40
dec, _ = get_price(enc, 0)
assert dec == -5
# ---------- OHLC 差分还原 ----------
def test_decode_ohlc_diffs():
# 手工构造:open_diff=100, close_diff=-20, high_diff=10, low_diff=-30
body = bytearray()
for v in (100, -20, 10, -30):
body.extend(put_price(v))
od, cd, hd, ld, pos = decode_ohlc_diffs(body, 0)
assert (od, cd, hd, ld) == (100, -20, 10, -30)
assert pos == len(body)
# ---------- 自定义4字节浮点 ----------
def test_decode_volume_zero():
assert decode_volume(0) == 0.0
def test_decode_volume_known():
# 经验值:0x00000001 应为极小正数(非0)
assert decode_volume(1) > 0
# ---------- 响应帧头 ----------
def test_frame_header():
# magic=7654321, 未压缩: zipsize==unzipsize
buf = struct.pack("<IIIHH", 7654321, 0, 0, 100, 100)
h = parse_header(buf)
assert h.magic == 7654321
assert h.zipsize == 100
assert h.unzipsize == 100
def test_decompress_body_uncompressed():
h = FrameHeader(7654321, 0, 0, 5, 5)
assert decompress_body(h, b"hello") == b"hello"
def test_decompress_body_zlib():
import zlib as z
raw = z.compress(b"x" * 100)
h = FrameHeader(7654321, 0, 0, len(raw), 100)
assert decompress_body(h, raw) == b"x" * 100
# ---------- K线命令 ----------
def test_kline_build_request():
cmd = GetSecurityBarsCmd(1, "600000", 4, 0, 100) # DAY
req = cmd.build_request()
assert len(req) == 38
assert req[0:2] == struct.pack("<H", 0x010C) # 命令ID
# category 字节应等于 4 (DAY, 与 easy-tdx KlineCategory 一致)
assert req[20] == 4
def test_kline_parse_empty_response():
cmd = GetSecurityBarsCmd(1, "600000", 5, 0, 100)
with pytest.raises(Exception):
# 2字节 header 声称100条但body为空 → 解析失败
cmd.parse_response(b"\x64\x00")
# ---------- 分时命令(0x0130 历史分时) ----------
def test_history_minute_parse():
# 响应: num(2B) + 4B头(skip=6 从num后算起) + 每条3变长字段(price_diff, unknown, vol)
body = bytearray(struct.pack("<H", 3) + b"\x00" * 4)
for pd, unk, v in ((914, 0, 553), (-1, 0, 784), (2, 0, 4157)):
body.extend(put_price(pd))
body.extend(put_price(unk))
body.extend(put_price(v))
cmd = GetHistoryMinuteTimeDataCmd(1, "600000", 20260813)
bars = cmd.parse_response(bytes(body))
assert [round(b.price, 2) for b in bars] == [9.14, 9.13, 9.15]
assert [int(b.vol) for b in bars] == [553, 784, 4157]
assert bars[0].hour == 9 and bars[0].minute == 30
assert bars[2].hour == 9 and bars[2].minute == 32
# ---------- EX K线(0x23FF)日期解码 ----------
def _ex_bars_body(raw_dates: list[int]) -> bytes:
"""构造 0x23FF 响应体:20B头(count@18) + 每条32B(4B时间 + 28B行情)。"""
body = bytearray(20 + len(raw_dates) * 32)
struct.pack_into("<H", body, 18, len(raw_dates))
for i, raw in enumerate(raw_dates):
off = 20 + i * 32
struct.pack_into("<I", body, off, raw)
struct.pack_into("<ffffIIf", body, off + 4, 10.0, 10.5, 9.5, 10.2, 0, 1000, 0)
return bytes(body)
def test_ex_bars_minute_datetime():
# category=0(分钟级): 4字节=2B zipday + 2B tminutes
# 53785305 = tminutes 820(13:40) + zipday 45785(2026-07-29);旧解析误读为 5378-53-05
raw = struct.unpack("<I", struct.pack("<HH", 45785, 820))[0]
assert raw == 53785305
cmd = GetExInstrumentBarsCmd(31, "00700", 0, 0, 5)
bars = cmd.parse_response(_ex_bars_body([raw]))
assert len(bars) == 1
assert (bars[0]["year"], bars[0]["month"], bars[0]["day"]) == (2026, 7, 29)
assert (bars[0]["hour"], bars[0]["minute"]) == (13, 40)
assert bars[0]["close"] == pytest.approx(10.2)
def test_ex_bars_day_datetime():
# category=4(日线): 4字节 YYYYMMDD,hour=15
cmd = GetExInstrumentBarsCmd(31, "00700", 4, 0, 5)
bars = cmd.parse_response(_ex_bars_body([20260729]))
assert len(bars) == 1
assert (bars[0]["year"], bars[0]["month"], bars[0]["day"]) == (2026, 7, 29)
assert (bars[0]["hour"], bars[0]["minute"]) == (15, 0)
# ---------- MAC 逐笔命令 ----------
def test_mac_transaction_request():
cmd = MacTransactionCmd(0, "000001", 20260811, 0, 100)
req = cmd.build_request()
assert len(req) == 10 + 2 + 44 # 10头 + 2msg_id + body
assert req[0] == 0x1C # head_flag
def test_mac_transaction_parse():
# 构造响应: 39字节头(count@29=2) + 2条×18字节
body = bytearray(39 + 2 * 18)
struct.pack_into("<H", body, 29, 2)
for i, (ts, price, vol, tc, flag) in enumerate([
(34200, 11.25, 100, 3, 1), # 09:30:00
(34201, 11.26, 50, 1, 0),
]):
off = 39 + i * 18
struct.pack_into("<IfIIH", body, off, ts, price, vol, tc, flag)
cmd = MacTransactionCmd(0, "000001", 20260811, 0, 100)
recs = cmd.parse_response(bytes(body))
assert len(recs) == 2
assert recs[0].time == "09:30:00"
assert recs[0].price == 11.25
assert recs[0].vol == 100
assert recs[0].trade_count == 3
assert recs[0].bs_flag == 1
assert recs[1].trade_count == 1