perf(frames): DataFrame 转换提速近一倍(24万行 K 线 894ms → 462ms) (#13)

- 普通列改 list(map(attrgetter(name), rows)):取值在 C 层,不再每格建 Python 帧
- trading_date 按时间字段缓存(24万行常只有 1~2 个键):218ms → 33ms
- IntEnum 归一化从每格判断改为按列判断、需要时才整列转
- 补 3 项逐字段等价测试(多市场多时段 trading_date、分时 day_offset、全列比对)
- docs/dataframe.md 性能一节改为实测数据 + 何时该用 DataFrame 出口

Co-authored-by: openbot-coder <openbot-coder@users.noreply.github.com>
This commit is contained in:
openbotandopenbot-coder authored and GitHub committed 2026-10-29 18:33:07 +08:00
1 parent 04e0c57569
commit 8c25feca18
4 files changed
+224 -25

No files matched your search

+58
View File
@@ -17,6 +17,7 @@ from pytdxdata import TdxData
from pytdxdata import frames as _frames
from pytdxdata.frames import FrameBackendError, framed, get_backend, to_frame
from pytdxdata.models import (
ExMarket,
IndexBar,
IndexTradeStats,
IndicatorSet,
@@ -323,3 +324,60 @@ def test_datetime_dtype_is_datetime_like():
pl = pytest.importorskip("polars")
df = to_frame([bar()], backend="polars")
assert isinstance(df.schema["datetime"], pl.Datetime)
# ---------------------------------------------------------------- 交易日列缓存
@pytest.mark.parametrize("backend", BACKENDS)
def test_trading_date_multi_key_batch_matches_model(backend):
"""一批里混着 A股与美股、跨 0~23 点,逐行都要等于模型自己的 ``trading_date``。
``trading_date`` 是按「时间字段」缓存算出来的(240k 行常只有 1~2 个键),
这里专门造出「多键同批」的极端情形,防止缓存键漏掉参与计算的因素。
注意:美股的跨日处理在 ``SecurityBar.__post_init__`` 里就做完了(凌晨段
的年月日会被推进一天),所以 ``trading_date`` 对每个 bar 只是自然日,
真正会分叉的是 ``datetime`` —— 两者都要逐行核对。
"""
batch = []
for market in (int(Market.SH), int(ExMarket.US_STOCK)):
for hour in range(24):
batch.append(bar(market=market, hour=hour, minute=15))
df = to_frame(batch, backend=backend)
got_td = [cell(df, backend, "trading_date", i) for i in range(rows(df, backend))]
got_dt = [cell(df, backend, "datetime", i) for i in range(rows(df, backend))]
assert got_td == [b.trading_date for b in batch]
assert got_dt == [b.datetime for b in batch]
# 美股凌晨段的 datetime 确实落在下一天(否则这条测试等于没测)
us = [b for b in batch if b.market == int(ExMarket.US_STOCK)]
assert {b.datetime.date().isoformat() for b in us} == {"2026-10-27", "2026-10-28"}
@pytest.mark.parametrize("backend", BACKENDS)
def test_minute_trading_date_cache_matches_model(backend):
"""分时的 ``day_offset`` 也参与交易日计算,缓存键不能漏。"""
batch = [
MinuteBar(price=10.0, vol=1.0, hour=9, minute=30, date=20261027),
MinuteBar(price=10.0, vol=1.0, hour=21, minute=30, date=20261027,
market=int(ExMarket.US_STOCK)),
MinuteBar(price=10.0, vol=1.0, hour=3, minute=0, date=20261027,
market=int(ExMarket.US_STOCK), day_offset=1),
]
df = to_frame(batch, backend=backend)
got = [cell(df, backend, "trading_date", i) for i in range(rows(df, backend))]
assert got == [b.trading_date for b in batch]
@pytest.mark.parametrize("backend", BACKENDS)
def test_every_bar_field_equals_model(backend):
"""逐字段等价:DataFrame 的每个值都必须等于 dataclass 的同名字段。
这条是取列路径(``attrgetter`` 批量取 + 按列转 Enum)改动的兜底,
避免「快是快了,但某列悄悄串了值」。
"""
src = bar(minute=15, settle=3.5)
df = to_frame([src], backend=backend)
for name in cols(df, backend):
want = getattr(src, name)
assert cell(df, backend, name) == want, name