Files
pyquantdata/tests/test_etl_update.py
T
openbot d37c697bbe M1 最小闭环:CLI + FastAPI 网关 + SQL 闸 + 100% 覆盖率验收链
- CLI:init/serve/update/query/export/status(typer + rich),单写者锁三元判活
- 存储:DuckDB schema migrations(14 表 + 8 联邦视图,语句元组逐条执行)
- ETL:A股证券/日历/日K,整区替换幂等;日历全年补全(calendar 语义唯一)
- 网关:POST /v1/query(线程池 + interrupt 超时 504)、/v1/export、meta/stats/state、healthz/ready
- 安全:SQL 闸 sqlglot fail-closed(文件/网络表函数默认拒)、非 loopback 空 token 拒启、导出路径双判
- 验收:254 测试全绿、1265 语句 100% 覆盖、[CI] 真起服验收链、smoke_m1.py 交付自检
- 文档:设计文档 v0.7(圆桌 45 条采纳 + 3 条实施校准),README 重写
- 发布:release.yml Trusted Publishing(OIDC 免密钥),action refs 已核验
2026-09-24 09:06:47 +08:00

128 lines
5.0 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""update ETL 编排:FakeAdapter 确定性数据,幂等重放。"""
from __future__ import annotations
import asyncio
from datetime import date
import pytest
from pyquantdata.etl import run_update
from pyquantdata.sources import FakeAdapter
from pyquantdata.sources.base import Bar1dRow, CalendarRow, SecurityRow
def _bars():
out = []
for sym in ("sh600000", "sh600519"):
for d in range(1, 6):
out.append(
Bar1dRow(
symbol=sym, date=date(2026, 9, d),
open=10.0 + d, high=11.0 + d, low=9.0 + d, close=10.5 + d,
volume=1000.0 + d, amount=10500.0 + d,
)
)
return out
def _securities():
return [
SecurityRow(symbol="sh600000", name="浦发银行", category="stock", exchange="sh", lot_size=100),
SecurityRow(symbol="sh600519", name="贵州茅台", category="stock", exchange="sh", lot_size=100),
SecurityRow(symbol="sh000001", name="上证指数", category="index", exchange="sh"),
]
def _calendar():
return [
CalendarRow(trade_date=date(2026, 9, d), is_open=True, note="")
for d in (1, 2, 3, 7, 8, 9)
]
@pytest.fixture
def adapter():
return FakeAdapter(
securities=_securities(), bars=_bars(), calendar={2026: _calendar()}
)
def test_run_update_full_chain(catalog, adapter):
reports = asyncio.run(
run_update(catalog, "cn", adapter, datasets=["securities", "calendar", "bars_1d"], years=[2026])
)
assert [r.dataset for r in reports] == ["securities", "calendar", "bars_1d"]
assert reports[0].rows == 3
assert catalog.execute("SELECT count(*) FROM securities").fetchone()[0] == 3
# calendar 落库恒为整年逐日(2026 非闰年 = 365),is_open 只标 6 个交易日
assert catalog.execute("SELECT count(*) FROM calendar WHERE market='cn'").fetchone()[0] == 365
assert catalog.execute("SELECT count(*) FROM calendar WHERE is_open").fetchone()[0] == 6
# 日K:只有 securities 里的 stock/index/etf/cbond 进入回填清单
assert reports[2].rows == 10 # 2 stock × 5 天;index 在清单内但 FakeAdapter 无其 bar
assert reports[2].date_min == date(2026, 9, 1)
assert reports[2].date_max == date(2026, 9, 5)
# checkpoint 已写
cps = catalog.execute(
"SELECT dataset, status FROM etl_checkpoint ORDER BY dataset"
).fetchall()
assert [c[0] for c in cps] == ["bars_1d", "calendar", "securities"]
def test_bars_replay_idempotent(catalog, adapter):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["securities", "bars_1d"]))
n1 = catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0]
asyncio.run(run_update(catalog, "cn", adapter, datasets=["bars_1d"]))
n2 = catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0]
assert n1 == n2 == 10
def test_bars_with_explicit_symbols_and_range(catalog, adapter):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["securities"]))
report = asyncio.run(
run_update(
catalog, "cn", adapter, datasets=["bars_1d"],
symbols=["sh600000"], start=date(2026, 9, 2), end=date(2026, 9, 4),
)
)[0]
assert report.rows == 3
rows = catalog.execute(
"SELECT count(*), min(date), max(date) FROM bars_1d WHERE symbol='sh600000'"
).fetchone()
assert rows == (3, date(2026, 9, 2), date(2026, 9, 4))
def test_bars_empty_securities_raises(catalog, adapter):
with pytest.raises(RuntimeError, match="securities"):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["bars_1d"]))
def test_partial_overlap_replay_replaces(catalog, adapter):
"""重放窗口小于已有区间:重放区替换、区外保留(§7.3 整区替换)。"""
asyncio.run(run_update(catalog, "cn", adapter, datasets=["securities", "bars_1d"]))
# 现在只重放 9/1~9/2,且数据变化(close 改为 99)
slim = FakeAdapter(securities=_securities())
slim.bars = [
Bar1dRow(symbol="sh600000", date=date(2026, 9, d), open=1, high=2, low=0.5,
close=99, volume=1, amount=2)
for d in (1, 2)
]
asyncio.run(
run_update(catalog, "cn", slim, datasets=["bars_1d"], symbols=["sh600000"],
start=date(2026, 9, 1), end=date(2026, 9, 2))
)
n_99 = catalog.execute("SELECT count(*) FROM bars_1d WHERE close=99").fetchone()[0]
n_kept = catalog.execute(
"SELECT count(*) FROM bars_1d WHERE symbol='sh600000' AND date >= DATE '2026-09-03'"
).fetchone()[0]
assert n_99 == 2
assert n_kept == 3 # 旧区间未被动
def test_calendar_replay_idempotent(catalog, adapter):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["calendar"], years=[2026]))
n1 = catalog.execute("SELECT count(*) FROM calendar").fetchone()[0]
asyncio.run(run_update(catalog, "cn", adapter, datasets=["calendar"], years=[2026]))
n2 = catalog.execute("SELECT count(*) FROM calendar").fetchone()[0]
assert n1 == n2 == 365 # 整年逐日,重放不翻倍