M1 最小闭环:CLI + FastAPI 网关 + SQL 闸 + 100% 覆盖率验收链

- CLI:init/serve/update/query/export/status(typer + rich),单写者锁三元判活
- 存储:DuckDB schema migrations(14 表 + 8 联邦视图,语句元组逐条执行)
- ETL:A股证券/日历/日K,整区替换幂等;日历全年补全(calendar 语义唯一)
- 网关:POST /v1/query(线程池 + interrupt 超时 504)、/v1/export、meta/stats/state、healthz/ready
- 安全:SQL 闸 sqlglot fail-closed(文件/网络表函数默认拒)、非 loopback 空 token 拒启、导出路径双判
- 验收:254 测试全绿、1265 语句 100% 覆盖、[CI] 真起服验收链、smoke_m1.py 交付自检
- 文档:设计文档 v0.7(圆桌 45 条采纳 + 3 条实施校准),README 重写
- 发布:release.yml Trusted Publishing(OIDC 免密钥),action refs 已核验
This commit is contained in:
openbot committed 2026-09-24 09:06:47 +08:00
commit d37c697bbe
59 files changed
+7586

No files matched your search

+127
View File
@@ -0,0 +1,127 @@
"""update ETL 编排:FakeAdapter 确定性数据,幂等重放。"""
from __future__ import annotations
import asyncio
from datetime import date
import pytest
from pyquantdata.etl import run_update
from pyquantdata.sources import FakeAdapter
from pyquantdata.sources.base import Bar1dRow, CalendarRow, SecurityRow
def _bars():
out = []
for sym in ("sh600000", "sh600519"):
for d in range(1, 6):
out.append(
Bar1dRow(
symbol=sym, date=date(2026, 9, d),
open=10.0 + d, high=11.0 + d, low=9.0 + d, close=10.5 + d,
volume=1000.0 + d, amount=10500.0 + d,
)
)
return out
def _securities():
return [
SecurityRow(symbol="sh600000", name="浦发银行", category="stock", exchange="sh", lot_size=100),
SecurityRow(symbol="sh600519", name="贵州茅台", category="stock", exchange="sh", lot_size=100),
SecurityRow(symbol="sh000001", name="上证指数", category="index", exchange="sh"),
]
def _calendar():
return [
CalendarRow(trade_date=date(2026, 9, d), is_open=True, note="")
for d in (1, 2, 3, 7, 8, 9)
]
@pytest.fixture
def adapter():
return FakeAdapter(
securities=_securities(), bars=_bars(), calendar={2026: _calendar()}
)
def test_run_update_full_chain(catalog, adapter):
reports = asyncio.run(
run_update(catalog, "cn", adapter, datasets=["securities", "calendar", "bars_1d"], years=[2026])
)
assert [r.dataset for r in reports] == ["securities", "calendar", "bars_1d"]
assert reports[0].rows == 3
assert catalog.execute("SELECT count(*) FROM securities").fetchone()[0] == 3
# calendar 落库恒为整年逐日(2026 非闰年 = 365),is_open 只标 6 个交易日
assert catalog.execute("SELECT count(*) FROM calendar WHERE market='cn'").fetchone()[0] == 365
assert catalog.execute("SELECT count(*) FROM calendar WHERE is_open").fetchone()[0] == 6
# 日K:只有 securities 里的 stock/index/etf/cbond 进入回填清单
assert reports[2].rows == 10 # 2 stock × 5 天;index 在清单内但 FakeAdapter 无其 bar
assert reports[2].date_min == date(2026, 9, 1)
assert reports[2].date_max == date(2026, 9, 5)
# checkpoint 已写
cps = catalog.execute(
"SELECT dataset, status FROM etl_checkpoint ORDER BY dataset"
).fetchall()
assert [c[0] for c in cps] == ["bars_1d", "calendar", "securities"]
def test_bars_replay_idempotent(catalog, adapter):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["securities", "bars_1d"]))
n1 = catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0]
asyncio.run(run_update(catalog, "cn", adapter, datasets=["bars_1d"]))
n2 = catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0]
assert n1 == n2 == 10
def test_bars_with_explicit_symbols_and_range(catalog, adapter):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["securities"]))
report = asyncio.run(
run_update(
catalog, "cn", adapter, datasets=["bars_1d"],
symbols=["sh600000"], start=date(2026, 9, 2), end=date(2026, 9, 4),
)
)[0]
assert report.rows == 3
rows = catalog.execute(
"SELECT count(*), min(date), max(date) FROM bars_1d WHERE symbol='sh600000'"
).fetchone()
assert rows == (3, date(2026, 9, 2), date(2026, 9, 4))
def test_bars_empty_securities_raises(catalog, adapter):
with pytest.raises(RuntimeError, match="securities"):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["bars_1d"]))
def test_partial_overlap_replay_replaces(catalog, adapter):
"""重放窗口小于已有区间:重放区替换、区外保留(§7.3 整区替换)。"""
asyncio.run(run_update(catalog, "cn", adapter, datasets=["securities", "bars_1d"]))
# 现在只重放 9/1~9/2,且数据变化(close 改为 99)
slim = FakeAdapter(securities=_securities())
slim.bars = [
Bar1dRow(symbol="sh600000", date=date(2026, 9, d), open=1, high=2, low=0.5,
close=99, volume=1, amount=2)
for d in (1, 2)
]
asyncio.run(
run_update(catalog, "cn", slim, datasets=["bars_1d"], symbols=["sh600000"],
start=date(2026, 9, 1), end=date(2026, 9, 2))
)
n_99 = catalog.execute("SELECT count(*) FROM bars_1d WHERE close=99").fetchone()[0]
n_kept = catalog.execute(
"SELECT count(*) FROM bars_1d WHERE symbol='sh600000' AND date >= DATE '2026-09-03'"
).fetchone()[0]
assert n_99 == 2
assert n_kept == 3 # 旧区间未被动
def test_calendar_replay_idempotent(catalog, adapter):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["calendar"], years=[2026]))
n1 = catalog.execute("SELECT count(*) FROM calendar").fetchone()[0]
asyncio.run(run_update(catalog, "cn", adapter, datasets=["calendar"], years=[2026]))
n2 = catalog.execute("SELECT count(*) FROM calendar").fetchone()[0]
assert n1 == n2 == 365 # 整年逐日,重放不翻倍