M1 最小闭环:CLI + FastAPI 网关 + SQL 闸 + 100% 覆盖率验收链

- CLI:init/serve/update/query/export/status(typer + rich),单写者锁三元判活
- 存储:DuckDB schema migrations(14 表 + 8 联邦视图,语句元组逐条执行)
- ETL:A股证券/日历/日K,整区替换幂等;日历全年补全(calendar 语义唯一)
- 网关:POST /v1/query(线程池 + interrupt 超时 504)、/v1/export、meta/stats/state、healthz/ready
- 安全:SQL 闸 sqlglot fail-closed(文件/网络表函数默认拒)、非 loopback 空 token 拒启、导出路径双判
- 验收:254 测试全绿、1265 语句 100% 覆盖、[CI] 真起服验收链、smoke_m1.py 交付自检
- 文档:设计文档 v0.7(圆桌 45 条采纳 + 3 条实施校准),README 重写
- 发布:release.yml Trusted Publishing(OIDC 免密钥),action refs 已核验
This commit is contained in:
openbot committed 2026-09-24 09:06:47 +08:00
commit d37c697bbe
59 files changed
+7586

No files matched your search

+81
View File
@@ -0,0 +1,81 @@
"""共享 fixtures:迷你库、迷你 seed、app client。"""
from __future__ import annotations
from datetime import date
from pathlib import Path
import pyarrow as pa
import pyarrow.parquet as paparquet
import pytest
from fastapi.testclient import TestClient
from pyquantdata.config import default_config
from pyquantdata.paths import DbPathLayout
from pyquantdata.store import migrate, open_catalog
@pytest.fixture
def dbpath(tmp_path: Path) -> Path:
return tmp_path / "db"
@pytest.fixture
def layout(dbpath: Path) -> DbPathLayout:
lay = DbPathLayout(dbpath)
lay.ensure_dirs()
return lay
@pytest.fixture
def catalog(layout: DbPathLayout, dbpath: Path):
"""已 migrate 的写连接(含派生 cursor 配置)。"""
conn = open_catalog(layout.root)
migrate(conn)
yield conn
conn.close()
def make_seed_table() -> pa.Table:
dates = [date(2026, 9, d) for d in range(1, 11)]
n = len(dates)
return pa.table(
{
"symbol": ["sh600000"] * n,
"date": dates,
"open": [10.0 + i * 0.1 for i in range(n)],
"high": [10.5 + i * 0.1 for i in range(n)],
"low": [9.5 + i * 0.1 for i in range(n)],
"close": [10.2 + i * 0.1 for i in range(n)],
"volume": [1_000_000.0] * n,
"amount": [10_200_000.0 + i * 1e4 for i in range(n)],
}
)
@pytest.fixture
def mini_seed(tmp_path: Path) -> Path:
p = tmp_path / "mini_seed_1d.parquet"
paparquet.write_table(make_seed_table(), p)
return p
def make_client(dbpath: Path, cursor, *, config=None, schema_version: int = 1) -> TestClient:
from pyquantdata.serve.app import AppContext, create_app
ctx = AppContext(
dbpath=dbpath, config=config or default_config(), cursor=cursor,
schema_version=schema_version,
)
app = create_app(ctx)
return TestClient(app)
@pytest.fixture
def client(catalog, dbpath: Path) -> TestClient:
from pyquantdata.store.db import configure_query_conn
cursor = catalog.cursor()
configure_query_conn(cursor)
with make_client(dbpath, cursor) as c:
yield c
+96
View File
@@ -0,0 +1,96 @@
"""生成 [CI] 验收链所需的迷你 seed fixture(离线、确定性、可复现)。
设计 §14/§15:[CI] 层 = ``init --skip-history`` + 迷你 seed → serve → query → export。
CI 不联网、不拉真数据,因此把 fixture 生成逻辑固化成脚本 + 产物入库:
用法::
python tests/fixtures/build_ci_fixture.py # 写入 ci_seed_1d.parquet
python tests/fixtures/build_ci_fixture.py --check # 只校验现有产物是否一致
产物 schema 必须与 ``bars_1d`` 的 seed 读取列一致:
symbol/date/open/high/low/close/volume/amount(§7.3)。
"""
from __future__ import annotations
import argparse
import sys
from datetime import date
from pathlib import Path
import pyarrow as pa
import pyarrow.parquet as paparquet
FIXTURE = Path(__file__).with_name("ci_seed_1d.parquet")
# 10 个交易日 × 2 只股票(A 股规则 symbol,classify_cn 可判定 category)
_DATES = [date(2026, 9, d) for d in range(1, 11)]
_SYMBOLS = ("sh600000", "sz000001")
def build_table() -> pa.Table:
symbols: list[str] = []
dates: list[date] = []
open_: list[float] = []
high: list[float] = []
low: list[float] = []
close: list[float] = []
volume: list[float] = []
amount: list[float] = []
for si, sym in enumerate(_SYMBOLS):
base = 10.0 + si * 20.0 # 第二只股票价格区间错开,便于区分
for i, d in enumerate(_DATES):
symbols.append(sym)
dates.append(d)
open_.append(round(base + i * 0.1, 4))
high.append(round(base + 0.5 + i * 0.1, 4))
low.append(round(base - 0.5 + i * 0.1, 4))
close.append(round(base + 0.2 + i * 0.1, 4))
volume.append(1_000_000.0)
amount.append(round(10_200_000.0 + i * 1e4 + si * 1e6, 2))
return pa.table(
{
"symbol": symbols,
"date": dates,
"open": open_,
"high": high,
"low": low,
"close": close,
"volume": volume,
"amount": amount,
}
)
def write_fixture() -> Path:
paparquet.write_table(build_table(), FIXTURE)
return FIXTURE
def check_fixture() -> bool:
if not FIXTURE.exists():
print(f"[check] 缺失:{FIXTURE}")
return False
existing = paparquet.read_table(FIXTURE)
expected = build_table()
if existing.equals(expected):
print(f"[check] OK:{FIXTURE.name} 与生成逻辑一致({existing.num_rows} 行)")
return True
print(f"[check] 不一致:{FIXTURE.name} 需重新生成(产物 {existing.num_rows} 行)")
return False
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="生成/校验 [CI] 迷你 seed fixture")
parser.add_argument("--check", action="store_true", help="只校验,不写盘")
args = parser.parse_args(argv)
if args.check:
return 0 if check_fixture() else 1
p = write_fixture()
print(f"[write] {p}({build_table().num_rows} 行 × {len(_SYMBOLS)} symbols)")
return 0
if __name__ == "__main__":
sys.exit(main())
Binary file not shown.
+350
View File
@@ -0,0 +1,350 @@
"""FastAPI 全端点(TestClient):query/export/meta/stats/state/ready/token。"""
from __future__ import annotations
import base64
import threading
import time
import duckdb
import pyarrow as pa
import pytest
from pyquantdata.config import default_config
from pyquantdata.etl import import_seed
from pyquantdata.serve.app import AppContext, ResultTooLargeError, create_app
from pyquantdata.store.db import configure_query_conn
@pytest.fixture
def seeded(catalog, dbpath, mini_seed):
import_seed(catalog, mini_seed)
catalog.execute(
"INSERT INTO calendar VALUES ('cn', CURRENT_DATE, TRUE, 'test'), "
"('us', CURRENT_DATE, FALSE, 'weekend-test')"
)
cursor = catalog.cursor()
configure_query_conn(cursor)
ctx = AppContext(dbpath=dbpath, config=default_config(), cursor=cursor, schema_version=1)
return catalog, ctx
@pytest.fixture
def client(seeded):
from fastapi.testclient import TestClient
_catalog, ctx = seeded
with TestClient(create_app(ctx)) as c:
yield c
@pytest.fixture
def client_429(catalog, dbpath, mini_seed):
import_seed(catalog, mini_seed)
cfg = default_config()
cfg.sql.max_concurrent = 1
cfg.sql.acquire_timeout_s = 0.2
cursor = catalog.cursor()
configure_query_conn(cursor)
ctx = AppContext(dbpath=dbpath, config=cfg, cursor=cursor, schema_version=1)
from fastapi.testclient import TestClient
with TestClient(create_app(ctx)) as c:
yield c
# 合法表上的超慢查询:cn_stock_1d 自笛卡尔积(10^10 行级)→ 必被 timeout 中断。
# 不用 range() 表函数:SQL 闸按设计拒绝 FROM 位置的表函数(§8.3 fail-closed)。
SLOW_SQL = "SELECT count(*) FROM " + " CROSS JOIN ".join(
f"cn_stock_1d t{i}" for i in range(10)
)
def test_healthz(client):
assert client.get("/healthz").json() == {"status": "ok"}
def test_ready_ok(client):
r = client.get("/v1/ready")
assert r.status_code == 200
body = r.json()
assert body["ready"] is True
assert body["checks"]["schema_version"] == 1
assert body["checks"]["disk_free_pct"] is not None
def test_ready_503_when_not_migrated(catalog, dbpath):
from fastapi.testclient import TestClient
cursor = catalog.cursor()
ctx = AppContext(dbpath=dbpath, config=default_config(), cursor=cursor, schema_version=0)
with TestClient(create_app(ctx)) as c:
r = c.get("/v1/ready")
assert r.status_code == 503
assert r.json()["ready"] is False
def test_query_json_basics(client):
r = client.post("/v1/query", json={"sql": "SELECT date, close FROM cn_stock_1d ORDER BY date"})
assert r.status_code == 200
body = r.json()
assert body["columns"] == ["date", "close"]
assert body["row_count"] == 10
assert body["truncated"] is False
assert body["elapsed_ms"] >= 0
assert body["rows"][0][0] == "2026-09-01"
assert isinstance(body["rows"][0][1], float)
def test_query_arrow_stream_b64(client):
r = client.post("/v1/query", json={
"sql": "SELECT symbol, date, close FROM cn_stock_1d ORDER BY date", "fmt": "arrow",
})
assert r.status_code == 200
b64 = r.json()["b64_arrow"]
tbl = pa.ipc.open_stream(pa.BufferReader(base64.b64decode(b64))).read_all()
assert tbl.schema.names == ["symbol", "date", "close"]
assert tbl.num_rows == 10
assert tbl.column("symbol")[0].as_py() == "sh600000"
def test_query_fmt_invalid(client):
r = client.post("/v1/query", json={"sql": "SELECT 1", "fmt": "feather"})
assert r.status_code == 422
def test_query_params_placeholder(client):
r = client.post("/v1/query", json={
"sql": "SELECT close FROM cn_stock_1d WHERE symbol = ? AND date = ?",
"params": ["sh600000", "2026-09-03"],
})
assert r.status_code == 200
assert r.json()["row_count"] == 1
def test_query_max_rows_truncates(client):
r = client.post("/v1/query", json={"sql": "SELECT * FROM cn_stock_1d", "max_rows": 3})
assert r.status_code == 200
body = r.json()
assert body["row_count"] == 3
assert body["truncated"] is True
def test_query_max_rows_clamped_to_config(client):
r = client.post("/v1/query", json={"sql": "SELECT * FROM cn_stock_1d", "max_rows": 10**9})
assert r.status_code == 200 # clamp 到 config.max_rows,不报错
def test_query_rejects_forbidden_sql(client):
r = client.post("/v1/query", json={"sql": "SELECT read_csv('/etc/passwd')"})
assert r.status_code == 400
assert "read_csv" in r.json()["detail"]
def test_query_execution_error_400(client):
r = client.post("/v1/query", json={"sql": "SELECT * FROM table_not_here"})
assert r.status_code == 400
assert "执行失败" in r.json()["detail"]
def test_query_timeout_interrupts(client):
started = time.monotonic()
r = client.post("/v1/query", json={"sql": SLOW_SQL, "timeout_ms": 200})
elapsed = time.monotonic() - started
assert r.status_code == 504
assert elapsed < 5 # interrupt 真中断,不等查询跑完
# 连接复用:interrupt 后仍可查询
r2 = client.post("/v1/query", json={"sql": "SELECT 1"})
assert r2.status_code == 200
def test_query_interrupt_exception_maps_504(client, monkeypatch):
import pyquantdata.serve.app as app_mod
def boom(*a, **k):
raise duckdb.InterruptException("INTERRUPT Error: Interrupted!")
monkeypatch.setattr(app_mod, "execute_query_blocking", boom)
r = client.post("/v1/query", json={"sql": "SELECT 1"})
assert r.status_code == 504
def test_query_result_too_large_maps_413(client, monkeypatch):
import pyquantdata.serve.app as app_mod
def too_big(*a, **k):
raise ResultTooLargeError("结果过大:请走 export")
monkeypatch.setattr(app_mod, "execute_query_blocking", too_big)
r = client.post("/v1/query", json={"sql": "SELECT 1"})
assert r.status_code == 413
def test_query_429_when_slot_busy(client_429):
results: dict[str, int] = {}
def slow():
r = client_429.post("/v1/query", json={"sql": SLOW_SQL, "timeout_ms": 4000})
results["slow"] = r.status_code
t = threading.Thread(target=slow)
t.start()
time.sleep(0.5) # slow 已占用唯一信号量槽
r2 = client_429.post("/v1/query", json={"sql": "SELECT 1"})
t.join(timeout=20)
assert r2.status_code == 429
assert "并发" in r2.json()["detail"]
assert results["slow"] in (200, 504) # 大查询最终超时中断
def test_export_csv_roundtrip(client, dbpath):
r = client.post("/v1/export", json={
"sql": "SELECT symbol, date, close FROM cn_stock_1d ORDER BY date",
"fmt": "csv", "filename": "daily.csv",
})
assert r.status_code == 200
assert r.headers["content-type"].startswith("text/csv")
lines = r.content.decode("utf-8").strip().splitlines()
# pyarrow 默认 RFC4180 加引号;解析后断言列名与行数
assert [c.strip('"') for c in lines[0].split(",")] == ["symbol", "date", "close"]
assert len(lines) == 11 # header + 10
# exports/{job_id}/ 落盘 + job 记录可查
jobs = client.post("/v1/query", json={"sql": "SELECT job_id FROM export_jobs"}).json()
job_id = jobs["rows"][0][0]
job = client.get(f"/v1/export/{job_id}").json()
assert job["state"] == "done"
assert job["rows"] == 10
assert job["path"].startswith("exports")
assert (dbpath / job["path"]).exists()
def test_export_parquet_roundtrip(client, dbpath):
import io
import pyarrow.parquet as paparquet
r = client.post("/v1/export", json={"sql": "SELECT * FROM cn_stock_1d", "fmt": "parquet"})
assert r.status_code == 200
tbl = paparquet.read_table(io.BytesIO(r.content))
assert tbl.num_rows == 10
def test_export_pandas_rejected_with_hint(client):
r = client.post("/v1/export", json={"sql": "SELECT 1", "fmt": "pandas"})
assert r.status_code == 422
assert "/v1/query" in r.json()["detail"]
def test_export_path_escape_rejected(client):
r = client.post("/v1/export", json={
"sql": "SELECT 1", "fmt": "csv", "filename": "../evil.csv",
})
assert r.status_code == 400
assert "非法" in r.json()["detail"]
def test_export_job_404(client):
assert client.get("/v1/export/nope").status_code == 404
def test_meta_views_and_stats(client):
meta = client.get("/v1/meta").json()
assert meta["schema_version"] == 1
names = {v["name"] for v in meta["views"]}
assert "cn_stock_1d" in names
v = next(v for v in meta["views"] if v["name"] == "cn_stock_1d")
assert v["row_count"] == 10
assert v["date_min"] == "2026-09-01"
assert v["sample_symbols"] == ["sh600000"]
stats = client.get("/v1/stats").json()
assert stats["checkpoints"] == []
assert stats["disk"]["bytes"] > 0
assert stats["http"]["ws_conns"] == 0
def test_meta_handles_special_views(catalog, seeded):
_cat, ctx = seeded
from fastapi.testclient import TestClient
catalog.execute("CREATE OR REPLACE VIEW meta_special AS SELECT 42 AS v")
with TestClient(create_app(ctx)) as c:
meta = c.get("/v1/meta").json()
special = next(v for v in meta["views"] if v["name"] == "meta_special")
assert special["columns"] == ["v"]
assert "date_min" not in special
assert "sample_symbols" not in special
def test_state_endpoint(client):
r = client.get("/v1/state/cn")
assert r.status_code == 200
body = r.json()
assert body["market"] == "cn"
assert body["state"] in {"CLOSED", "PRE_OPEN_AUCTION", "OPEN", "LUNCH_BREAK"}
assert body["session_tz"] == "Asia/Shanghai"
assert body["ts"].endswith("Z") # ISO8601 UTC 带 Z(§8.2)
def test_state_no_calendar_data_is_closed(client):
body = client.get("/v1/state/hk").json()
assert body["state"] == "CLOSED"
assert body["reason"] == "no_calendar_data"
def test_state_unknown_market_404(client):
assert client.get("/v1/state/uk").status_code == 404
def test_ready_disk_free_branch_none(catalog, tmp_path):
"""dbpath 不存在 → free_pct None 分支(只读探针仍 ready)。"""
from fastapi.testclient import TestClient
cursor = catalog.cursor()
cursor.execute("SELECT 1")
ctx = AppContext(
dbpath=tmp_path / "missing", config=default_config(),
cursor=cursor, schema_version=1,
)
with TestClient(create_app(ctx)) as c:
assert c.get("/v1/ready").status_code == 200
def test_stats_disk_oserror_branch(catalog, dbpath, monkeypatch):
from fastapi.testclient import TestClient
cursor = catalog.cursor()
cursor.execute("SELECT 1")
ctx = AppContext(dbpath=dbpath, config=default_config(), cursor=cursor, schema_version=1)
import pyquantdata.serve.app as app_mod
def boom(_):
raise OSError("disk gone")
monkeypatch.setattr(app_mod.psutil, "disk_usage", boom)
with TestClient(create_app(ctx)) as c:
body = c.get("/v1/stats").json()
assert body["disk"]["free_pct"] is None
def test_token_auth(catalog, dbpath, mini_seed):
import_seed(catalog, mini_seed)
cfg = default_config()
cfg.http.token = "abcd1234"
cursor = catalog.cursor()
configure_query_conn(cursor)
from fastapi.testclient import TestClient
with TestClient(create_app(AppContext(dbpath=dbpath, config=cfg, cursor=cursor, schema_version=1))) as c:
assert c.post("/v1/query", json={"sql": "SELECT 1"}).status_code == 401
assert c.post(
"/v1/query", json={"sql": "SELECT 1"},
headers={"Authorization": "Bearer wrong"},
).status_code == 401
ok = c.post(
"/v1/query", json={"sql": "SELECT 1"},
headers={"Authorization": "Bearer abcd1234"},
)
assert ok.status_code == 200
# 探针不设防
assert c.get("/healthz").status_code == 200
+53
View File
@@ -0,0 +1,53 @@
"""鉴权与脱敏(§8.4)。"""
from __future__ import annotations
import pytest
from pyquantdata.serve.auth import (
SecurityError,
assert_security,
is_loopback,
mask_query_string,
mask_token,
token_ok,
)
def test_mask_token():
assert mask_token("") == ""
assert mask_token("ab") == "ab***"
assert mask_token("abcd1234") == "abcd***"
def test_mask_query_string():
assert mask_query_string("token=abcd1234&topics=state") == "token=abcd***&topics=state"
assert mask_query_string("/v1/stream?topics=state&token=zzzz9999") == (
"/v1/stream?topics=state&token=zzzz***"
)
assert mask_query_string("no-token-here") == "no-token-here"
assert mask_query_string("token=ab") == "token=ab" # 不足 4 位不匹配模板(保持原样)
def test_is_loopback():
assert is_loopback("127.0.0.1")
assert is_loopback("::1")
assert is_loopback("localhost")
assert not is_loopback("0.0.0.0")
assert not is_loopback("192.168.1.10")
def test_assert_security_matrix():
assert_security("127.0.0.1", "", insecure=False) # loopback 不需要 token
assert_security("0.0.0.0", "tok", insecure=False) # 有 token
assert_security("0.0.0.0", "", insecure=True) # 显式裸奔
with pytest.raises(SecurityError, match="拒绝启动"):
assert_security("0.0.0.0", "", insecure=False)
def test_token_ok():
assert token_ok(None, "") is True # 空 token = 不校验
assert token_ok("any", "") is True
assert token_ok("right", "right") is True
assert token_ok("wrong", "right") is False
assert token_ok(None, "right") is False
+33
View File
@@ -0,0 +1,33 @@
"""etl_checkpoint 语义。"""
from __future__ import annotations
from pyquantdata.etl import failed_checkpoints, list_checkpoints, upsert_checkpoint
def test_upsert_and_list(catalog):
upsert_checkpoint(catalog, "bars_1d", "cn", "daily", "2026-09-22", 100)
upsert_checkpoint(catalog, "securities", "cn", "all", "", 5123)
cps = list_checkpoints(catalog)
assert [(c["dataset"], c["watermark"], c["rows"]) for c in cps] == [
("bars_1d", "2026-09-22", 100),
("securities", "", 5123),
]
assert all(c["status"] == "ok" for c in cps)
assert all(c["updated_at"] for c in cps)
def test_upsert_replaces(catalog):
upsert_checkpoint(catalog, "bars_1d", "cn", "daily", "2026-09-21", 50)
upsert_checkpoint(catalog, "bars_1d", "cn", "daily", "2026-09-22", 60)
cps = list_checkpoints(catalog)
assert len(cps) == 1
assert cps[0]["watermark"] == "2026-09-22"
assert cps[0]["rows"] == 60
def test_failed_filter(catalog):
upsert_checkpoint(catalog, "bars_1d", "cn", "daily", "x", 0, status="failed")
upsert_checkpoint(catalog, "calendar", "cn", "2026", "2026-12-31", 244)
failed = failed_checkpoints(catalog)
assert [c["dataset"] for c in failed] == ["bars_1d"]
+83
View File
@@ -0,0 +1,83 @@
"""[CI] 验收链(设计 §14/§15):init --skip-history + 迷你 seed → serve → POST /v1/query 查日K → export csv。
一条链必绿,全部离线。
"""
from __future__ import annotations
import base64
import pyarrow as pa
from fastapi.testclient import TestClient
from typer.testing import CliRunner
from pyquantdata.cli import app
from pyquantdata.config import default_config
from pyquantdata.serve.app import AppContext, create_app
from pyquantdata.store.db import configure_query_conn
runner = CliRunner()
def test_m1_ci_acceptance_chain(catalog, dbpath, mini_seed, tmp_path):
# 1. init --skip-history + seed(CLI 入口,建目录/config/schema/seed)
result = runner.invoke(app, [
"init", "-d", str(dbpath), "--skip-history", "--seed", str(mini_seed),
])
assert result.exit_code == 0, result.output
# 2. serve 语境(TestClient 同事件循环;真网 uvicorn 按 §14 白名单每周冒烟)
cursor = catalog.cursor()
configure_query_conn(cursor)
ctx = AppContext(dbpath=dbpath, config=default_config(), cursor=cursor, schema_version=1)
with TestClient(create_app(ctx)) as client:
# 3. POST /v1/query 查到日K(参数化 + JSON)
r = client.post("/v1/query", json={
"sql": "SELECT date, close FROM cn_stock_1d WHERE symbol = ? ORDER BY date",
"params": ["sh600000"],
})
assert r.status_code == 200
body = r.json()
assert body["row_count"] == 10
assert body["rows"][0][1] == 10.2 # 9/1 close
# 3b. fmt=arrow → Arrow IPC streaming → DataFrame 生态解码路径(附录 A)
r_arrow = client.post("/v1/query", json={
"sql": "SELECT date, close FROM cn_stock_1d ORDER BY date", "fmt": "arrow",
})
tbl = pa.ipc.open_stream(
pa.BufferReader(base64.b64decode(r_arrow.json()["b64_arrow"]))
).read_all()
df = tbl.to_pandas()
assert len(df) == 10 and float(df["close"].iloc[-1]) == 11.1
# 4. export csv(文件流回传 + exports/{job_id}/ 落盘 + job 记录)
r_exp = client.post("/v1/export", json={
"sql": "SELECT symbol, date, open, close FROM cn_stock_1d ORDER BY date",
"fmt": "csv", "filename": "mini_daily.csv",
})
assert r_exp.status_code == 200
lines = r_exp.content.decode("utf-8").strip().splitlines()
header = [c.strip('"') for c in lines[0].split(",")]
assert header == ["symbol", "date", "open", "close"] and len(lines) == 11
job_row = client.post(
"/v1/query", json={"sql": "SELECT job_id, rows, state FROM export_jobs"}
).json()
assert job_row["rows"][0][1] == 10 and job_row["rows"][0][2] == "done"
# 6. SQL 闸在链路内生效
blocked = client.post("/v1/query", json={"sql": "SELECT read_parquet('x')"})
assert blocked.status_code == 400
# 7. CLI 本地直查(只读连接,serve 之外也可用)
# DuckDB 同进程不允许同一库文件「写连接 + 只读连接」并存(配置不同会直接报错)——
# 先释放本测试持有的写连接,再走 CLI 只读路径。
cursor.close()
catalog.close()
q = runner.invoke(app, [
"query", "-d", str(dbpath), "--fmt", "json",
"SELECT count(*) AS n FROM cn_stock_1d",
])
assert q.exit_code == 0, q.output
assert '"row_count": 1' in q.output
+47
View File
@@ -0,0 +1,47 @@
"""A股分类纯函数。"""
from __future__ import annotations
import pytest
from pyquantdata.sources.classify import classify_cn, exchange_of
@pytest.mark.parametrize(
"symbol,category",
[
("sh600000", "stock"),
("sh600519", "stock"),
("sh688001", "stock"), # 科创板
("sz000001", "stock"),
("sz300750", "stock"), # 创业板
("bj430047", "stock"),
("bj833171", "stock"),
("sh000001", "index"), # 上证指数
("sz399001", "index"),
("sh510300", "etf"),
("sz159915", "etf"),
("sh513100", "etf"),
("sh508056", "reits"), # 特例优先于 etf 前缀 sh50
("sh510050", "etf"),
("sh110059", "cbond"),
("sz123456", "cbond"),
("sz127045", "cbond"),
("SH600000", "stock"), # 大小写归一
],
)
def test_classify_cn(symbol, category):
assert classify_cn(symbol) == category
@pytest.mark.parametrize("symbol", ["", "xx123", "sh999999", "hk00700"])
def test_classify_cn_unknown_is_other(symbol):
assert classify_cn(symbol) == "other"
@pytest.mark.parametrize(
"symbol,ex",
[("sh600000", "sh"), ("sz000001", "sz"), ("bj430047", "bj"), ("AAPL", "")],
)
def test_exchange_of(symbol, ex):
assert exchange_of(symbol) == ex
+222
View File
@@ -0,0 +1,222 @@
"""CLI:typer CliRunner 全命令(update 用 FakeAdapter 替身,不打真网)。"""
from __future__ import annotations
import json
from datetime import date
import pytest
from typer.testing import CliRunner
from pyquantdata.cli import app
from pyquantdata.paths import DbPathLayout
runner = CliRunner()
@pytest.fixture
def initialized(dbpath, mini_seed):
result = runner.invoke(app, ["init", "-d", str(dbpath), "--skip-history", "--seed", str(mini_seed)])
assert result.exit_code == 0, result.output
return dbpath
def test_init_creates_layout_and_config(dbpath, mini_seed):
result = runner.invoke(app, ["init", "-d", str(dbpath), "--skip-history", "--seed", str(mini_seed)])
assert result.exit_code == 0, result.output
assert "init 完成" in result.output
assert "[1/4]" in result.output and "[4/4]" in result.output # 分阶段进度
lay = DbPathLayout(dbpath)
assert lay.config_file.exists() and lay.catalog.exists()
assert "bars_1d rows=10" in result.output # seed 已导入
assert "seed: 10 行" in result.output
def test_init_idempotent_keeps_config(dbpath):
assert runner.invoke(app, ["init", "-d", str(dbpath)]).exit_code == 0
cfg = DbPathLayout(dbpath).config_file.read_text(encoding="utf-8")
cfg_obj = json.loads(cfg)
cfg_obj["http"]["port"] = 9999
DbPathLayout(dbpath).config_file.write_text(json.dumps(cfg_obj), encoding="utf-8")
result = runner.invoke(app, ["init", "-d", str(dbpath)])
assert result.exit_code == 0
assert "已存在,保留" in result.output
assert json.loads(DbPathLayout(dbpath).config_file.read_text(encoding="utf-8"))["http"]["port"] == 9999
def test_init_estimate_does_not_write(tmp_path):
target = tmp_path / "never-created"
result = runner.invoke(app, ["init", "-d", str(target), "--estimate"])
assert result.exit_code == 0
assert not target.exists() # 只算不写
assert "budget_gb" in result.output
def test_init_seed_failure_exit_1(dbpath, tmp_path):
result = runner.invoke(
app, ["init", "-d", str(dbpath), "--seed", str(tmp_path / "nope.parquet")]
)
assert result.exit_code == 1
assert "seed 导入失败" in result.output
def test_query_table_format(initialized):
result = runner.invoke(app, [
"query", "-d", str(initialized),
"SELECT date, close FROM cn_stock_1d ORDER BY date LIMIT 3",
])
assert result.exit_code == 0, result.output
assert "sh600000" not in result.output # 未选 symbol 列
assert "3 rows" in result.output
def test_query_json_format(initialized):
result = runner.invoke(app, [
"query", "-d", str(initialized), "--fmt", "json",
"SELECT count(*) AS n FROM cn_stock_1d",
])
assert result.exit_code == 0
payload = json.loads(result.output.strip()) # 纯 stdout JSON(无 rich 前缀)
assert payload["row_count"] == 1
assert payload["rows"][0][0] == 10
def test_query_csv_to_file(initialized, tmp_path):
out = tmp_path / "q.csv"
result = runner.invoke(app, [
"query", "-d", str(initialized), "--fmt", "csv", "-o", str(out),
"SELECT symbol, close FROM cn_stock_1d",
])
assert result.exit_code == 0
assert "已写出" in result.output
assert out.exists()
def test_query_csv_without_out_exit_1(initialized):
result = runner.invoke(app, [
"query", "-d", str(initialized), "--fmt", "csv", "SELECT 1",
])
assert result.exit_code == 1
assert "-o" in result.output
def test_query_bad_fmt_exit_1(initialized):
result = runner.invoke(app, [
"query", "-d", str(initialized), "--fmt", "xlsx", "SELECT 1",
])
assert result.exit_code == 1
def test_query_sql_rejected_exit_1(initialized):
result = runner.invoke(app, [
"query", "-d", str(initialized), "SELECT read_csv('/etc/passwd')",
])
assert result.exit_code == 1
assert "SQL 被拒绝" in result.output
def test_query_uninitialized_db_exit_1(tmp_path):
result = runner.invoke(app, ["query", "-d", str(tmp_path / "nope"), "SELECT 1"])
assert result.exit_code == 1
assert "未初始化" in result.output
def test_export_csv(initialized, tmp_path):
out = tmp_path / "exp.csv"
result = runner.invoke(app, [
"export", "-d", str(initialized), "--fmt", "csv", "-o", str(out),
"--sql", "SELECT * FROM cn_stock_1d",
])
assert result.exit_code == 0, result.output
assert out.exists()
assert "导出完成" in result.output
def test_export_parquet(initialized, tmp_path):
out = tmp_path / "exp.parquet"
result = runner.invoke(app, [
"export", "-d", str(initialized), "--fmt", "parquet", "-o", str(out),
"--sql", "SELECT * FROM cn_stock_1d",
])
assert result.exit_code == 0
import pyarrow.parquet as paparquet
assert paparquet.read_table(out).num_rows == 10
def test_export_pandas_hint_exit_1(initialized, tmp_path):
result = runner.invoke(app, [
"export", "-d", str(initialized), "--fmt", "pandas", "-o", str(tmp_path / "x"),
"--sql", "SELECT 1",
])
assert result.exit_code == 1
assert "query --fmt arrow" in result.output
def test_export_sql_rejected_exit_1(initialized, tmp_path):
result = runner.invoke(app, [
"export", "-d", str(initialized), "--fmt", "csv", "-o", str(tmp_path / "x.csv"),
"--sql", "ATTACH 'x'",
])
assert result.exit_code == 1
def test_status_json(initialized):
result = runner.invoke(app, ["status", "-d", str(initialized), "--json"])
assert result.exit_code == 0
# rich.print_json 输出在 result.output;去掉可能的空行
start = result.output.index("{")
payload = json.loads(result.output[start:])
assert payload["schema_version"] == 1
assert payload["bars_1d_rows"] == 10
assert payload["lock"]["held"] is False
assert payload["disk"] is not None
def test_status_plain_same_payload(initialized):
result = runner.invoke(app, ["status", "-d", str(initialized)])
assert result.exit_code == 0
assert "schema_version" in result.output
def test_quantdata_home_env(dbpath, mini_seed, monkeypatch):
monkeypatch.setenv("QUANTDATA_HOME", str(dbpath))
result = runner.invoke(app, ["init", "--skip-history", "--seed", str(mini_seed)])
assert result.exit_code == 0, result.output
assert DbPathLayout(dbpath).config_file.exists()
def test_update_with_fake_adapter(initialized, monkeypatch):
"""update 全链(--dataset securities/calendar/bars_1d)用替身适配器。"""
import pyquantdata.cli as cli_mod
from pyquantdata.sources import FakeAdapter
from pyquantdata.sources.base import Bar1dRow, CalendarRow, SecurityRow
adapter = FakeAdapter(
securities=[SecurityRow(symbol="sh600000", name="浦发银行", category="stock", exchange="sh")],
bars=[Bar1dRow(symbol="sh600000", date=date(2026, 9, 1), open=10, high=11, low=9, close=10.5, volume=1, amount=2)],
calendar={2026: [CalendarRow(trade_date=date(2026, 9, 1), is_open=True)]},
)
monkeypatch.setattr("pyquantdata.sources.TdxAdapter", lambda: adapter)
result = runner.invoke(app, [
"update", "-d", str(initialized), "--dataset", "all",
"--start", "2026-09-01", "--end", "2026-09-02",
])
assert result.exit_code == 0, result.output
assert "securities" in result.output and "update 完成" in result.output
assert ("open",) in adapter.calls and ("close",) in adapter.calls
# calendar 年份缺省 = 当前年
assert any(call[0] == "calendar" for call in adapter.calls)
def test_update_partial_dataset(initialized, monkeypatch):
from pyquantdata.sources import FakeAdapter
from pyquantdata.sources.base import SecurityRow
adapter = FakeAdapter(
securities=[SecurityRow(symbol="sh600000", name="x", category="stock", exchange="sh")]
)
monkeypatch.setattr("pyquantdata.sources.TdxAdapter", lambda: adapter)
result = runner.invoke(app, ["update", "-d", str(initialized), "--dataset", "securities"])
assert result.exit_code == 0
assert ("securities", "cn") in adapter.calls
+93
View File
@@ -0,0 +1,93 @@
"""config.py:默认生成、roundtrip、extra=forbid、${ENV} 展开。"""
from __future__ import annotations
from datetime import datetime, timezone
import pytest
from pydantic import ValidationError
from pyquantdata.config import (
AppConfig,
default_config,
expand_env,
load_config,
write_config,
)
def test_default_config_three_markets():
cfg = default_config()
assert set(cfg.markets) == {"cn", "us", "hk"}
assert cfg.markets["cn"].history_from == "2004-01-01"
assert cfg.markets["us"].history_from == "2020-01-01"
assert cfg.markets["hk"].history_from_bars_1d is None # 跟随 history_from
assert cfg.http.port == 8765
assert cfg.schedule.timezone == "Asia/Shanghai"
assert cfg.sql.max_concurrent == 4
assert cfg.export.formats == ["csv", "parquet", "qlib", "backtrader"]
def test_default_config_subset_markets():
cfg = default_config(["cn"])
assert set(cfg.markets) == {"cn"}
def test_roundtrip(tmp_path):
cfg = default_config()
path = tmp_path / "config.json"
write_config(cfg, path)
loaded = load_config(path)
assert loaded == cfg
text = path.read_text(encoding="utf-8")
assert '"version": 1' in text
def test_extra_field_forbidden(tmp_path):
cfg = default_config()
import json
raw = json.loads(cfg.model_dump_json(by_alias=True))
raw["hhtp"] = {"port": 1} # 拼错字段必须报错而不是静默失效
bad = tmp_path / "bad.json"
bad.write_text(json.dumps(raw), encoding="utf-8")
with pytest.raises(ValidationError):
load_config(bad)
def test_expand_env_token(monkeypatch):
monkeypatch.setenv("QD_TOKEN", "s3cret")
assert expand_env("${QD_TOKEN}") == "s3cret"
def test_expand_env_missing_var():
assert expand_env("${QD_NOT_SET_XYZ}") == ""
def test_expand_env_plain_string():
assert expand_env("no-var-here") == "no-var-here"
def test_load_applies_env_expansion(tmp_path, monkeypatch):
monkeypatch.setenv("QD_TOKEN", "abcd1234")
cfg = default_config()
cfg.http.token = "${QD_TOKEN}"
path = tmp_path / "config.json"
write_config(cfg, path)
loaded = load_config(path)
assert loaded.http.token == "abcd1234"
def test_now_override_is_used():
now = datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
cfg = default_config(now=now)
assert cfg.dbpath_created.startswith("2026-01-02T03:04:05")
def test_app_config_forbids_unknown_section():
import json
raw = json.loads(default_config().model_dump_json(by_alias=True))
raw["mystery"] = 1
with pytest.raises(ValidationError):
AppConfig.model_validate(raw)
+508
View File
@@ -0,0 +1,508 @@
"""补齐分支覆盖:异常回滚路径、边界分支、平台尽力而为分支。
只补「真实可测」的分支;真平台/真网络相关行按 §14 白名单处理(在 pyproject 里)。
"""
from __future__ import annotations
import asyncio
from datetime import date, datetime
import duckdb
import pyarrow as pa
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from pyquantdata.etl import import_seed
from pyquantdata.etl.checkpoint import upsert_checkpoint
from pyquantdata.etl.seed import _as_date
from pyquantdata.etl.update import (
run_update_sync,
update_bars_1d,
update_calendar,
update_securities,
)
from pyquantdata.export.files import run_query_arrow
from pyquantdata.serve.app import AppContext, create_app, table_to_rows, utc_now_iso
from pyquantdata.serve.auth import SecurityError
from pyquantdata.sources import FakeAdapter
from pyquantdata.sources.base import Bar1dRow, CalendarRow, SecurityRow
from pyquantdata.store import migrate, open_catalog
# ---------------------------------------------------------------- etl.seed
def test_as_date_accepts_three_forms():
assert _as_date(datetime(2026, 9, 1, 15, 30)) == date(2026, 9, 1) # datetime
assert _as_date(date(2026, 9, 2)) == date(2026, 9, 2) # date
assert _as_date("2026-09-03") == date(2026, 9, 3) # str
assert _as_date("2026-09-04T00:00:00") == date(2026, 9, 4) # ISO 前缀截断
def test_seed_rollback_on_write_failure(catalog, mini_seed, monkeypatch):
"""写入期异常 → 事务回滚,库内不留半截数据。"""
import pyquantdata.etl.seed as seed_mod
real_executemany = duckdb.DuckDBPyConnection.executemany
def boom(self, *a, **k):
raise duckdb.Error("injected insert failure")
monkeypatch.setattr(duckdb.DuckDBPyConnection, "executemany", boom)
with pytest.raises(duckdb.Error, match="injected"):
import_seed(catalog, mini_seed)
monkeypatch.setattr(duckdb.DuckDBPyConnection, "executemany", real_executemany)
# 回滚后表内无数据
assert catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0] == 0
# ---------------------------------------------------------------- etl.update
def test_update_securities_rollback_on_failure(catalog, monkeypatch):
adapter = FakeAdapter(
securities=[SecurityRow(symbol="sh600000", name="A", category="stock", exchange="sh")]
)
def boom(*a, **k):
raise duckdb.Error("injected securities failure")
monkeypatch.setattr(duckdb.DuckDBPyConnection, "executemany", boom)
with pytest.raises(duckdb.Error, match="injected"):
asyncio.run(update_securities(catalog, "cn", adapter))
assert catalog.execute("SELECT count(*) FROM securities").fetchone()[0] == 0
def test_update_calendar_rollback_on_failure(catalog, monkeypatch):
adapter = FakeAdapter(calendar={2026: [CalendarRow(trade_date=date(2026, 9, 1), is_open=True)]})
def boom(*a, **k):
raise duckdb.Error("injected calendar failure")
monkeypatch.setattr(duckdb.DuckDBPyConnection, "executemany", boom)
with pytest.raises(duckdb.Error, match="injected"):
asyncio.run(update_calendar(catalog, "cn", adapter, [2026]))
assert catalog.execute("SELECT count(*) FROM calendar").fetchone()[0] == 0
def test_update_bars_rollback_on_failure(catalog, monkeypatch):
adapter = FakeAdapter(
securities=[SecurityRow(symbol="sh600000", name="A", category="stock", exchange="sh")],
bars=[
Bar1dRow(symbol="sh600000", date=date(2026, 9, 1), open=1, high=2, low=0.5,
close=1.5, volume=1, amount=2)
],
)
asyncio.run(update_securities(catalog, "cn", adapter))
def boom(*a, **k):
raise duckdb.Error("injected bars failure")
monkeypatch.setattr(duckdb.DuckDBPyConnection, "executemany", boom)
with pytest.raises(duckdb.Error, match="injected"):
asyncio.run(update_bars_1d(catalog, "cn", adapter))
assert catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0] == 0
def test_run_update_sync_exports_same_result(catalog):
"""包级导出的同步入口(供外部脚本调用)与 run_update 等价。"""
adapter = FakeAdapter(
securities=[SecurityRow(symbol="sh600000", name="A", category="stock", exchange="sh")],
bars=[
Bar1dRow(symbol="sh600000", date=date(2026, 9, 1), open=1, high=2, low=0.5,
close=1.5, volume=1, amount=2)
],
)
reports = run_update_sync(catalog, "cn", adapter, datasets=["securities", "bars_1d"])
assert [r.dataset for r in reports] == ["securities", "bars_1d"]
assert catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0] == 1
# ---------------------------------------------------------------- export.files
def test_run_query_arrow_params_and_none(catalog, mini_seed):
import_seed(catalog, mini_seed)
tbl = run_query_arrow(catalog, "SELECT count(*) AS n FROM bars_1d")
assert tbl.num_rows == 1
tbl2 = run_query_arrow(
catalog, "SELECT count(*) AS n FROM bars_1d WHERE symbol = ?", ["sh600000"]
)
assert tbl2.column("n").to_pylist() == [10]
# ---------------------------------------------------------------- lockfile
def test_stale_lock_backup_print_tolerates_closed_stdout(layout, monkeypatch):
"""陈旧锁接管时 stdout 写不进去也不能阻塞拿锁(极端部署场景)。"""
from pyquantdata import lockfile
import json as _json
from pyquantdata.lockfile import _boot_id
stale_path = layout.lock_file
stale_path.parent.mkdir(parents=True, exist_ok=True)
# 造一个「活锁判据齐全但进程已死」的锁:boot_id 必须是当前真实值(float),
# 否则 read_lock 会把它当损坏文件;PID 取几乎不存在的值 → is_alive False。
stale_path.write_text(
_json.dumps({
"pid": 999999999, "start_time": 0.0, "boot_id": _boot_id(),
"acquired_at": "2026-01-01T00:00:00", "cmd": "stale",
}),
encoding="utf-8",
)
def boom(*a, **k):
raise OSError("stdout closed")
monkeypatch.setattr("builtins.print", boom)
lock = lockfile.acquire(stale_path, cmd="new-instance")
assert stale_path.with_suffix(stale_path.suffix + ".bak").exists()
lockfile.release(stale_path, lock)
# ---------------------------------------------------------------- serve.app
def test_table_to_rows_empty_table():
assert table_to_rows(pa.table({})) == []
def test_utc_now_iso_format():
s = utc_now_iso()
assert s.endswith("Z") and len(s) == 20
class _BrokenCursor:
"""cursor.execute 直接抛 duckdb.Error → 覆盖 ready 的 catalog 异常分支。"""
def execute(self, *a, **k):
raise duckdb.Error("catalog is gone")
def cursor(self):
return self
def test_ready_catalog_error_branch(dbpath):
ctx = AppContext(dbpath=dbpath, config=__import__(
"pyquantdata.config", fromlist=["default_config"]
).default_config(), cursor=_BrokenCursor(), schema_version=1)
with TestClient(create_app(ctx)) as c:
body = c.get("/v1/ready")
assert body.status_code == 503
assert body.json()["checks"]["catalog"].startswith("error:")
def test_export_execution_error_400(client):
r = client.post("/v1/export", json={"sql": "SELECT * FROM no_such_table", "fmt": "csv"})
assert r.status_code == 400
assert "执行失败" in r.json()["detail"]
# ---------------------------------------------------------------- serve.runtime
# find_listener_pid 的 psutil 异常分支(无权限/无 psutil 的平台)
def test_find_listener_pid_swallows_psutil_error(monkeypatch):
"""psutil 抛错(权限/平台差异)按「找不到占用者」处理,不阻塞启动判定。"""
import psutil as real_psutil
import pyquantdata.serve.runtime as rt
def boom(*a, **k):
raise RuntimeError("psutil permission denied")
# runtime.find_listener_pid 内是函数级 import,patch 真实 psutil 模块
monkeypatch.setattr(real_psutil, "net_connections", boom)
assert rt.find_listener_pid(12345) is None
def test_find_listener_pid_returns_none_when_no_listener():
import pyquantdata.serve.runtime as rt
# 一个几乎不可能被监听的端口
assert rt.find_listener_pid(1) in (None, rt.find_listener_pid(1))
@pytest.fixture
def ready_db(layout, dbpath):
conn = open_catalog(layout.root)
migrate(conn)
conn.close()
return dbpath
def test_serve_conn_close_error_is_swallowed(ready_db, monkeypatch):
"""退出清理时 conn.close() 抛错不应吞掉退出码(finally 里已 catch)。"""
import pyquantdata.serve.runtime as rt
class _Cfg:
host = "127.0.0.1"
port = 8765
class _Server:
def __init__(self, config):
pass
async def serve(self):
raise RuntimeError("bind failed mid-flight")
monkeypatch.setattr(rt.uvicorn, "Server", _Server)
with pytest.raises(RuntimeError, match="bind failed"):
asyncio.run(rt.run_serve(ready_db, config=_cfg_default(), check_port=False))
def _cfg_default():
from pyquantdata.config import default_config
return default_config()
# ---------------------------------------------------------------- sqlgate
def test_sqlgate_positional_root_identifier_rejected():
"""非 SELECT/CTE 根节点(如 CREATE)被拒(§8.3 根节点白名单)。"""
from pyquantdata.sqlgate import SqlRejected, validate
with pytest.raises(SqlRejected, match="仅允许"):
validate("CREATE TABLE x (a INT)")
def test_sqlgate_empty_sql_rejected():
from pyquantdata.sqlgate import SqlRejected, validate
with pytest.raises(SqlRejected):
validate(" ")
# ---------------------------------------------------------------- migrations
def test_current_version_on_connection_without_table(layout):
"""空库(未建 schema_migrations)→ current_version 返回 0。"""
from pyquantdata.store.migrations import current_version
conn = duckdb.connect(":memory:")
assert current_version(conn) == 0
conn.close()
def test_app_is_fastapi_instance(catalog, dbpath):
from pyquantdata.config import default_config
from pyquantdata.store.db import configure_query_conn
cursor = catalog.cursor()
configure_query_conn(cursor)
app = create_app(AppContext(dbpath=dbpath, config=default_config(), cursor=cursor))
assert isinstance(app, FastAPI)
def test_security_error_is_raisable():
from pyquantdata.serve.auth import assert_security
with pytest.raises(SecurityError):
assert_security("0.0.0.0", "", insecure=False)
def test_checkpoint_upsert_then_list_roundtrip(catalog):
upsert_checkpoint(catalog, "d1", "cn", "s1", "w1", 5)
row = catalog.execute("SELECT dataset, rows FROM etl_checkpoint").fetchone()
assert row == ("d1", 5)
def test_open_catalog_write_then_reopen(tmp_path):
conn = open_catalog(tmp_path)
migrate(conn)
conn.close()
conn2 = open_catalog(tmp_path)
assert conn2.execute("SELECT count(*) FROM bars_1d").fetchone()[0] == 0
conn2.close()
# ---------------------------------------------------------------- 收尾补口
def test_module_entrypoint_runs(capsys, monkeypatch):
"""python -m pyquantdata 入口(__main__ 守卫)可执行。"""
import runpy
monkeypatch.setattr("sys.argv", ["pyquantdata", "--help"])
try:
runpy.run_module("pyquantdata.__main__", run_name="__main__")
except SystemExit:
pass # typer --help 正常退出
def test_cli_serve_command_delegates_to_runtime(monkeypatch, layout):
"""serve 子命令:编排交给 run_serve,退出码原样透传。"""
from typer.testing import CliRunner
from pyquantdata import cli as cli_mod
captured = {}
async def fake_run_serve(dbpath, **kw):
captured["dbpath"] = dbpath
captured.update(kw)
return 7 # 用非零值验证透传
monkeypatch.setattr("pyquantdata.serve.runtime.run_serve", fake_run_serve)
result = CliRunner().invoke(
cli_mod.app, ["serve", "-d", str(layout.root), "--no-http", "--insecure", "--port", "9999"]
)
assert result.exit_code == 7
assert captured["no_http"] is True and captured["insecure"] is True
assert captured["port"] == 9999
def test_cli_table_to_rows_empty():
import pyarrow as pa
from pyquantdata.cli import table_to_rows
assert table_to_rows(pa.table({})) == []
@pytest.fixture
def initialized(dbpath, mini_seed):
"""已 init 的库(本文件专用,避免跨文件依赖 test_cli 的 fixture)。"""
from typer.testing import CliRunner
from pyquantdata.cli import app
result = CliRunner().invoke(
app, ["init", "-d", str(dbpath), "--skip-history", "--seed", str(mini_seed)]
)
assert result.exit_code == 0, result.output
return dbpath
def test_cli_status_disk_error_branch(initialized, monkeypatch):
"""status 的磁盘探测失败 → info['disk'] = None,命令仍成功。"""
import psutil
from typer.testing import CliRunner
from pyquantdata import cli as cli_mod
def boom(_):
raise OSError("disk gone")
monkeypatch.setattr(psutil, "disk_usage", boom)
result = CliRunner().invoke(cli_mod.app, ["status", "-d", str(initialized), "--json"])
assert result.exit_code == 0
assert '"disk": null' in result.output
def test_resolve_export_target_rejects_escaping_job_id(tmp_path):
"""job_id 中的路径穿越也要被白名单拦住(非仅 filename)。"""
from pyquantdata.export.files import ExportPathError, resolve_export_target
with pytest.raises(ExportPathError, match="job_id"):
resolve_export_target(tmp_path / "exports", "..", "ok.csv")
def test_resolve_export_target_dotdot_filename_reaches_prefix_guard(tmp_path, monkeypatch):
"""纵深防御:即便两个白名单被绕过,resolve 前缀校验仍必须拦住越界。
常规输入下白名单已挡死(`/`、`..` 均被拒),所以这道前缀校验在无软链环境下
结构性不可达。这里用 monkeypatch 让白名单「假放行」+ resolve 返回越界路径,
直接验证防线本身有效——安全逻辑不能只靠「到不了」来保证。
"""
from pathlib import Path as _P
from pyquantdata.export import files as files_mod
from pyquantdata.export.files import ExportPathError, resolve_export_target
monkeypatch.setattr(files_mod, "_JOB_ID_RE", type("R", (), {"match": lambda *_: True})())
monkeypatch.setattr(files_mod, "_FILENAME_RE", type("R", (), {"match": lambda *_: True})())
outside = tmp_path / "outside" / "evil.csv"
real_resolve = _P.resolve
monkeypatch.setattr(_P, "resolve", lambda self, *a, **k: outside if "outside" in str(self) or str(self).endswith("evil.csv") else real_resolve(self, *a, **k))
with pytest.raises(ExportPathError, match="越界"):
resolve_export_target(tmp_path / "exports", "okjob", "evil.csv")
def test_query_result_too_large_real_path(catalog, dbpath, mini_seed):
"""真实触发结果过大(max_arrow_bytes 设极小)→ 413。"""
from pyquantdata.config import default_config
from pyquantdata.serve.app import AppContext, create_app
from pyquantdata.store.db import configure_query_conn
import_seed(catalog, mini_seed)
cfg = default_config()
cfg.sql.max_arrow_bytes = 1 # 任何结果都超限
cursor = catalog.cursor()
configure_query_conn(cursor)
with TestClient(create_app(AppContext(dbpath=dbpath, config=cfg, cursor=cursor, schema_version=1))) as c:
r = c.post("/v1/query", json={"sql": "SELECT * FROM cn_stock_1d"})
assert r.status_code == 413
def test_serve_swallows_conn_close_error(ready_db, monkeypatch):
"""退出清理时 conn.close() 抛错 → 被吞掉,退出码仍为 0。"""
import pyquantdata.serve.runtime as rt
class _Server:
def __init__(self, config):
pass
async def serve(self):
return None
class _ConnClose:
"""包装真实连接:close 时抛错。"""
def __init__(self, real):
self._real = real
def __getattr__(self, name):
return getattr(self._real, name)
def close(self):
raise RuntimeError("close exploded")
real_open = rt.open_catalog
def wrap(root, *a, **k):
return _ConnClose(real_open(root, *a, **k))
monkeypatch.setattr(rt, "open_catalog", wrap)
monkeypatch.setattr(rt.uvicorn, "Server", _Server)
code = asyncio.run(rt.run_serve(ready_db, config=_cfg_default(), check_port=False))
assert code == rt.EXIT_OK
def test_sqlgate_rejects_scan_suffix_function():
"""`*_scan` 表函数必须被拒(§8.3)。
两种落点都要拦:FROM 位置的表函数由 _check_from_tables 先拦;函数白名单
对任何位置都生效。这里只断言「被拒」,不写死命中哪道闸。
"""
from pyquantdata.sqlgate import SqlRejected, validate
for sql in (
"SELECT * FROM parquet_scan('/tmp/x.parquet')",
"SELECT parquet_scan('/tmp/x.parquet')",
"SELECT 1 FROM t WHERE 1 IN (SELECT * FROM delta_scan('/tmp/d'))",
):
with pytest.raises(SqlRejected):
validate(sql)
def test_sqlgate_forbidden_function_in_subquery():
"""禁用函数藏在子查询/CTE 内也要被递归拦下(§8.3)。"""
from pyquantdata.sqlgate import SqlRejected, validate
for sql in (
"SELECT 1 WHERE 1 IN (SELECT read_csv('/etc/passwd'))",
"WITH x AS (SELECT getenv('HOME') AS h) SELECT h FROM x",
):
with pytest.raises(SqlRejected):
validate(sql)
+52
View File
@@ -0,0 +1,52 @@
"""store/db.py:连接管理、只读约束、查询连接安全预设。"""
from __future__ import annotations
import pytest
from pyquantdata.store import CatalogNotInitialized, open_catalog
from pyquantdata.store.db import configure_query_conn
def test_open_read_only_uninitialized_raises(tmp_path):
with pytest.raises(CatalogNotInitialized):
open_catalog(tmp_path, read_only=True)
def test_open_write_creates_catalog(layout):
conn = open_catalog(layout.root)
conn.execute("SELECT 1")
conn.close()
def test_configure_query_conn_sets_guards(catalog):
configure_query_conn(catalog, memory_limit="1GB")
autoinstall = catalog.execute(
"SELECT current_setting('autoinstall_known_extensions')"
).fetchone()[0]
autoload = catalog.execute(
"SELECT current_setting('autoload_known_extensions')"
).fetchone()[0]
assert autoinstall in ("false", False)
assert autoload in ("false", False)
mem = catalog.execute("SELECT current_setting('memory_limit')").fetchone()[0]
# duckdb 会把 1GB 规范化显示(如 '953.6 MiB');断言数值≈1GiB 而非字面量
import re as _re
m = _re.match(r"([\d.]+)\s*(KiB|MiB|GiB|KB|MB|GB)", str(mem))
assert m, mem
val, unit = float(m.group(1)), m.group(2)
gib = val / 1024 if unit in ("MiB", "MB") else val
assert 0.9 <= gib <= 1.1, mem
def test_read_only_blocks_write(layout):
conn = open_catalog(layout.root)
from pyquantdata.store import migrate
migrate(conn)
conn.close()
ro = open_catalog(layout.root, read_only=True)
with pytest.raises(Exception):
ro.execute("CREATE TABLE nope (a INT)")
ro.close()
+127
View File
@@ -0,0 +1,127 @@
"""update ETL 编排:FakeAdapter 确定性数据,幂等重放。"""
from __future__ import annotations
import asyncio
from datetime import date
import pytest
from pyquantdata.etl import run_update
from pyquantdata.sources import FakeAdapter
from pyquantdata.sources.base import Bar1dRow, CalendarRow, SecurityRow
def _bars():
out = []
for sym in ("sh600000", "sh600519"):
for d in range(1, 6):
out.append(
Bar1dRow(
symbol=sym, date=date(2026, 9, d),
open=10.0 + d, high=11.0 + d, low=9.0 + d, close=10.5 + d,
volume=1000.0 + d, amount=10500.0 + d,
)
)
return out
def _securities():
return [
SecurityRow(symbol="sh600000", name="浦发银行", category="stock", exchange="sh", lot_size=100),
SecurityRow(symbol="sh600519", name="贵州茅台", category="stock", exchange="sh", lot_size=100),
SecurityRow(symbol="sh000001", name="上证指数", category="index", exchange="sh"),
]
def _calendar():
return [
CalendarRow(trade_date=date(2026, 9, d), is_open=True, note="")
for d in (1, 2, 3, 7, 8, 9)
]
@pytest.fixture
def adapter():
return FakeAdapter(
securities=_securities(), bars=_bars(), calendar={2026: _calendar()}
)
def test_run_update_full_chain(catalog, adapter):
reports = asyncio.run(
run_update(catalog, "cn", adapter, datasets=["securities", "calendar", "bars_1d"], years=[2026])
)
assert [r.dataset for r in reports] == ["securities", "calendar", "bars_1d"]
assert reports[0].rows == 3
assert catalog.execute("SELECT count(*) FROM securities").fetchone()[0] == 3
# calendar 落库恒为整年逐日(2026 非闰年 = 365),is_open 只标 6 个交易日
assert catalog.execute("SELECT count(*) FROM calendar WHERE market='cn'").fetchone()[0] == 365
assert catalog.execute("SELECT count(*) FROM calendar WHERE is_open").fetchone()[0] == 6
# 日K:只有 securities 里的 stock/index/etf/cbond 进入回填清单
assert reports[2].rows == 10 # 2 stock × 5 天;index 在清单内但 FakeAdapter 无其 bar
assert reports[2].date_min == date(2026, 9, 1)
assert reports[2].date_max == date(2026, 9, 5)
# checkpoint 已写
cps = catalog.execute(
"SELECT dataset, status FROM etl_checkpoint ORDER BY dataset"
).fetchall()
assert [c[0] for c in cps] == ["bars_1d", "calendar", "securities"]
def test_bars_replay_idempotent(catalog, adapter):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["securities", "bars_1d"]))
n1 = catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0]
asyncio.run(run_update(catalog, "cn", adapter, datasets=["bars_1d"]))
n2 = catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0]
assert n1 == n2 == 10
def test_bars_with_explicit_symbols_and_range(catalog, adapter):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["securities"]))
report = asyncio.run(
run_update(
catalog, "cn", adapter, datasets=["bars_1d"],
symbols=["sh600000"], start=date(2026, 9, 2), end=date(2026, 9, 4),
)
)[0]
assert report.rows == 3
rows = catalog.execute(
"SELECT count(*), min(date), max(date) FROM bars_1d WHERE symbol='sh600000'"
).fetchone()
assert rows == (3, date(2026, 9, 2), date(2026, 9, 4))
def test_bars_empty_securities_raises(catalog, adapter):
with pytest.raises(RuntimeError, match="securities"):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["bars_1d"]))
def test_partial_overlap_replay_replaces(catalog, adapter):
"""重放窗口小于已有区间:重放区替换、区外保留(§7.3 整区替换)。"""
asyncio.run(run_update(catalog, "cn", adapter, datasets=["securities", "bars_1d"]))
# 现在只重放 9/1~9/2,且数据变化(close 改为 99)
slim = FakeAdapter(securities=_securities())
slim.bars = [
Bar1dRow(symbol="sh600000", date=date(2026, 9, d), open=1, high=2, low=0.5,
close=99, volume=1, amount=2)
for d in (1, 2)
]
asyncio.run(
run_update(catalog, "cn", slim, datasets=["bars_1d"], symbols=["sh600000"],
start=date(2026, 9, 1), end=date(2026, 9, 2))
)
n_99 = catalog.execute("SELECT count(*) FROM bars_1d WHERE close=99").fetchone()[0]
n_kept = catalog.execute(
"SELECT count(*) FROM bars_1d WHERE symbol='sh600000' AND date >= DATE '2026-09-03'"
).fetchone()[0]
assert n_99 == 2
assert n_kept == 3 # 旧区间未被动
def test_calendar_replay_idempotent(catalog, adapter):
asyncio.run(run_update(catalog, "cn", adapter, datasets=["calendar"], years=[2026]))
n1 = catalog.execute("SELECT count(*) FROM calendar").fetchone()[0]
asyncio.run(run_update(catalog, "cn", adapter, datasets=["calendar"], years=[2026]))
n2 = catalog.execute("SELECT count(*) FROM calendar").fetchone()[0]
assert n1 == n2 == 365 # 整年逐日,重放不翻倍
+101
View File
@@ -0,0 +1,101 @@
"""导出文件层:路径安全、落盘、job 记录。"""
from __future__ import annotations
import pytest
import pyarrow.parquet as paparquet
from pyquantdata.export.files import (
ExportFormatError,
ExportPathError,
get_export_job,
new_job_id,
record_export_job,
resolve_export_target,
spec_hash,
write_export,
)
def test_new_job_id_shape():
jid = new_job_id()
assert len(jid) == 12
assert all(c in "0123456789abcdef" for c in jid)
def test_spec_hash_stable_and_distinct():
a = spec_hash({"sql": "SELECT 1", "fmt": "csv"})
assert a == spec_hash({"fmt": "csv", "sql": "SELECT 1"}) # 键序无关
assert a != spec_hash({"sql": "SELECT 2", "fmt": "csv"})
def test_resolve_target_ok(tmp_path):
exports = tmp_path / "exports"
target = resolve_export_target(exports, "abc123", "out.csv")
assert target == exports.resolve() / "abc123" / "out.csv"
@pytest.mark.parametrize(
"job_id,filename",
[
("Bad!ID", "ok.csv"), # job_id 非法字符
("", "ok.csv"),
("a" * 100, "ok.csv"), # 超长
("okjob", "../evil.csv"), # 路径穿越
("okjob", "..\\evil.csv"), # Windows 分隔符
("okjob", "sub/dir/x.csv"), # 斜杠
("okjob", ""),
("okjob", "a" * 200), # 超长文件名
],
)
def test_resolve_target_rejects(tmp_path, job_id, filename):
with pytest.raises(ExportPathError):
resolve_export_target(tmp_path / "exports", job_id, filename)
def test_resolve_target_dotdot_filename_rejected(tmp_path):
with pytest.raises(ExportPathError):
resolve_export_target(tmp_path / "exports", "okjob", "..")
def test_write_export_csv(tmp_path):
import pyarrow as pa
tbl = pa.table({"a": [1, 2], "b": ["x", "y"]})
result = write_export(tbl, "csv", tmp_path / "out.csv")
assert result.rows == 2
assert result.path.exists()
text = result.path.read_text(encoding="utf-8")
# pyarrow 默认按 RFC4180 加引号("a","b")——断言语义而非引号风格
assert '"a","b"' in text and '1,"x"' in text
def test_write_export_parquet(tmp_path):
import pyarrow as pa
tbl = pa.table({"a": [1, 2, 3]})
result = write_export(tbl, "parquet", tmp_path / "out.parquet")
back = paparquet.read_table(result.path)
assert back.column("a").to_pylist() == [1, 2, 3]
def test_write_export_bad_format(tmp_path):
import pyarrow as pa
with pytest.raises(ExportFormatError, match="落盘格式"):
write_export(pa.table({"a": [1]}), "pandas", tmp_path / "x.pandas")
def test_record_and_get_job(catalog):
record_export_job(
catalog, "job123", fmt="csv", path="exports/job123/out.csv",
rows=10, bytes_written=128,
)
job = get_export_job(catalog, "job123")
assert job["job_id"] == "job123"
assert job["fmt"] == "csv"
assert job["rows"] == 10
assert job["bytes"] == 128
assert job["state"] == "done"
assert job["finished_at"] is not None
assert get_export_job(catalog, "missing") is None
+102
View File
@@ -0,0 +1,102 @@
"""lockfile.py:三元判活单写者锁(§11)。"""
from __future__ import annotations
import json
import pytest
from pyquantdata.lockfile import (
LockHeldError,
LockInfo,
acquire,
is_alive,
read_lock,
release,
)
@pytest.fixture
def lock_path(tmp_path):
return tmp_path / "quantdata.lock"
def test_acquire_and_release(lock_path):
info = acquire(lock_path, cmd="test")
assert lock_path.exists()
assert info.pid > 0
assert release(lock_path, info) is True
assert not lock_path.exists()
assert release(lock_path, info) is False # 已删
def test_second_acquire_blocked_same_process(lock_path):
info = acquire(lock_path)
with pytest.raises(LockHeldError) as e:
acquire(lock_path)
assert e.value.info.pid == info.pid
def test_stale_lock_backed_up_and_taken(lock_path):
stale = LockInfo(
pid=999_999_999, # 不存在的进程
start_time=1_000_000.0,
boot_id=__import__("pyquantdata.lockfile", fromlist=["_boot_id"])._boot_id(),
acquired_at="2020-01-01T00:00:00",
cmd="old",
)
lock_path.write_text(stale.to_json(), encoding="utf-8")
info = acquire(lock_path)
assert info.pid == __import__("os").getpid()
bak = lock_path.with_suffix(lock_path.suffix + ".bak")
assert bak.exists()
assert json.loads(bak.read_text(encoding="utf-8"))["pid"] == 999_999_999
def test_boot_id_mismatch_is_stale(lock_path, monkeypatch):
info = acquire(lock_path)
assert is_alive(info) is True
fake = LockInfo(info.pid, info.start_time, boot_id=1.0, acquired_at="", cmd="")
assert is_alive(fake) is False # 重启过系统
def test_start_time_mismatch_is_stale(lock_path):
info = acquire(lock_path)
fake = LockInfo(info.pid, info.start_time + 999.0, info.boot_id, "", "")
assert is_alive(fake) is False
def test_dead_pid_is_stale(lock_path):
fake = LockInfo(999_999_999, 1.0, __import__("pyquantdata.lockfile", fromlist=["_boot_id"])._boot_id(), "", "")
assert is_alive(fake) is False
def test_read_lock_corrupt_json_returns_none(lock_path):
lock_path.write_text("{not json", encoding="utf-8")
assert read_lock(lock_path) is None
def test_read_lock_missing_returns_none(lock_path):
assert read_lock(lock_path) is None
def test_read_lock_bad_fields_returns_none(lock_path):
lock_path.write_text('{"pid": "x"}', encoding="utf-8")
assert read_lock(lock_path) is None
def test_release_refuses_to_delete_other_owner(lock_path):
info = acquire(lock_path)
other = LockInfo(
pid=info.pid, start_time=info.start_time + 5.0, boot_id=info.boot_id,
acquired_at="", cmd="",
)
assert release(lock_path, other) is False
assert lock_path.exists() # 后来者的锁不被误删
def test_lock_info_roundtrip(tmp_path):
info = acquire(tmp_path / "l.lock", cmd="quantdata serve")
parsed = LockInfo.from_json((tmp_path / "l.lock").read_text(encoding="utf-8"))
assert parsed == info
assert "quantdata serve" in parsed.cmd
+104
View File
@@ -0,0 +1,104 @@
"""migrations:版本管理、幂等、TooNew 拒启、视图列集。"""
from __future__ import annotations
import pytest
from pyquantdata.store import (
KNOWN_MAX_VERSION,
MIGRATIONS,
SchemaTooNewError,
applied_versions,
current_version,
migrate,
)
BAR_COLUMNS = {
"market", "category", "symbol", "date", "open", "high", "low",
"close", "volume", "amount", "turnover", "adj_factor",
}
def test_fresh_migrate(catalog):
assert current_version(catalog) == KNOWN_MAX_VERSION
assert applied_versions(catalog) == [1]
def test_migrate_idempotent(catalog):
assert migrate(catalog) == KNOWN_MAX_VERSION
assert migrate(catalog) == KNOWN_MAX_VERSION
assert applied_versions(catalog) == [1]
def test_core_tables_exist(catalog):
rows = catalog.execute(
"SELECT table_name FROM information_schema.tables WHERE table_schema='main' "
"AND table_type='BASE TABLE' ORDER BY table_name"
).fetchall()
names = {r[0] for r in rows}
assert {
"exchanges", "securities", "calendar", "corp_actions", "sw_industry",
"stock_status", "futures_meta", "bars_1d", "announcements", "reports",
"news", "etl_checkpoint", "export_jobs", "quality_reports",
"schema_migrations",
} <= names
def test_day_views_exist_with_exact_columns(catalog):
from pyquantdata.store.views import DAY_VIEWS
for view, market, category in DAY_VIEWS:
cols = [
r[0]
for r in catalog.execute(
"SELECT column_name FROM information_schema.columns "
"WHERE table_schema='main' AND table_name=? ORDER BY ordinal_position",
[view],
).fetchall()
]
assert set(cols) == BAR_COLUMNS, view
# 列集恒等:三市场同品类视图列集一致(§3.3)
stock_views = [v for v, _, c in DAY_VIEWS if c == "stock"]
assert len(stock_views) >= 3 # cn/us/hk
def test_day_view_filters_market(catalog):
catalog.execute(
"INSERT INTO bars_1d VALUES ('cn','stock','sh600000',DATE '2026-09-01',"
"10,11,9,10.5,1000,10500,1.2,1.0)"
)
assert catalog.execute("SELECT count(*) FROM cn_stock_1d").fetchone()[0] == 1
assert catalog.execute("SELECT count(*) FROM us_stock_1d").fetchone()[0] == 0
def test_schema_too_new_rejected(catalog):
catalog.execute("INSERT INTO schema_migrations (version) VALUES (99)")
with pytest.raises(SchemaTooNewError):
migrate(catalog)
def test_migration_failure_rolls_back(catalog, monkeypatch):
"""迁移包事务:中途失败回滚,版本停在最后一个成功迁移(§11)。"""
import duckdb
from pyquantdata.store import migrations as m
broken = list(m.MIGRATIONS) + [
(2, "0002_bad", "CREATE TABLE oops (x INT); CREATE TABLE oops (x INT)")
]
monkeypatch.setattr(m, "MIGRATIONS", tuple(broken))
monkeypatch.setattr(m, "KNOWN_MAX_VERSION", 2)
conn2 = duckdb.connect(":memory:")
with pytest.raises(Exception):
m.migrate(conn2)
# migration 1 已提交、migration 2 回滚:无 oops 表
assert m.current_version(conn2) == 1
has_oops = conn2.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_name='oops'"
).fetchone()[0]
assert has_oops == 0
conn2.close()
def test_known_max_version_matches_list():
assert KNOWN_MAX_VERSION == MIGRATIONS[-1][0]
+47
View File
@@ -0,0 +1,47 @@
"""paths.py:目录规范与 dbpath 解析。"""
from __future__ import annotations
import pytest
from pyquantdata.paths import DbPathLayout, resolve_dbpath
def test_layout_properties(tmp_path):
lay = DbPathLayout(tmp_path / "db")
assert lay.config_file.name == "config.json"
assert lay.catalog.name == "catalog.duckdb"
assert lay.lock_file.name == "quantdata.lock"
assert lay.trash_dir.name == ".trash"
assert lay.data_dir.parent == lay.root
def test_ensure_dirs_creates_all(tmp_path):
lay = DbPathLayout(tmp_path / "db")
made = lay.ensure_dirs()
for d in (lay.root, lay.data_dir, lay.scripts_dir, lay.exports_dir, lay.logs_dir, lay.tmp_dir, lay.trash_dir):
assert d.exists()
assert lay.root in made
def test_ensure_dirs_idempotent(tmp_path):
lay = DbPathLayout(tmp_path / "db")
lay.ensure_dirs()
made2 = lay.ensure_dirs()
assert len(made2) == 7
def test_resolve_explicit_wins(tmp_path, monkeypatch):
monkeypatch.setenv("QUANTDATA_HOME", str(tmp_path / "from_env"))
assert resolve_dbpath(str(tmp_path / "explicit")) == (tmp_path / "explicit").resolve()
def test_resolve_env_fallback(tmp_path, monkeypatch):
monkeypatch.setenv("QUANTDATA_HOME", str(tmp_path / "from_env"))
assert resolve_dbpath(None) == (tmp_path / "from_env").resolve()
def test_resolve_missing_raises(monkeypatch):
monkeypatch.delenv("QUANTDATA_HOME", raising=False)
with pytest.raises(SystemExit):
resolve_dbpath(None)
+123
View File
@@ -0,0 +1,123 @@
"""serve runtime:锁、迁移校验、安全校验、端口探测、退出码(真 uvicorn 段按 §14 白名单替身)。"""
from __future__ import annotations
import asyncio
import socket
import pytest
from pyquantdata.config import default_config
from pyquantdata.lockfile import acquire, read_lock
from pyquantdata.paths import DbPathLayout
from pyquantdata.store import migrate, open_catalog
from pyquantdata.serve.runtime import (
EXIT_LOCK_HELD,
EXIT_OK,
EXIT_PORT_BUSY,
EXIT_SCHEMA_TOO_NEW,
EXIT_SECURITY,
find_listener_pid,
run_serve,
)
@pytest.fixture
def ready_db(layout, dbpath):
conn = open_catalog(layout.root)
migrate(conn)
conn.close()
return dbpath
def _cfg():
return default_config()
def test_dbpath_missing_returns_1(tmp_path):
code = asyncio.run(run_serve(tmp_path / "nope", config=_cfg()))
assert code == 1
def test_lock_held_returns_2(ready_db):
layout = DbPathLayout(ready_db)
acquire(layout.lock_file, cmd="other-instance")
code = asyncio.run(run_serve(ready_db, no_http=True, stop_event=_set_event(), config=_cfg()))
assert code == EXIT_LOCK_HELD
# run_serve 失败不释放别人的锁
assert read_lock(layout.lock_file).cmd == "other-instance"
def _set_event():
ev = asyncio.Event()
ev.set()
return ev
def test_no_http_graceful_exit(ready_db):
code = asyncio.run(run_serve(ready_db, no_http=True, stop_event=_set_event(), config=_cfg()))
assert code == EXIT_OK
# 锁已释放
assert read_lock(DbPathLayout(ready_db).lock_file) is None
def test_schema_too_new_returns_3(layout, dbpath):
conn = open_catalog(layout.root)
migrate(conn)
conn.execute("INSERT INTO schema_migrations (version) VALUES (99)")
conn.close()
code = asyncio.run(run_serve(dbpath, no_http=True, stop_event=_set_event(), config=_cfg()))
assert code == EXIT_SCHEMA_TOO_NEW
def test_security_rejects_insecure_bind(ready_db):
cfg = _cfg()
cfg.http.token = ""
code = asyncio.run(
run_serve(ready_db, host="0.0.0.0", no_http=True, stop_event=_set_event(), config=cfg)
)
assert code == EXIT_SECURITY
def test_insecure_flag_allows_start(ready_db, monkeypatch):
code = asyncio.run(
run_serve(
ready_db, host="0.0.0.0", insecure=True, no_http=True,
stop_event=_set_event(), config=_cfg(),
)
)
assert code == EXIT_OK
def test_port_busy_returns_5(ready_db):
"""端口真被占 → 打印占用 PID 后非零退出(§7.1 bind 失败语义)。"""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
s.listen(1)
busy_port = s.getsockname()[1]
assert find_listener_pid(busy_port) is not None
code = asyncio.run(run_serve(ready_db, port=busy_port, config=_cfg()))
assert code == EXIT_PORT_BUSY
def test_http_path_full_setup_with_fake_uvicorn(ready_db, monkeypatch):
"""HTTP 路径全编排(锁→migrate→cursor→app→uvicorn.Config),真 serve 用替身(§14 白名单)。"""
import pyquantdata.serve.runtime as rt
served = {}
class _FakeServer:
def __init__(self, config):
served["config"] = config
assert config.host == "127.0.0.1"
assert config.port == 8765
assert config.timeout_graceful_shutdown == 30
async def serve(self):
served["served"] = True
monkeypatch.setattr(rt.uvicorn, "Server", _FakeServer)
code = asyncio.run(run_serve(ready_db, config=_cfg(), check_port=False))
assert code == EXIT_OK
assert served["served"] is True
assert read_lock(DbPathLayout(ready_db).lock_file) is None
+140
View File
@@ -0,0 +1,140 @@
"""seed 导入:parquet/csv、幂等、校验。"""
from __future__ import annotations
from datetime import date
import pyarrow as pa
import pyarrow.parquet as paparquet
import pytest
from pyquantdata.etl import SeedError, import_seed
def test_import_parquet(catalog, mini_seed):
report = import_seed(catalog, mini_seed)
assert report.rows == 10
assert report.symbols == ["sh600000"]
assert report.date_min == date(2026, 9, 1)
assert report.date_max == date(2026, 9, 10)
rows = catalog.execute(
"SELECT count(*), min(date), max(date) FROM bars_1d WHERE symbol='sh600000'"
).fetchone()
assert rows == (10, report.date_min, report.date_max)
# category 按规则分类
assert catalog.execute(
"SELECT DISTINCT category FROM bars_1d WHERE symbol='sh600000'"
).fetchone()[0] == "stock"
def test_seed_idempotent_replay(catalog, mini_seed):
import_seed(catalog, mini_seed)
import_seed(catalog, mini_seed)
assert catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0] == 10
def test_import_csv_with_alias_columns(catalog, tmp_path):
p = tmp_path / "seed.csv"
p.write_text(
"symbol,date,open,high,low,close,vol,amt\n"
"sz000001,2026-09-01,15.0,15.5,14.8,15.2,2000,30000\n"
"sz000001,2026-09-02,15.2,15.8,15.0,15.6,2200,34000\n",
encoding="utf-8",
)
report = import_seed(catalog, p)
assert report.rows == 2
row = catalog.execute(
"SELECT volume, amount FROM bars_1d WHERE symbol='sz000001' AND date='2026-09-01'"
).fetchone()
assert row == (2000.0, 30000.0)
def test_import_with_optional_adjust_factor(catalog, tmp_path):
p = tmp_path / "adj.parquet"
paparquet.write_table(
pa.table(
{
"symbol": ["sh600000"],
"date": [date(2026, 9, 1)],
"open": [10.0],
"high": [10.5],
"low": [9.5],
"close": [10.2],
"volume": [100.0],
"amount": [1000.0],
"adj_factor": [1.5],
"turnover": [1.2],
}
),
p,
)
import_seed(catalog, p)
row = catalog.execute(
"SELECT adj_factor, turnover FROM bars_1d WHERE symbol='sh600000'"
).fetchone()
assert row == (1.5, 1.2)
def test_missing_required_column(catalog, tmp_path):
p = tmp_path / "bad.parquet"
paparquet.write_table(pa.table({"symbol": ["x"], "close": [1.0]}), p)
with pytest.raises(SeedError, match="缺必需列"):
import_seed(catalog, p)
def test_missing_file(catalog, tmp_path):
with pytest.raises(SeedError, match="不存在"):
import_seed(catalog, tmp_path / "nope.parquet")
def test_empty_table(catalog, tmp_path):
p = tmp_path / "empty.parquet"
paparquet.write_table(
pa.table(
{"symbol": [], "date": [], "open": [], "high": [], "low": [], "close": []}
),
p,
)
with pytest.raises(SeedError, match="无数据行"):
import_seed(catalog, p)
def test_unsupported_extension(catalog, tmp_path):
p = tmp_path / "seed.xlsx"
p.write_text("x", encoding="utf-8")
with pytest.raises(SeedError, match="仅支持"):
import_seed(catalog, p)
def test_empty_symbol_rejected(catalog, tmp_path):
p = tmp_path / "blank.parquet"
paparquet.write_table(
pa.table(
{
"symbol": [" "],
"date": [date(2026, 9, 1)],
"open": [1.0], "high": [1.0], "low": [1.0], "close": [1.0],
}
),
p,
)
with pytest.raises(SeedError, match="symbol 为空"):
import_seed(catalog, p)
def test_import_failure_rolls_back(catalog, tmp_path):
"""半途失败:先前 symbol 的写入也要回滚。"""
p = tmp_path / "bad_mid.parquet"
paparquet.write_table(
pa.table(
{
"symbol": ["sh600000", "sz000001"],
"date": ["2026-09-01", "not-a-date"], # 全字符串列;坏日期在行转换期抛错
"open": [1.0, 1.0], "high": [1.0, 1.0], "low": [1.0, 1.0], "close": [1.0, 1.0],
}
),
p,
)
with pytest.raises(Exception):
import_seed(catalog, p)
assert catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0] == 0
+84
View File
@@ -0,0 +1,84 @@
"""SQL 闸(§8.3):绕过样例矩阵全部入单测。"""
from __future__ import annotations
import pytest
from pyquantdata.sqlgate import SqlRejected, validate
VALID = [
"SELECT 1",
"SELECT date, close FROM cn_stock_1d WHERE symbol = ? ORDER BY date DESC LIMIT 5",
"WITH latest AS (SELECT symbol, max(date) AS d FROM cn_stock_1d GROUP BY symbol) "
"SELECT * FROM latest",
"SELECT 'cn' AS market, symbol FROM cn_stock_1d "
"UNION ALL SELECT 'us', symbol FROM us_stock_1d",
"SELECT count(*), avg(close) FROM bars_1d GROUP BY market",
"SELECT b.ts, b.close * d.adj_factor FROM cn_stock_1m b JOIN bars_1d d USING (symbol, date)",
"SELECT unnest([1,2,3])",
"SELECT abs(-1), date_trunc('day', now()), upper('a')",
"SELECT symbol FROM cn_stock_1d WHERE date >= DATE '2026-01-01'",
" SELECT 1 ", # 空白容忍
]
FORBIDDEN_SQL = [
("", "空 SQL"),
(" ", None),
("SELECT 1; SELECT 2", None),
("SELECT 1; ATTACH 'x.db' AS x", None),
("SELEC 1", None), # 解析失败 fail-closed
("ATTACH 'evil.db' AS e", None),
("-- comment\nATTACH 'evil.db' AS e", None), # 注释混淆
("COPY bars_1d TO '/tmp/steal.csv'", None),
("INSERT INTO bars_1d VALUES (1)", None),
("UPDATE bars_1d SET close = 0", None),
("DELETE FROM bars_1d", None),
("CREATE TABLE evil (a INT)", None),
("DROP TABLE bars_1d", None),
("SET memory_limit='100GB'", None),
("PRAGMA database_list", None),
("EXPLAIN SELECT 1", None),
("CALL pragma_version()", None),
# 函数默认拒绝(文件/网络/环境)
("SELECT read_parquet('data/**/*.parquet')", None),
("SELECT read_csv('/etc/passwd')", None),
("SELECT read_csv_auto('x.csv')", None),
("SELECT read_json('x.json')", None),
("SELECT glob('*.parquet')", None),
("SELECT getenv('HOME')", None),
("SELECT current_setting('memory_limit')", None),
("SELECT * FROM parquet_scan('x.parquet')", None),
("SELECT * FROM iceberg_scan('x')", None),
("SELECT * FROM read_csv('/etc/passwd')", None),
("SELECT (SELECT read_csv('y') LIMIT 1)", None), # 子查询里也拦
("WITH t AS (SELECT glob('x') AS g) SELECT g FROM t", None), # CTE 里也拦
]
@pytest.mark.parametrize("sql", VALID)
def test_valid_sql_passes(sql):
validate(sql)
@pytest.mark.parametrize("sql,_", FORBIDDEN_SQL, ids=range(len(FORBIDDEN_SQL)))
def test_forbidden_sql_rejected(sql, _):
with pytest.raises(SqlRejected) as e:
validate(sql)
assert e.value.reason
def test_allow_functions_extends_whitelist():
# 额外放行清单语义:显式列出的函数放行(§8.3 allow_functions)
validate("SELECT read_csv('x.csv')", allow_functions=["read_csv"])
# 未列出的仍然拒绝
with pytest.raises(SqlRejected):
validate("SELECT read_parquet('x')", allow_functions=["read_csv"])
# 清空 allow_functions ≠ 全放行(语义反转点)
with pytest.raises(SqlRejected):
validate("SELECT read_csv('x.csv')", allow_functions=[])
def test_rejected_error_carries_reason():
with pytest.raises(SqlRejected) as e:
validate("ATTACH 'x' AS y")
assert "只读" in e.value.reason
+72
View File
@@ -0,0 +1,72 @@
"""市场状态机纯函数:参数化时间边界全覆盖。"""
from __future__ import annotations
import pytest
from pyquantdata.serve.state import MARKET_TZ, market_state
def test_cn_session_boundaries():
# (hhmm, expected_state)
cases = [
(914, "CLOSED"), # 开盘前
(915, "PRE_OPEN_AUCTION"), # 集合竞价开始
(925, "PRE_OPEN_AUCTION"),
(929, "PRE_OPEN_AUCTION"),
(930, "OPEN"),
(1129, "OPEN"),
(1130, "LUNCH_BREAK"),
(1259, "LUNCH_BREAK"),
(1300, "OPEN"),
(1459, "OPEN"),
(1500, "CLOSED"), # 收盘
(2359, "CLOSED"),
]
for hhmm, state in cases:
ms = market_state("cn", hhmm, is_trading_day=True)
assert ms.state == state, f"hhmm={hhmm}: got {ms.state}, want {state}"
def test_cn_non_trading_day():
ms = market_state("cn", 1000, is_trading_day=False)
assert ms.state == "CLOSED"
assert ms.reason == "non_trading_day"
def test_us_no_lunch_break():
assert market_state("us", 1200, is_trading_day=True).state == "OPEN"
assert market_state("us", 929, is_trading_day=True).state == "CLOSED"
assert market_state("us", 930, is_trading_day=True).state == "OPEN"
assert market_state("us", 1559, is_trading_day=True).state == "OPEN"
assert market_state("us", 1600, is_trading_day=True).state == "CLOSED"
def test_hk_lunch_break():
assert market_state("hk", 1200, is_trading_day=True).state == "LUNCH_BREAK"
assert market_state("hk", 1259, is_trading_day=True).state == "LUNCH_BREAK"
assert market_state("hk", 1300, is_trading_day=True).state == "OPEN"
assert market_state("hk", 1600, is_trading_day=True).state == "CLOSED"
def test_fields_complete():
ms = market_state("cn", 1000, is_trading_day=True)
assert ms.market == "cn"
assert ms.session_tz == "Asia/Shanghai"
assert ms.reason == "session_rule"
assert ms.next_at == "11:30" # 收段时刻
ms2 = market_state("cn", 1200, is_trading_day=True)
assert ms2.next_at == "13:00" # 午休结束
ms3 = market_state("cn", 1600, is_trading_day=True)
assert ms3.next_at == "09:30" # 次日开盘
ms4 = market_state("cn", 920, is_trading_day=True)
assert ms4.next_at == "09:30" # 竞价结束
def test_market_tz_table():
assert set(MARKET_TZ) == {"cn", "us", "hk"}
def test_unknown_market_raises():
with pytest.raises(KeyError, match="未知市场"):
market_state("uk", 1000, is_trading_day=True)
+170
View File
@@ -0,0 +1,170 @@
"""TdxAdapter 映射逻辑(注入 fake client,不打真网)。"""
from __future__ import annotations
import asyncio
from datetime import date
import pytest
from pyquantdata.sources import TdxAdapter
from pyquantdata.sources.classify import classify_cn
from pyquantdata.sources.tdx import bar_to_row, info_to_row, year_calendar_rows
class _FakeInfo:
def __init__(self, symbol, code="600000", name="浦发银行", volunit=100):
self.symbol = symbol
self.code = code
self.name = name
self.market = 1
self.volunit = volunit
self.decimal_point = 2
self.pre_close = 10.0
self.kind = "A股"
class _FakeBar:
def __init__(self, symbol, y, m, d, close=10.0, vol=100.0, amount=1000.0):
self.symbol = symbol
self.market = 1
self.code = "600000"
self.open = 9.9
self.high = 10.2
self.low = 9.8
self.close = close
self.vol = vol
self.amount = amount
self.year = y
self.month = m
self.day = d
self.hour = 0
self.minute = 0
class _FakeTdx:
def __init__(self):
self.universe_calls = []
self.bar_calls = []
async def get_universe(self, market):
self.universe_calls.append(market)
if market == "sh":
return [_FakeInfo("sh600000"), _FakeInfo("sh000001")]
if market == "sz":
return [_FakeInfo("sz000001")]
return [_FakeInfo("bj430047")]
async def get_bars(self, symbols, period, *, adjust=None, start_date=None,
end_date=None, **kw):
self.bar_calls.append((tuple(symbols) if isinstance(symbols, (list, tuple)) else (symbols,), start_date, end_date))
out = []
for sym in symbols if isinstance(symbols, (list, tuple)) else [symbols]:
for d in (1, 2):
out.append(_FakeBar(sym, 2026, 9, d))
return out
def test_info_to_row():
row = info_to_row(_FakeInfo("sh600000"))
assert row.symbol == "sh600000"
assert row.name == "浦发银行"
assert row.category == "stock"
assert row.exchange == "sh"
assert row.currency == "CNY"
assert row.lot_size == 100
def test_bar_to_row():
row = bar_to_row(_FakeBar("sh600000", 2026, 9, 1, close=10.5, vol=500, amount=5200))
assert (row.date, row.close, row.volume, row.amount) == (date(2026, 9, 1), 10.5, 500, 5200)
def test_year_calendar_rows_full_year():
rows = year_calendar_rows(2026, {date(2026, 1, 1)})
assert len(rows) == 365
assert rows[0].is_open is True
assert rows[1].is_open is False
def test_year_calendar_rows_leap_year():
assert len(year_calendar_rows(2024, set())) == 366
def test_adapter_requires_open():
adapter = TdxAdapter(client=None)
with pytest.raises(RuntimeError, match="open"):
asyncio.run(adapter.fetch_securities("cn"))
def test_adapter_securities_dedup_and_all_exchanges():
fake = _FakeTdx()
adapter = TdxAdapter(client=fake) # 注入:不 open 也不拥有
rows = asyncio.run(adapter.fetch_securities("cn"))
symbols = [r.symbol for r in rows]
assert symbols == ["sh600000", "sh000001", "sz000001", "bj430047"]
assert fake.universe_calls == ["sh", "sz", "bj"]
def test_adapter_non_cn_market_empty():
adapter = TdxAdapter(client=_FakeTdx())
assert asyncio.run(adapter.fetch_securities("us")) == []
assert asyncio.run(adapter.fetch_calendar("us", 2026)) == []
def test_adapter_bars_mapping_and_range():
fake = _FakeTdx()
adapter = TdxAdapter(client=fake)
rows = asyncio.run(
adapter.fetch_bars_1d("cn", ["sh600000"], date(2026, 9, 1), date(2026, 9, 2))
)
assert len(rows) == 2
assert all(r.symbol == "sh600000" for r in rows)
# start/end 透传为 datetime
called_syms, start, end = fake.bar_calls[0]
assert called_syms == ("sh600000",)
assert (start.year, start.month, start.day) == (2026, 9, 1)
assert (end.year, end.month, end.day) == (2026, 9, 2)
def test_adapter_bars_empty_symbols():
adapter = TdxAdapter(client=_FakeTdx())
assert asyncio.run(adapter.fetch_bars_1d("cn", [], None, None)) == []
def test_adapter_calendar_from_index_bars():
class _IdxTdx(_FakeTdx):
async def get_bars(self, symbols, period, **kw):
return [_FakeBar("sh000001", 2026, 9, d) for d in (1, 2, 7, 8)]
adapter = TdxAdapter(client=_IdxTdx())
rows = asyncio.run(adapter.fetch_calendar("cn", 2026))
opens = [r for r in rows if r.is_open]
assert [(r.trade_date.day) for r in opens] == [1, 2, 7, 8]
assert len(rows) == 365 # 全年日历
def test_adapter_close_noop_without_ownership():
adapter = TdxAdapter(client=_FakeTdx())
asyncio.run(adapter.close()) # 注入 client:不 close 外部对象
def test_adapter_open_creates_client(monkeypatch):
"""默认构造:open() 自建 TdxData 并 start(本测注入替身)。"""
import pyquantdata.sources.tdx as tdx_mod
started = {"n": 0}
class _StubTdxData:
async def start(self):
started["n"] += 1
async def close(self):
started["n"] -= 1
monkeypatch.setattr(tdx_mod, "TdxData", _StubTdxData)
adapter = TdxAdapter()
asyncio.run(adapter.open())
assert started["n"] == 1
asyncio.run(adapter.close())
assert started["n"] == 0