Files
pyquantdata/tests/test_app.py
T
openbot d37c697bbe M1 最小闭环:CLI + FastAPI 网关 + SQL 闸 + 100% 覆盖率验收链
- CLI:init/serve/update/query/export/status(typer + rich),单写者锁三元判活
- 存储:DuckDB schema migrations(14 表 + 8 联邦视图,语句元组逐条执行)
- ETL:A股证券/日历/日K,整区替换幂等;日历全年补全(calendar 语义唯一)
- 网关:POST /v1/query(线程池 + interrupt 超时 504)、/v1/export、meta/stats/state、healthz/ready
- 安全:SQL 闸 sqlglot fail-closed(文件/网络表函数默认拒)、非 loopback 空 token 拒启、导出路径双判
- 验收:254 测试全绿、1265 语句 100% 覆盖、[CI] 真起服验收链、smoke_m1.py 交付自检
- 文档:设计文档 v0.7(圆桌 45 条采纳 + 3 条实施校准),README 重写
- 发布:release.yml Trusted Publishing(OIDC 免密钥),action refs 已核验
2026-09-24 09:06:47 +08:00

351 lines
12 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""FastAPI 全端点(TestClient):query/export/meta/stats/state/ready/token。"""
from __future__ import annotations
import base64
import threading
import time
import duckdb
import pyarrow as pa
import pytest
from pyquantdata.config import default_config
from pyquantdata.etl import import_seed
from pyquantdata.serve.app import AppContext, ResultTooLargeError, create_app
from pyquantdata.store.db import configure_query_conn
@pytest.fixture
def seeded(catalog, dbpath, mini_seed):
import_seed(catalog, mini_seed)
catalog.execute(
"INSERT INTO calendar VALUES ('cn', CURRENT_DATE, TRUE, 'test'), "
"('us', CURRENT_DATE, FALSE, 'weekend-test')"
)
cursor = catalog.cursor()
configure_query_conn(cursor)
ctx = AppContext(dbpath=dbpath, config=default_config(), cursor=cursor, schema_version=1)
return catalog, ctx
@pytest.fixture
def client(seeded):
from fastapi.testclient import TestClient
_catalog, ctx = seeded
with TestClient(create_app(ctx)) as c:
yield c
@pytest.fixture
def client_429(catalog, dbpath, mini_seed):
import_seed(catalog, mini_seed)
cfg = default_config()
cfg.sql.max_concurrent = 1
cfg.sql.acquire_timeout_s = 0.2
cursor = catalog.cursor()
configure_query_conn(cursor)
ctx = AppContext(dbpath=dbpath, config=cfg, cursor=cursor, schema_version=1)
from fastapi.testclient import TestClient
with TestClient(create_app(ctx)) as c:
yield c
# 合法表上的超慢查询:cn_stock_1d 自笛卡尔积(10^10 行级)→ 必被 timeout 中断。
# 不用 range() 表函数:SQL 闸按设计拒绝 FROM 位置的表函数(§8.3 fail-closed)。
SLOW_SQL = "SELECT count(*) FROM " + " CROSS JOIN ".join(
f"cn_stock_1d t{i}" for i in range(10)
)
def test_healthz(client):
assert client.get("/healthz").json() == {"status": "ok"}
def test_ready_ok(client):
r = client.get("/v1/ready")
assert r.status_code == 200
body = r.json()
assert body["ready"] is True
assert body["checks"]["schema_version"] == 1
assert body["checks"]["disk_free_pct"] is not None
def test_ready_503_when_not_migrated(catalog, dbpath):
from fastapi.testclient import TestClient
cursor = catalog.cursor()
ctx = AppContext(dbpath=dbpath, config=default_config(), cursor=cursor, schema_version=0)
with TestClient(create_app(ctx)) as c:
r = c.get("/v1/ready")
assert r.status_code == 503
assert r.json()["ready"] is False
def test_query_json_basics(client):
r = client.post("/v1/query", json={"sql": "SELECT date, close FROM cn_stock_1d ORDER BY date"})
assert r.status_code == 200
body = r.json()
assert body["columns"] == ["date", "close"]
assert body["row_count"] == 10
assert body["truncated"] is False
assert body["elapsed_ms"] >= 0
assert body["rows"][0][0] == "2026-09-01"
assert isinstance(body["rows"][0][1], float)
def test_query_arrow_stream_b64(client):
r = client.post("/v1/query", json={
"sql": "SELECT symbol, date, close FROM cn_stock_1d ORDER BY date", "fmt": "arrow",
})
assert r.status_code == 200
b64 = r.json()["b64_arrow"]
tbl = pa.ipc.open_stream(pa.BufferReader(base64.b64decode(b64))).read_all()
assert tbl.schema.names == ["symbol", "date", "close"]
assert tbl.num_rows == 10
assert tbl.column("symbol")[0].as_py() == "sh600000"
def test_query_fmt_invalid(client):
r = client.post("/v1/query", json={"sql": "SELECT 1", "fmt": "feather"})
assert r.status_code == 422
def test_query_params_placeholder(client):
r = client.post("/v1/query", json={
"sql": "SELECT close FROM cn_stock_1d WHERE symbol = ? AND date = ?",
"params": ["sh600000", "2026-09-03"],
})
assert r.status_code == 200
assert r.json()["row_count"] == 1
def test_query_max_rows_truncates(client):
r = client.post("/v1/query", json={"sql": "SELECT * FROM cn_stock_1d", "max_rows": 3})
assert r.status_code == 200
body = r.json()
assert body["row_count"] == 3
assert body["truncated"] is True
def test_query_max_rows_clamped_to_config(client):
r = client.post("/v1/query", json={"sql": "SELECT * FROM cn_stock_1d", "max_rows": 10**9})
assert r.status_code == 200 # clamp 到 config.max_rows,不报错
def test_query_rejects_forbidden_sql(client):
r = client.post("/v1/query", json={"sql": "SELECT read_csv('/etc/passwd')"})
assert r.status_code == 400
assert "read_csv" in r.json()["detail"]
def test_query_execution_error_400(client):
r = client.post("/v1/query", json={"sql": "SELECT * FROM table_not_here"})
assert r.status_code == 400
assert "执行失败" in r.json()["detail"]
def test_query_timeout_interrupts(client):
started = time.monotonic()
r = client.post("/v1/query", json={"sql": SLOW_SQL, "timeout_ms": 200})
elapsed = time.monotonic() - started
assert r.status_code == 504
assert elapsed < 5 # interrupt 真中断,不等查询跑完
# 连接复用:interrupt 后仍可查询
r2 = client.post("/v1/query", json={"sql": "SELECT 1"})
assert r2.status_code == 200
def test_query_interrupt_exception_maps_504(client, monkeypatch):
import pyquantdata.serve.app as app_mod
def boom(*a, **k):
raise duckdb.InterruptException("INTERRUPT Error: Interrupted!")
monkeypatch.setattr(app_mod, "execute_query_blocking", boom)
r = client.post("/v1/query", json={"sql": "SELECT 1"})
assert r.status_code == 504
def test_query_result_too_large_maps_413(client, monkeypatch):
import pyquantdata.serve.app as app_mod
def too_big(*a, **k):
raise ResultTooLargeError("结果过大:请走 export")
monkeypatch.setattr(app_mod, "execute_query_blocking", too_big)
r = client.post("/v1/query", json={"sql": "SELECT 1"})
assert r.status_code == 413
def test_query_429_when_slot_busy(client_429):
results: dict[str, int] = {}
def slow():
r = client_429.post("/v1/query", json={"sql": SLOW_SQL, "timeout_ms": 4000})
results["slow"] = r.status_code
t = threading.Thread(target=slow)
t.start()
time.sleep(0.5) # slow 已占用唯一信号量槽
r2 = client_429.post("/v1/query", json={"sql": "SELECT 1"})
t.join(timeout=20)
assert r2.status_code == 429
assert "并发" in r2.json()["detail"]
assert results["slow"] in (200, 504) # 大查询最终超时中断
def test_export_csv_roundtrip(client, dbpath):
r = client.post("/v1/export", json={
"sql": "SELECT symbol, date, close FROM cn_stock_1d ORDER BY date",
"fmt": "csv", "filename": "daily.csv",
})
assert r.status_code == 200
assert r.headers["content-type"].startswith("text/csv")
lines = r.content.decode("utf-8").strip().splitlines()
# pyarrow 默认 RFC4180 加引号;解析后断言列名与行数
assert [c.strip('"') for c in lines[0].split(",")] == ["symbol", "date", "close"]
assert len(lines) == 11 # header + 10
# exports/{job_id}/ 落盘 + job 记录可查
jobs = client.post("/v1/query", json={"sql": "SELECT job_id FROM export_jobs"}).json()
job_id = jobs["rows"][0][0]
job = client.get(f"/v1/export/{job_id}").json()
assert job["state"] == "done"
assert job["rows"] == 10
assert job["path"].startswith("exports")
assert (dbpath / job["path"]).exists()
def test_export_parquet_roundtrip(client, dbpath):
import io
import pyarrow.parquet as paparquet
r = client.post("/v1/export", json={"sql": "SELECT * FROM cn_stock_1d", "fmt": "parquet"})
assert r.status_code == 200
tbl = paparquet.read_table(io.BytesIO(r.content))
assert tbl.num_rows == 10
def test_export_pandas_rejected_with_hint(client):
r = client.post("/v1/export", json={"sql": "SELECT 1", "fmt": "pandas"})
assert r.status_code == 422
assert "/v1/query" in r.json()["detail"]
def test_export_path_escape_rejected(client):
r = client.post("/v1/export", json={
"sql": "SELECT 1", "fmt": "csv", "filename": "../evil.csv",
})
assert r.status_code == 400
assert "非法" in r.json()["detail"]
def test_export_job_404(client):
assert client.get("/v1/export/nope").status_code == 404
def test_meta_views_and_stats(client):
meta = client.get("/v1/meta").json()
assert meta["schema_version"] == 1
names = {v["name"] for v in meta["views"]}
assert "cn_stock_1d" in names
v = next(v for v in meta["views"] if v["name"] == "cn_stock_1d")
assert v["row_count"] == 10
assert v["date_min"] == "2026-09-01"
assert v["sample_symbols"] == ["sh600000"]
stats = client.get("/v1/stats").json()
assert stats["checkpoints"] == []
assert stats["disk"]["bytes"] > 0
assert stats["http"]["ws_conns"] == 0
def test_meta_handles_special_views(catalog, seeded):
_cat, ctx = seeded
from fastapi.testclient import TestClient
catalog.execute("CREATE OR REPLACE VIEW meta_special AS SELECT 42 AS v")
with TestClient(create_app(ctx)) as c:
meta = c.get("/v1/meta").json()
special = next(v for v in meta["views"] if v["name"] == "meta_special")
assert special["columns"] == ["v"]
assert "date_min" not in special
assert "sample_symbols" not in special
def test_state_endpoint(client):
r = client.get("/v1/state/cn")
assert r.status_code == 200
body = r.json()
assert body["market"] == "cn"
assert body["state"] in {"CLOSED", "PRE_OPEN_AUCTION", "OPEN", "LUNCH_BREAK"}
assert body["session_tz"] == "Asia/Shanghai"
assert body["ts"].endswith("Z") # ISO8601 UTC 带 Z(§8.2)
def test_state_no_calendar_data_is_closed(client):
body = client.get("/v1/state/hk").json()
assert body["state"] == "CLOSED"
assert body["reason"] == "no_calendar_data"
def test_state_unknown_market_404(client):
assert client.get("/v1/state/uk").status_code == 404
def test_ready_disk_free_branch_none(catalog, tmp_path):
"""dbpath 不存在 → free_pct None 分支(只读探针仍 ready)。"""
from fastapi.testclient import TestClient
cursor = catalog.cursor()
cursor.execute("SELECT 1")
ctx = AppContext(
dbpath=tmp_path / "missing", config=default_config(),
cursor=cursor, schema_version=1,
)
with TestClient(create_app(ctx)) as c:
assert c.get("/v1/ready").status_code == 200
def test_stats_disk_oserror_branch(catalog, dbpath, monkeypatch):
from fastapi.testclient import TestClient
cursor = catalog.cursor()
cursor.execute("SELECT 1")
ctx = AppContext(dbpath=dbpath, config=default_config(), cursor=cursor, schema_version=1)
import pyquantdata.serve.app as app_mod
def boom(_):
raise OSError("disk gone")
monkeypatch.setattr(app_mod.psutil, "disk_usage", boom)
with TestClient(create_app(ctx)) as c:
body = c.get("/v1/stats").json()
assert body["disk"]["free_pct"] is None
def test_token_auth(catalog, dbpath, mini_seed):
import_seed(catalog, mini_seed)
cfg = default_config()
cfg.http.token = "abcd1234"
cursor = catalog.cursor()
configure_query_conn(cursor)
from fastapi.testclient import TestClient
with TestClient(create_app(AppContext(dbpath=dbpath, config=cfg, cursor=cursor, schema_version=1))) as c:
assert c.post("/v1/query", json={"sql": "SELECT 1"}).status_code == 401
assert c.post(
"/v1/query", json={"sql": "SELECT 1"},
headers={"Authorization": "Bearer wrong"},
).status_code == 401
ok = c.post(
"/v1/query", json={"sql": "SELECT 1"},
headers={"Authorization": "Bearer abcd1234"},
)
assert ok.status_code == 200
# 探针不设防
assert c.get("/healthz").status_code == 200