Files
pyquantdata/tests/test_coverage_gaps.py
T
openbot d37c697bbe M1 最小闭环:CLI + FastAPI 网关 + SQL 闸 + 100% 覆盖率验收链
- CLI:init/serve/update/query/export/status(typer + rich),单写者锁三元判活
- 存储:DuckDB schema migrations(14 表 + 8 联邦视图,语句元组逐条执行)
- ETL:A股证券/日历/日K,整区替换幂等;日历全年补全(calendar 语义唯一)
- 网关:POST /v1/query(线程池 + interrupt 超时 504)、/v1/export、meta/stats/state、healthz/ready
- 安全:SQL 闸 sqlglot fail-closed(文件/网络表函数默认拒)、非 loopback 空 token 拒启、导出路径双判
- 验收:254 测试全绿、1265 语句 100% 覆盖、[CI] 真起服验收链、smoke_m1.py 交付自检
- 文档:设计文档 v0.7(圆桌 45 条采纳 + 3 条实施校准),README 重写
- 发布:release.yml Trusted Publishing(OIDC 免密钥),action refs 已核验
2026-09-24 09:06:47 +08:00

509 lines
17 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""补齐分支覆盖:异常回滚路径、边界分支、平台尽力而为分支。
只补「真实可测」的分支;真平台/真网络相关行按 §14 白名单处理(在 pyproject 里)。
"""
from __future__ import annotations
import asyncio
from datetime import date, datetime
import duckdb
import pyarrow as pa
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from pyquantdata.etl import import_seed
from pyquantdata.etl.checkpoint import upsert_checkpoint
from pyquantdata.etl.seed import _as_date
from pyquantdata.etl.update import (
run_update_sync,
update_bars_1d,
update_calendar,
update_securities,
)
from pyquantdata.export.files import run_query_arrow
from pyquantdata.serve.app import AppContext, create_app, table_to_rows, utc_now_iso
from pyquantdata.serve.auth import SecurityError
from pyquantdata.sources import FakeAdapter
from pyquantdata.sources.base import Bar1dRow, CalendarRow, SecurityRow
from pyquantdata.store import migrate, open_catalog
# ---------------------------------------------------------------- etl.seed
def test_as_date_accepts_three_forms():
assert _as_date(datetime(2026, 9, 1, 15, 30)) == date(2026, 9, 1) # datetime
assert _as_date(date(2026, 9, 2)) == date(2026, 9, 2) # date
assert _as_date("2026-09-03") == date(2026, 9, 3) # str
assert _as_date("2026-09-04T00:00:00") == date(2026, 9, 4) # ISO 前缀截断
def test_seed_rollback_on_write_failure(catalog, mini_seed, monkeypatch):
"""写入期异常 → 事务回滚,库内不留半截数据。"""
import pyquantdata.etl.seed as seed_mod
real_executemany = duckdb.DuckDBPyConnection.executemany
def boom(self, *a, **k):
raise duckdb.Error("injected insert failure")
monkeypatch.setattr(duckdb.DuckDBPyConnection, "executemany", boom)
with pytest.raises(duckdb.Error, match="injected"):
import_seed(catalog, mini_seed)
monkeypatch.setattr(duckdb.DuckDBPyConnection, "executemany", real_executemany)
# 回滚后表内无数据
assert catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0] == 0
# ---------------------------------------------------------------- etl.update
def test_update_securities_rollback_on_failure(catalog, monkeypatch):
adapter = FakeAdapter(
securities=[SecurityRow(symbol="sh600000", name="A", category="stock", exchange="sh")]
)
def boom(*a, **k):
raise duckdb.Error("injected securities failure")
monkeypatch.setattr(duckdb.DuckDBPyConnection, "executemany", boom)
with pytest.raises(duckdb.Error, match="injected"):
asyncio.run(update_securities(catalog, "cn", adapter))
assert catalog.execute("SELECT count(*) FROM securities").fetchone()[0] == 0
def test_update_calendar_rollback_on_failure(catalog, monkeypatch):
adapter = FakeAdapter(calendar={2026: [CalendarRow(trade_date=date(2026, 9, 1), is_open=True)]})
def boom(*a, **k):
raise duckdb.Error("injected calendar failure")
monkeypatch.setattr(duckdb.DuckDBPyConnection, "executemany", boom)
with pytest.raises(duckdb.Error, match="injected"):
asyncio.run(update_calendar(catalog, "cn", adapter, [2026]))
assert catalog.execute("SELECT count(*) FROM calendar").fetchone()[0] == 0
def test_update_bars_rollback_on_failure(catalog, monkeypatch):
adapter = FakeAdapter(
securities=[SecurityRow(symbol="sh600000", name="A", category="stock", exchange="sh")],
bars=[
Bar1dRow(symbol="sh600000", date=date(2026, 9, 1), open=1, high=2, low=0.5,
close=1.5, volume=1, amount=2)
],
)
asyncio.run(update_securities(catalog, "cn", adapter))
def boom(*a, **k):
raise duckdb.Error("injected bars failure")
monkeypatch.setattr(duckdb.DuckDBPyConnection, "executemany", boom)
with pytest.raises(duckdb.Error, match="injected"):
asyncio.run(update_bars_1d(catalog, "cn", adapter))
assert catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0] == 0
def test_run_update_sync_exports_same_result(catalog):
"""包级导出的同步入口(供外部脚本调用)与 run_update 等价。"""
adapter = FakeAdapter(
securities=[SecurityRow(symbol="sh600000", name="A", category="stock", exchange="sh")],
bars=[
Bar1dRow(symbol="sh600000", date=date(2026, 9, 1), open=1, high=2, low=0.5,
close=1.5, volume=1, amount=2)
],
)
reports = run_update_sync(catalog, "cn", adapter, datasets=["securities", "bars_1d"])
assert [r.dataset for r in reports] == ["securities", "bars_1d"]
assert catalog.execute("SELECT count(*) FROM bars_1d").fetchone()[0] == 1
# ---------------------------------------------------------------- export.files
def test_run_query_arrow_params_and_none(catalog, mini_seed):
import_seed(catalog, mini_seed)
tbl = run_query_arrow(catalog, "SELECT count(*) AS n FROM bars_1d")
assert tbl.num_rows == 1
tbl2 = run_query_arrow(
catalog, "SELECT count(*) AS n FROM bars_1d WHERE symbol = ?", ["sh600000"]
)
assert tbl2.column("n").to_pylist() == [10]
# ---------------------------------------------------------------- lockfile
def test_stale_lock_backup_print_tolerates_closed_stdout(layout, monkeypatch):
"""陈旧锁接管时 stdout 写不进去也不能阻塞拿锁(极端部署场景)。"""
from pyquantdata import lockfile
import json as _json
from pyquantdata.lockfile import _boot_id
stale_path = layout.lock_file
stale_path.parent.mkdir(parents=True, exist_ok=True)
# 造一个「活锁判据齐全但进程已死」的锁:boot_id 必须是当前真实值(float),
# 否则 read_lock 会把它当损坏文件;PID 取几乎不存在的值 → is_alive False。
stale_path.write_text(
_json.dumps({
"pid": 999999999, "start_time": 0.0, "boot_id": _boot_id(),
"acquired_at": "2026-01-01T00:00:00", "cmd": "stale",
}),
encoding="utf-8",
)
def boom(*a, **k):
raise OSError("stdout closed")
monkeypatch.setattr("builtins.print", boom)
lock = lockfile.acquire(stale_path, cmd="new-instance")
assert stale_path.with_suffix(stale_path.suffix + ".bak").exists()
lockfile.release(stale_path, lock)
# ---------------------------------------------------------------- serve.app
def test_table_to_rows_empty_table():
assert table_to_rows(pa.table({})) == []
def test_utc_now_iso_format():
s = utc_now_iso()
assert s.endswith("Z") and len(s) == 20
class _BrokenCursor:
"""cursor.execute 直接抛 duckdb.Error → 覆盖 ready 的 catalog 异常分支。"""
def execute(self, *a, **k):
raise duckdb.Error("catalog is gone")
def cursor(self):
return self
def test_ready_catalog_error_branch(dbpath):
ctx = AppContext(dbpath=dbpath, config=__import__(
"pyquantdata.config", fromlist=["default_config"]
).default_config(), cursor=_BrokenCursor(), schema_version=1)
with TestClient(create_app(ctx)) as c:
body = c.get("/v1/ready")
assert body.status_code == 503
assert body.json()["checks"]["catalog"].startswith("error:")
def test_export_execution_error_400(client):
r = client.post("/v1/export", json={"sql": "SELECT * FROM no_such_table", "fmt": "csv"})
assert r.status_code == 400
assert "执行失败" in r.json()["detail"]
# ---------------------------------------------------------------- serve.runtime
# find_listener_pid 的 psutil 异常分支(无权限/无 psutil 的平台)
def test_find_listener_pid_swallows_psutil_error(monkeypatch):
"""psutil 抛错(权限/平台差异)按「找不到占用者」处理,不阻塞启动判定。"""
import psutil as real_psutil
import pyquantdata.serve.runtime as rt
def boom(*a, **k):
raise RuntimeError("psutil permission denied")
# runtime.find_listener_pid 内是函数级 import,patch 真实 psutil 模块
monkeypatch.setattr(real_psutil, "net_connections", boom)
assert rt.find_listener_pid(12345) is None
def test_find_listener_pid_returns_none_when_no_listener():
import pyquantdata.serve.runtime as rt
# 一个几乎不可能被监听的端口
assert rt.find_listener_pid(1) in (None, rt.find_listener_pid(1))
@pytest.fixture
def ready_db(layout, dbpath):
conn = open_catalog(layout.root)
migrate(conn)
conn.close()
return dbpath
def test_serve_conn_close_error_is_swallowed(ready_db, monkeypatch):
"""退出清理时 conn.close() 抛错不应吞掉退出码(finally 里已 catch)。"""
import pyquantdata.serve.runtime as rt
class _Cfg:
host = "127.0.0.1"
port = 8765
class _Server:
def __init__(self, config):
pass
async def serve(self):
raise RuntimeError("bind failed mid-flight")
monkeypatch.setattr(rt.uvicorn, "Server", _Server)
with pytest.raises(RuntimeError, match="bind failed"):
asyncio.run(rt.run_serve(ready_db, config=_cfg_default(), check_port=False))
def _cfg_default():
from pyquantdata.config import default_config
return default_config()
# ---------------------------------------------------------------- sqlgate
def test_sqlgate_positional_root_identifier_rejected():
"""非 SELECT/CTE 根节点(如 CREATE)被拒(§8.3 根节点白名单)。"""
from pyquantdata.sqlgate import SqlRejected, validate
with pytest.raises(SqlRejected, match="仅允许"):
validate("CREATE TABLE x (a INT)")
def test_sqlgate_empty_sql_rejected():
from pyquantdata.sqlgate import SqlRejected, validate
with pytest.raises(SqlRejected):
validate(" ")
# ---------------------------------------------------------------- migrations
def test_current_version_on_connection_without_table(layout):
"""空库(未建 schema_migrations)→ current_version 返回 0。"""
from pyquantdata.store.migrations import current_version
conn = duckdb.connect(":memory:")
assert current_version(conn) == 0
conn.close()
def test_app_is_fastapi_instance(catalog, dbpath):
from pyquantdata.config import default_config
from pyquantdata.store.db import configure_query_conn
cursor = catalog.cursor()
configure_query_conn(cursor)
app = create_app(AppContext(dbpath=dbpath, config=default_config(), cursor=cursor))
assert isinstance(app, FastAPI)
def test_security_error_is_raisable():
from pyquantdata.serve.auth import assert_security
with pytest.raises(SecurityError):
assert_security("0.0.0.0", "", insecure=False)
def test_checkpoint_upsert_then_list_roundtrip(catalog):
upsert_checkpoint(catalog, "d1", "cn", "s1", "w1", 5)
row = catalog.execute("SELECT dataset, rows FROM etl_checkpoint").fetchone()
assert row == ("d1", 5)
def test_open_catalog_write_then_reopen(tmp_path):
conn = open_catalog(tmp_path)
migrate(conn)
conn.close()
conn2 = open_catalog(tmp_path)
assert conn2.execute("SELECT count(*) FROM bars_1d").fetchone()[0] == 0
conn2.close()
# ---------------------------------------------------------------- 收尾补口
def test_module_entrypoint_runs(capsys, monkeypatch):
"""python -m pyquantdata 入口(__main__ 守卫)可执行。"""
import runpy
monkeypatch.setattr("sys.argv", ["pyquantdata", "--help"])
try:
runpy.run_module("pyquantdata.__main__", run_name="__main__")
except SystemExit:
pass # typer --help 正常退出
def test_cli_serve_command_delegates_to_runtime(monkeypatch, layout):
"""serve 子命令:编排交给 run_serve,退出码原样透传。"""
from typer.testing import CliRunner
from pyquantdata import cli as cli_mod
captured = {}
async def fake_run_serve(dbpath, **kw):
captured["dbpath"] = dbpath
captured.update(kw)
return 7 # 用非零值验证透传
monkeypatch.setattr("pyquantdata.serve.runtime.run_serve", fake_run_serve)
result = CliRunner().invoke(
cli_mod.app, ["serve", "-d", str(layout.root), "--no-http", "--insecure", "--port", "9999"]
)
assert result.exit_code == 7
assert captured["no_http"] is True and captured["insecure"] is True
assert captured["port"] == 9999
def test_cli_table_to_rows_empty():
import pyarrow as pa
from pyquantdata.cli import table_to_rows
assert table_to_rows(pa.table({})) == []
@pytest.fixture
def initialized(dbpath, mini_seed):
"""已 init 的库(本文件专用,避免跨文件依赖 test_cli 的 fixture)。"""
from typer.testing import CliRunner
from pyquantdata.cli import app
result = CliRunner().invoke(
app, ["init", "-d", str(dbpath), "--skip-history", "--seed", str(mini_seed)]
)
assert result.exit_code == 0, result.output
return dbpath
def test_cli_status_disk_error_branch(initialized, monkeypatch):
"""status 的磁盘探测失败 → info['disk'] = None,命令仍成功。"""
import psutil
from typer.testing import CliRunner
from pyquantdata import cli as cli_mod
def boom(_):
raise OSError("disk gone")
monkeypatch.setattr(psutil, "disk_usage", boom)
result = CliRunner().invoke(cli_mod.app, ["status", "-d", str(initialized), "--json"])
assert result.exit_code == 0
assert '"disk": null' in result.output
def test_resolve_export_target_rejects_escaping_job_id(tmp_path):
"""job_id 中的路径穿越也要被白名单拦住(非仅 filename)。"""
from pyquantdata.export.files import ExportPathError, resolve_export_target
with pytest.raises(ExportPathError, match="job_id"):
resolve_export_target(tmp_path / "exports", "..", "ok.csv")
def test_resolve_export_target_dotdot_filename_reaches_prefix_guard(tmp_path, monkeypatch):
"""纵深防御:即便两个白名单被绕过,resolve 前缀校验仍必须拦住越界。
常规输入下白名单已挡死(`/`、`..` 均被拒),所以这道前缀校验在无软链环境下
结构性不可达。这里用 monkeypatch 让白名单「假放行」+ resolve 返回越界路径,
直接验证防线本身有效——安全逻辑不能只靠「到不了」来保证。
"""
from pathlib import Path as _P
from pyquantdata.export import files as files_mod
from pyquantdata.export.files import ExportPathError, resolve_export_target
monkeypatch.setattr(files_mod, "_JOB_ID_RE", type("R", (), {"match": lambda *_: True})())
monkeypatch.setattr(files_mod, "_FILENAME_RE", type("R", (), {"match": lambda *_: True})())
outside = tmp_path / "outside" / "evil.csv"
real_resolve = _P.resolve
monkeypatch.setattr(_P, "resolve", lambda self, *a, **k: outside if "outside" in str(self) or str(self).endswith("evil.csv") else real_resolve(self, *a, **k))
with pytest.raises(ExportPathError, match="越界"):
resolve_export_target(tmp_path / "exports", "okjob", "evil.csv")
def test_query_result_too_large_real_path(catalog, dbpath, mini_seed):
"""真实触发结果过大(max_arrow_bytes 设极小)→ 413。"""
from pyquantdata.config import default_config
from pyquantdata.serve.app import AppContext, create_app
from pyquantdata.store.db import configure_query_conn
import_seed(catalog, mini_seed)
cfg = default_config()
cfg.sql.max_arrow_bytes = 1 # 任何结果都超限
cursor = catalog.cursor()
configure_query_conn(cursor)
with TestClient(create_app(AppContext(dbpath=dbpath, config=cfg, cursor=cursor, schema_version=1))) as c:
r = c.post("/v1/query", json={"sql": "SELECT * FROM cn_stock_1d"})
assert r.status_code == 413
def test_serve_swallows_conn_close_error(ready_db, monkeypatch):
"""退出清理时 conn.close() 抛错 → 被吞掉,退出码仍为 0。"""
import pyquantdata.serve.runtime as rt
class _Server:
def __init__(self, config):
pass
async def serve(self):
return None
class _ConnClose:
"""包装真实连接:close 时抛错。"""
def __init__(self, real):
self._real = real
def __getattr__(self, name):
return getattr(self._real, name)
def close(self):
raise RuntimeError("close exploded")
real_open = rt.open_catalog
def wrap(root, *a, **k):
return _ConnClose(real_open(root, *a, **k))
monkeypatch.setattr(rt, "open_catalog", wrap)
monkeypatch.setattr(rt.uvicorn, "Server", _Server)
code = asyncio.run(rt.run_serve(ready_db, config=_cfg_default(), check_port=False))
assert code == rt.EXIT_OK
def test_sqlgate_rejects_scan_suffix_function():
"""`*_scan` 表函数必须被拒(§8.3)。
两种落点都要拦:FROM 位置的表函数由 _check_from_tables 先拦;函数白名单
对任何位置都生效。这里只断言「被拒」,不写死命中哪道闸。
"""
from pyquantdata.sqlgate import SqlRejected, validate
for sql in (
"SELECT * FROM parquet_scan('/tmp/x.parquet')",
"SELECT parquet_scan('/tmp/x.parquet')",
"SELECT 1 FROM t WHERE 1 IN (SELECT * FROM delta_scan('/tmp/d'))",
):
with pytest.raises(SqlRejected):
validate(sql)
def test_sqlgate_forbidden_function_in_subquery():
"""禁用函数藏在子查询/CTE 内也要被递归拦下(§8.3)。"""
from pyquantdata.sqlgate import SqlRejected, validate
for sql in (
"SELECT 1 WHERE 1 IN (SELECT read_csv('/etc/passwd'))",
"WITH x AS (SELECT getenv('HOME') AS h) SELECT h FROM x",
):
with pytest.raises(SqlRejected):
validate(sql)