Files
openbot d37c697bbe M1 最小闭环:CLI + FastAPI 网关 + SQL 闸 + 100% 覆盖率验收链
- CLI:init/serve/update/query/export/status(typer + rich),单写者锁三元判活
- 存储:DuckDB schema migrations(14 表 + 8 联邦视图,语句元组逐条执行)
- ETL:A股证券/日历/日K,整区替换幂等;日历全年补全(calendar 语义唯一)
- 网关:POST /v1/query(线程池 + interrupt 超时 504)、/v1/export、meta/stats/state、healthz/ready
- 安全:SQL 闸 sqlglot fail-closed(文件/网络表函数默认拒)、非 loopback 空 token 拒启、导出路径双判
- 验收:254 测试全绿、1265 语句 100% 覆盖、[CI] 真起服验收链、smoke_m1.py 交付自检
- 文档:设计文档 v0.7(圆桌 45 条采纳 + 3 条实施校准),README 重写
- 发布:release.yml Trusted Publishing(OIDC 免密钥),action refs 已核验
2026-09-24 09:06:47 +08:00

236 lines
9.7 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""M1 交付自检冒烟脚本([CI] 验收链的可复用副本 + 每周 smoke workflow 素材)。
设计 §14 白名单:uvicorn 真起服 / socket 层不计入覆盖率,替代验证 = 本脚本。
与 `.github/workflows/ci.yml` 的验收链同源,可本地或 CI 直接跑:
python scripts/smoke_m1.py # 用临时 dbpath,自建自销
python scripts/smoke_m1.py -d /path/db # 用已有库
python scripts/smoke_m1.py --port 18766
覆盖:init --skip-history + 迷你 seed → 真 serve(真 socket)→
/healthz、/v1/ready、POST /v1/query(json + arrow)、POST /v1/export(csv 落盘 + 行数)、
SQL 闸 fail-closed 400 → 关闭 → 端口释放。任一步失败非零退出。
"""
from __future__ import annotations
import argparse
import base64
import csv
import io
import json
import os
import shutil
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
from pathlib import Path
REPO = Path(__file__).resolve().parent.parent
FIXTURE = REPO / "tests" / "fixtures" / "ci_seed_1d.parquet"
PY = REPO / ".venv" / "Scripts" / "python.exe"
if not PY.exists(): # 非 venv 环境(Linux CI 用 uv run 时走 sys.executable)
PY = Path(sys.executable)
FAILED: list[str] = []
# 本机常见坑:环境里挂着 `http_proxy=http://127.0.0.1:xxxxx`(指向已死的代理),
# urllib 默认走它 → 连 127.0.0.1 也超时,误判成「服务没起来」。
# 冒烟目标是本机 loopback,必须显式 bypass。
_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
def check(label: str, ok: bool, detail: str = "") -> None:
mark = "PASS" if ok else "FAIL"
print(f" [{mark}] {label}" + (f" — {detail}" if detail else ""), flush=True)
if not ok:
FAILED.append(label)
def http(base: str, path: str, *, data: dict | None = None, method: str = "GET",
timeout: float = 20.0) -> tuple[int, bytes]:
req = urllib.request.Request(base + path, method=method)
body = None
if data is not None:
req.add_header("Content-Type", "application/json")
body = json.dumps(data).encode()
with _OPENER.open(req, data=body, timeout=timeout) as r:
return r.status, r.read()
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(description="M1 交付自检冒烟")
ap.add_argument("-d", "--dbpath", default=None, help="库目录(缺省=临时目录,跑完清理)")
ap.add_argument("--port", type=int, default=18766)
args = ap.parse_args(argv)
tmp_root: Path | None = None
if args.dbpath:
dbpath = Path(args.dbpath).resolve()
else:
tmp_root = Path(tempfile.mkdtemp(prefix="pyqd_smoke_"))
dbpath = tmp_root / "db"
base = f"http://127.0.0.1:{args.port}"
proc: subprocess.Popen | None = None
try:
# ---------- 1. init --skip-history + 迷你 seed ----------
print("[1/4] init --skip-history + 迷你 seed")
r = subprocess.run(
[str(PY), "-m", "pyquantdata.cli", "init", "-d", str(dbpath),
"--skip-history", "--seed", str(FIXTURE)],
capture_output=True, text=True, cwd=str(REPO),
)
check("init 退出码 0", r.returncode == 0, r.stdout.strip().splitlines()[-1:] and
r.stdout.strip().splitlines()[-1] or r.stderr[:200])
check("init 自检输出含 bars_1d rows=20", "bars_1d rows=20" in r.stdout)
# ---------- 2. 真 serve(真 socket) ----------
print("[2/4] 起真 serve(真 uvicorn + 真端口)")
log = tempfile.NamedTemporaryFile(prefix="pyqd_smoke_srv_", suffix=".log",
delete=False)
proc = subprocess.Popen(
[str(PY), "-m", "pyquantdata.cli", "serve", "-d", str(dbpath),
"--port", str(args.port)],
stdout=log, stderr=subprocess.STDOUT, cwd=str(REPO),
)
up = False
for _ in range(60): # 最多等 30s
if proc.poll() is not None:
break
try:
http(base, "/healthz", timeout=2)
up = True
break
except Exception:
time.sleep(0.5)
check("serve 起服可连(真端口)", up,
"" if up else f"进程退出码={proc.poll()} 日志={Path(log.name).read_text()[-300:]}")
if up:
# ---------- 3. 端点面 ----------
print("[3/4] 端点面")
s, b = http(base, "/healthz")
check("/healthz 200", s == 200, b.decode()[:80])
s, b = http(base, "/v1/ready")
ready = json.loads(b)
check("/v1/ready ready=true", s == 200 and ready.get("ready") is True,
f"schema_version={ready.get('checks', {}).get('schema_version')}")
s, b = http(base, "/v1/query", method="POST",
data={"sql": "SELECT count(*) AS n FROM cn_stock_1d"})
q = json.loads(b)
check("POST /v1/query 日K row_count=1 且 n=20",
s == 200 and q.get("row_count") == 1 and q["rows"][0][0] == 20,
f"n={q.get('rows')}")
# arrow IPC 解码路径(附录 A:pandas/polars 消费端)
s, b = http(base, "/v1/query", method="POST",
data={"sql": "SELECT date, close FROM cn_stock_1d ORDER BY date",
"fmt": "arrow"})
arrow_ok = False
n_rows = -1
if s == 200:
try:
import pyarrow as pa
tbl = pa.ipc.open_stream(
pa.BufferReader(base64.b64decode(json.loads(b)["b64_arrow"]))
).read_all()
n_rows = tbl.num_rows
arrow_ok = n_rows == 20
except Exception as e: # noqa: BLE001
arrow_ok = False
n_rows = repr(e)
check("POST /v1/query fmt=arrow 可解出 20 行", arrow_ok, f"rows={n_rows}")
# 恶意查询必须被 SQL 闸拒(fail-closed,§8.3)
evil = "SELECT read_parquet('/etc/passwd')"
code = -1
try:
code, _ = http(base, "/v1/query", method="POST", data={"sql": evil})
except urllib.error.HTTPError as e:
code = e.code
check("SQL 闸拒绝 read_parquet(400)", code == 400, f"status={code}")
# ---------- 4. export csv(真落盘 exports/{job_id}/) ----------
print("[4/4] export csv")
s, b = http(base, "/v1/export", method="POST", data={
"sql": "SELECT symbol, date, close FROM cn_stock_1d ORDER BY date",
"fmt": "csv", "filename": "smoke_daily.csv",
})
lines = b.decode("utf-8").strip().splitlines() if s == 200 else []
header = [c.strip('"') for c in lines[0].split(",")] if lines else []
check("export csv 表头+20 行", s == 200 and len(lines) == 21
and header == ["symbol", "date", "close"],
f"status={s} lines={len(lines)} header={header}")
s, b = http(base, "/v1/query", method="POST", data={
"sql": "SELECT job_id, rows, state FROM export_jobs"})
job = json.loads(b)
check("export_jobs 落库 rows=20 state=done",
s == 200 and job["rows"][0][1] == 20 and job["rows"][0][2] == "done",
str(job.get("rows")))
# 落盘校验(CSV 内容语义,不受引号风格影响)
on_disk = list((dbpath / "exports").glob("*/smoke_daily.csv"))
disk_ok = False
if on_disk:
with on_disk[0].open(encoding="utf-8", newline="") as fh:
rows = list(csv.reader(fh))
disk_ok = len(rows) == 21 and rows[0] == ["symbol", "date", "close"]
check("export 落盘 exports/{job_id}/ 内容正确", disk_ok,
str(on_disk[0].relative_to(dbpath)) if on_disk else "未找到文件")
# CLI 只读直查(需先释放 serve 的写连接)
print("[5/5] CLI 只读直查(停服后)")
else:
print("[3/4] 跳过(服务未起)")
print("[4/4] 跳过(服务未起)")
# ---------- 关闭 + 端口释放 ----------
if proc is not None and proc.poll() is None:
proc.terminate()
try:
proc.wait(timeout=15)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=10)
freed = False
for _ in range(20):
try:
http(base, "/healthz", timeout=1)
time.sleep(0.3)
except Exception:
freed = True
break
check("停服后端口已释放", freed)
if up:
r = subprocess.run(
[str(PY), "-m", "pyquantdata.cli", "query", "-d", str(dbpath),
"--fmt", "json", "SELECT count(*) AS n FROM cn_stock_1d"],
capture_output=True, text=True, cwd=str(REPO),
)
check("CLI 只读直查 --fmt json 正常",
r.returncode == 0 and '"row_count": 1' in r.stdout,
(r.stdout or r.stderr)[:150])
# ---------- 汇总 ----------
print()
if FAILED:
print(f"SMOKE FAILED:{len(FAILED)} 项 —— " + "; ".join(FAILED))
return 1
print("SMOKE OK:M1 全链路(真起服 + 真端口 + 全端点)通过")
return 0
finally:
if tmp_root is not None:
shutil.rmtree(tmp_root, ignore_errors=True)
if __name__ == "__main__":
sys.exit(main())