M1 最小闭环:CLI + FastAPI 网关 + SQL 闸 + 100% 覆盖率验收链

- CLI:init/serve/update/query/export/status(typer + rich),单写者锁三元判活
- 存储:DuckDB schema migrations(14 表 + 8 联邦视图,语句元组逐条执行)
- ETL:A股证券/日历/日K,整区替换幂等;日历全年补全(calendar 语义唯一)
- 网关:POST /v1/query(线程池 + interrupt 超时 504)、/v1/export、meta/stats/state、healthz/ready
- 安全:SQL 闸 sqlglot fail-closed(文件/网络表函数默认拒)、非 loopback 空 token 拒启、导出路径双判
- 验收:254 测试全绿、1265 语句 100% 覆盖、[CI] 真起服验收链、smoke_m1.py 交付自检
- 文档:设计文档 v0.7(圆桌 45 条采纳 + 3 条实施校准),README 重写
- 发布:release.yml Trusted Publishing(OIDC 免密钥),action refs 已核验
This commit is contained in:
openbot committed 2026-09-24 09:06:47 +08:00
commit d37c697bbe
59 files changed
+7586

No files matched your search

+3
View File
@@ -0,0 +1,3 @@
"""pyquantdata: quantdata CLI —— 单目录自包含的量化数据中台。"""
__version__ = "0.1.0"
+6
View File
@@ -0,0 +1,6 @@
"""python -m pyquantdata 入口。"""
from .cli import main
if __name__ == "__main__":
main()
+339
View File
@@ -0,0 +1,339 @@
"""quantdata CLI(设计 §6):init / serve / update / query / export / status。
约定:``-d`` 可用环境变量 QUANTDATA_HOME 兜底;输出对齐「简洁分阶段」偏好;
CLI 一次性命令可离线跑(serve --no-http 纯批处理)。
"""
from __future__ import annotations
import asyncio
import json
import sys
from datetime import date
from pathlib import Path
from typing import Optional
import typer
from rich.console import Console
from rich.table import Table
from . import __version__
from .config import AppConfig, default_config, load_config, write_config
from .etl import SeedError, import_seed
from .export.files import (
EXPORT_FORMATS,
write_export,
)
from .paths import DbPathLayout, resolve_dbpath
from .sqlgate import SqlRejected, validate
from .store import CatalogNotInitialized, migrate, open_catalog
app = typer.Typer(add_completion=False, help="quantdata:单目录自包含的量化数据中台")
console = Console()
err = Console(stderr=True)
# 容量估算表(设计 §3.4,GB):数据终态 + buffer ≥200GB ≈ 600GB 预算口径
_CAPACITY_GB = {"cn": (150, 200), "us": (80, 90), "hk": (40, 45)}
_BUFFER_GB = 200
def estimate_capacity(markets: list[str]) -> dict:
"""init --estimate 只算不写(§3.4)。"""
per = {}
lo_total = hi_total = 0
for m in markets:
lo, hi = _CAPACITY_GB.get(m, (0, 0))
per[m] = {"gb_low": lo, "gb_high": hi}
lo_total += lo
hi_total += hi
return {
"per_market_gb": per,
"data_gb": [lo_total, hi_total],
"buffer_gb": _BUFFER_GB,
"budget_gb": [lo_total + _BUFFER_GB, hi_total + _BUFFER_GB],
}
def _open_ro(dbpath: Path):
"""只读打开;未初始化时打印可操作提示并 exit(1)(所有只读命令共用)。"""
try:
return open_catalog(dbpath, read_only=True)
except CatalogNotInitialized as e:
err.print(f"[red]未初始化:[/red]{e}")
raise typer.Exit(1) from e
@app.command()
def init(
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME", help="库目录"),
markets: str = typer.Option("cn,us,hk", "--markets", help="启用市场,逗号分隔"),
seed: Optional[Path] = typer.Option(None, "--seed", help="现成数据文件(parquet/csv)导入日K"),
skip_history: bool = typer.Option(False, "--skip-history", help="跳过历史回填([CI] 层)"),
estimate: bool = typer.Option(False, "--estimate", help="只估算磁盘预算,不写库"),
) -> None:
"""建目录 + config.json + schema(migrations)+ seed 导入 + 自检。幂等:重跑=续跑。"""
dbpath = resolve_dbpath(d)
layout = DbPathLayout(dbpath)
market_list = [m.strip() for m in markets.split(",") if m.strip()]
if estimate:
console.print_json(json.dumps(estimate_capacity(market_list)))
raise typer.Exit(0)
console.print(f"[bold]quantdata init[/bold] → {dbpath}")
console.print(" [1/4] 目录结构")
layout.ensure_dirs()
config_file = layout.config_file
if config_file.exists():
cfg = load_config(config_file)
console.print(" [2/4] config.json 已存在,保留(幂等)")
else:
cfg = default_config(market_list)
write_config(cfg, config_file)
console.print(f" [2/4] config.json 生成(markets={market_list})")
conn = open_catalog(layout.root)
try:
console.print(" [3/4] schema migrations")
version = migrate(conn)
console.print(f" schema 版本 = {version}")
console.print(" [4/4] 自检")
seed_report = None
if seed is not None:
try:
seed_report = import_seed(conn, seed)
except SeedError as e:
err.print(f"[red]seed 导入失败:[/red]{e}")
raise typer.Exit(1) from e
_print_selfcheck(conn, layout, seed_report)
finally:
conn.close()
if skip_history:
console.print("[dim]--skip-history:历史回填跳过([CI] 层验收口径)[/dim]")
console.print("[green]init 完成[/green]")
def _print_selfcheck(conn, layout: DbPathLayout, seed_report) -> None:
tables = conn.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_schema='main' "
"AND table_type='BASE TABLE'"
).fetchone()[0]
views = conn.execute(
"SELECT count(*) FROM information_schema.tables WHERE table_schema='main' "
"AND table_type='VIEW'"
).fetchone()[0]
bars_rows = conn.execute("SELECT count(*) FROM bars_1d").fetchone()[0]
console.print(f" tables={tables} views={views} bars_1d rows={bars_rows}")
if seed_report is not None:
console.print(
f" seed: {seed_report.rows} 行 / {len(seed_report.symbols)} symbols "
f"[{seed_report.date_min} ~ {seed_report.date_max}]"
)
@app.command()
def serve(
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME"),
host: Optional[str] = typer.Option(None, "--host", help="监听地址(默认 config http.host)"),
port: Optional[int] = typer.Option(None, "--port", help="监听端口(默认 config http.port)"),
no_http: bool = typer.Option(False, "--no-http", help="纯 CLI 批处理,不起网关"),
insecure: bool = typer.Option(False, "--insecure", help="显式允许非 loopback 且无 token(§8.4)"),
) -> None:
"""常驻:单写者锁 + 调度 + FastAPI 网关(恒前台,Linux 交 supervisord)。"""
from .serve.runtime import run_serve
dbpath = resolve_dbpath(d)
code = asyncio.run(
run_serve(dbpath, host=host, port=port, no_http=no_http, insecure=insecure)
)
raise typer.Exit(code)
@app.command()
def update(
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME"),
dataset: str = typer.Option("all", "--dataset", help="securities,calendar,bars_1d 或 all"),
market: str = typer.Option("cn", "--market"),
symbols: Optional[str] = typer.Option(None, "--symbols", help="逗号分隔,缺省=securities 全量"),
start: Optional[str] = typer.Option(None, "--start", help="日K 起始 YYYY-MM-DD"),
end: Optional[str] = typer.Option(None, "--end", help="日K 结束 YYYY-MM-DD"),
) -> None:
"""单次更新(cron 兜底):主源 pytdxdata。幂等,重跑安全。"""
from .sources import TdxAdapter
dbpath = resolve_dbpath(d)
layout = DbPathLayout(dbpath)
conn = open_catalog(layout.root)
try:
migrate(conn)
datasets = (
["securities", "calendar", "bars_1d"] if dataset == "all" else
[x.strip() for x in dataset.split(",") if x.strip()]
)
sym_list = [s.strip() for s in symbols.split(",")] if symbols else None
adapter = TdxAdapter()
console.print(f"[bold]quantdata update[/bold] market={market} datasets={datasets}")
# 适配器 open/close 与 ETL 必须在同一事件循环(pytdxdata 纯 asyncio)
from .etl.update import run_update as _run_update
async def _run():
await adapter.open()
try:
return await _run_update(
conn, market, adapter,
datasets=datasets, symbols=sym_list,
start=date.fromisoformat(start) if start else None,
end=date.fromisoformat(end) if end else None,
)
finally:
await adapter.close()
reports = asyncio.run(_run())
for r in reports:
console.print(
f" {r.dataset}: rows={r.rows} symbols={len(r.symbols)} "
f"range=[{r.date_min} ~ {r.date_max}]"
)
console.print("[green]update 完成[/green]")
finally:
conn.close()
@app.command()
def query(
sql: str = typer.Argument(..., help="SQL(仅只读查询)"),
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME"),
fmt: str = typer.Option("table", "--fmt", help="table|json|csv|parquet|arrow"),
o: Optional[Path] = typer.Option(None, "-o", help="csv/parquet/arrow 输出文件"),
) -> None:
"""本地直查(只读连接,不开服务也可用)。"""
if fmt not in ("table", "json", "csv", "parquet", "arrow"):
err.print(f"[red]--fmt 不支持:{fmt}[/red]")
raise typer.Exit(1)
if fmt in ("csv", "parquet", "arrow") and o is None:
err.print(f"[red]--fmt {fmt} 需要 -o 输出文件[/red]")
raise typer.Exit(1)
dbpath = resolve_dbpath(d)
conn = _open_ro(dbpath)
try:
from .store.db import configure_query_conn
configure_query_conn(conn)
try:
validate(sql) # 本地 CLI 也过闸:与 HTTP 同规则(§8.3)
except SqlRejected as e:
err.print(f"[red]SQL 被拒绝:[/red]{e.reason}")
raise typer.Exit(1) from e
tbl = conn.execute(sql).arrow().read_all()
if fmt == "table":
table = Table(*tbl.schema.names)
for row in table_to_rows(tbl):
table.add_row(*[str(v) for v in row])
console.print(table)
console.print(f"[dim]{tbl.num_rows} rows[/dim]")
elif fmt == "json":
# 纯 stdout JSON(无 rich 前缀/换行噪声),便于 `| jq` 与程序消费
print(
json.dumps(
{
"columns": tbl.schema.names,
"dtypes": [str(f.type) for f in tbl.schema],
"rows": table_to_rows(tbl),
"row_count": tbl.num_rows,
},
default=str,
ensure_ascii=False,
)
)
else:
result = write_export(tbl, "csv" if fmt == "csv" else fmt, Path(o))
console.print(f"[green]已写出[/green] {result.path}({result.rows} 行 / {result.bytes_written} 字节)")
finally:
conn.close()
def table_to_rows(tbl):
if tbl.num_columns == 0:
return []
cols = [c.to_pylist() for c in tbl.columns]
return [list(row) for row in zip(*cols)]
@app.command()
def export(
sql: str = typer.Option(..., "--sql", help="导出查询"),
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME"),
fmt: str = typer.Option("csv", "--fmt", help="csv|parquet"),
o: Path = typer.Option(..., "-o", help="输出文件"),
) -> None:
"""导出落盘格式(csv/parquet);pandas/polars 走 query --fmt arrow(§9)。"""
if fmt not in EXPORT_FORMATS:
err.print(f"[red]export 仅支持 {sorted(EXPORT_FORMATS)};pandas/polars 走 query --fmt arrow[/red]")
raise typer.Exit(1)
dbpath = resolve_dbpath(d)
conn = _open_ro(dbpath)
try:
from .store.db import configure_query_conn
configure_query_conn(conn)
try:
validate(sql)
except SqlRejected as e:
err.print(f"[red]SQL 被拒绝:[/red]{e.reason}")
raise typer.Exit(1) from e
tbl = conn.execute(sql).arrow().read_all()
result = write_export(tbl, fmt, Path(o))
console.print(f"[green]导出完成[/green] {result.path}({result.rows} 行 / {result.bytes_written} 字节)")
finally:
conn.close()
@app.command()
def status(
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME"),
json_out: bool = typer.Option(False, "--json", help="结构化 JSON 输出(脚本化)"),
) -> None:
"""水位、锁、HTTP 配置、磁盘占用(运维入口)。"""
from .lockfile import is_alive, read_lock
from .store import current_version
dbpath = resolve_dbpath(d)
layout = DbPathLayout(dbpath)
info: dict = {"dbpath": str(layout.root)}
lock = read_lock(layout.lock_file)
info["lock"] = (
{"held": True, "pid": lock.pid, "alive": is_alive(lock)} if lock else {"held": False}
)
info["http"] = {"enabled": True, "host": "127.0.0.1", "port": 8765, "note": "config 段 http"}
conn = _open_ro(layout.root)
try:
info["schema_version"] = current_version(conn)
cps = conn.execute(
"SELECT dataset, market, scope, watermark, rows, status FROM etl_checkpoint "
"ORDER BY dataset, market, scope"
).fetchall()
info["checkpoints"] = [
{"dataset": r[0], "market": r[1], "scope": r[2], "watermark": r[3], "rows": r[4], "status": r[5]}
for r in cps
]
info["bars_1d_rows"] = conn.execute("SELECT count(*) FROM bars_1d").fetchone()[0]
finally:
conn.close()
try:
import psutil
du = psutil.disk_usage(str(layout.root))
info["disk"] = {"used_pct": round(du.percent, 1), "free_gb": round(du.free / 1e9, 1)}
except Exception: # noqa: BLE001 — 磁盘信息尽力而为
info["disk"] = None
# --json 与默认都输出同一份 JSON(默认人类可读,--json 供程序消费;M1 共用渲染)
console.print_json(json.dumps(info, default=str, ensure_ascii=False))
def main() -> None:
app()
if __name__ == "__main__": # pragma: no cover
main()
+190
View File
@@ -0,0 +1,190 @@
"""config.json 模型与读写(设计 §12)。JSON 格式;extra=forbid 防拼错字段静默失效。"""
from __future__ import annotations
import json
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from pydantic import BaseModel, ConfigDict, Field
_ENV_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
def expand_env(value: str) -> str:
"""展开 ``${ENV_VAR}``(设计 §12:http.token 支持环境变量展开,不明文入库)。"""
return _ENV_PATTERN.sub(lambda m: os_environ_get(m.group(1)), value)
def os_environ_get(name: str) -> str:
import os
return os.environ.get(name, "")
class MarketConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
enabled: bool = True
history_from: str
history_from_bars_1d: str | None = None # null 时跟随 history_from(§3.4)
class HttpConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
enabled: bool = True
host: str = "127.0.0.1"
port: int = 8765
token: str = ""
max_ws_conns: int = 64
ws_queue_len: int = 256
class SourcesConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
bars: str = "tdx"
realtime: str = "westock"
reference: list[str] = Field(default_factory=lambda: ["tdx", "westock"])
futures: list[str] = Field(default_factory=lambda: ["tdx", "yfinance"])
news: list[str] = Field(default_factory=lambda: ["scripts/fetch_news.py"])
fallback: list[str] = Field(default_factory=lambda: ["akshare_port"])
cross_check: bool = True
cross_check_ratio: float = 0.01
class ProxyConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
global_: str = Field(default="", alias="global")
per_source: dict[str, str] = Field(default_factory=dict)
class FetchConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
rate_limit_rps: dict[str, float] = Field(default_factory=dict)
timeout_ms: int = 10000
max_retry: int = 3
class DailyChainConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
cn: str = "15:05"
hk: str = "16:10"
us: str = "04:30"
class ScheduleConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
timezone: str = "Asia/Shanghai" # 显式绑定,不读系统 TZ(评审 P1)
cn_intraday_interval_s: int = 90
snapshot_interval_s: int = 3
daily_chain: DailyChainConfig = Field(default_factory=DailyChainConfig)
compaction: str = "monthly"
class QualityConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
enabled: bool = True
checks: list[str] = Field(
default_factory=lambda: ["gap", "ohlc", "limit", "adj", "rowcount", "dualsource"]
)
class SqlConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
max_rows: int = 1_000_000
timeout_ms: int = 30_000
max_arrow_bytes: int = 64 * 1024 * 1024
max_concurrent: int = 4
acquire_timeout_s: float = 5.0 # 信号量等待超过即 429(§8.3)
memory_limit: str = "2GB"
allow_functions: list[str] = Field(default_factory=list) # 额外放行清单(§8.3)
class ExportConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
out_dir: str = "exports"
cache_ttl_h: int = 24
keep_days: int = 7
formats: list[str] = Field(default_factory=lambda: ["csv", "parquet", "qlib", "backtrader"])
class LogConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
level: str = "INFO"
keep_days: int = 30
class AppConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
version: int = 1
dbpath_created: str
markets: dict[str, MarketConfig]
categories: dict[str, list[str]]
sources: SourcesConfig = Field(default_factory=SourcesConfig)
proxy: ProxyConfig = Field(default_factory=ProxyConfig)
fetch: FetchConfig = Field(default_factory=FetchConfig)
http: HttpConfig = Field(default_factory=HttpConfig)
schedule: ScheduleConfig = Field(default_factory=ScheduleConfig)
quality: QualityConfig = Field(default_factory=QualityConfig)
sql: SqlConfig = Field(default_factory=SqlConfig)
export: ExportConfig = Field(default_factory=ExportConfig)
log: LogConfig = Field(default_factory=LogConfig)
_DEFAULT_CATEGORIES = {
"cn": ["index", "stock", "future", "cbond", "etf", "reits", "option"],
"us": ["index", "stock", "future", "etf", "reits"],
"hk": ["index", "stock", "future", "etf", "reits"],
}
def default_config(markets: list[str] | None = None, now: datetime | None = None) -> AppConfig:
"""init 生成的默认 config(§12)。markets 默认三市场全开。"""
markets = markets or ["cn", "us", "hk"]
defaults = {
"cn": ("2004-01-01", None),
"us": ("2020-01-01", None),
"hk": ("2020-01-01", None),
}
now = now or datetime.now(timezone.utc)
return AppConfig(
dbpath_created=now.isoformat(),
markets={
m: MarketConfig(enabled=True, history_from=defaults[m][0], history_from_bars_1d=None)
for m in markets
},
categories={m: list(_DEFAULT_CATEGORIES[m]) for m in markets},
)
def write_config(config: AppConfig, path: Path) -> None:
path.write_text(config.model_dump_json(indent=2, by_alias=True), encoding="utf-8")
def _expand_strings(obj: Any) -> Any:
if isinstance(obj, str):
return expand_env(obj)
if isinstance(obj, list):
return [_expand_strings(v) for v in obj]
if isinstance(obj, dict):
return {k: _expand_strings(v) for k, v in obj.items()}
return obj
def load_config(path: Path) -> AppConfig:
"""读 config.json 并展开 ${ENV};字段拼错/多余直接报错(extra=forbid)。"""
raw = json.loads(path.read_text(encoding="utf-8"))
return AppConfig.model_validate(_expand_strings(raw))
+19
View File
@@ -0,0 +1,19 @@
"""ETL:checkpoint、seed 导入、update 编排。"""
from .checkpoint import failed_checkpoints, list_checkpoints, upsert_checkpoint
from .seed import SeedError, SeedReport, import_seed
from .update import run_update, run_update_sync, update_bars_1d, update_calendar, update_securities
__all__ = [
"SeedError",
"SeedReport",
"failed_checkpoints",
"import_seed",
"list_checkpoints",
"run_update",
"run_update_sync",
"update_bars_1d",
"update_calendar",
"update_securities",
"upsert_checkpoint",
]
+45
View File
@@ -0,0 +1,45 @@
"""etl_checkpoint 断点与水位(设计 §7.4)。"""
from __future__ import annotations
import duckdb
def upsert_checkpoint(
conn: duckdb.DuckDBPyConnection,
dataset: str,
market: str,
scope: str,
watermark: str,
rows: int,
status: str = "ok",
) -> None:
conn.execute(
"INSERT OR REPLACE INTO etl_checkpoint "
"(dataset, market, scope, watermark, rows, status, updated_at) "
"VALUES (?, ?, ?, ?, ?, ?, now())",
[dataset, market, scope, watermark, rows, status],
)
def list_checkpoints(conn: duckdb.DuckDBPyConnection) -> list[dict]:
rows = conn.execute(
"SELECT dataset, market, scope, watermark, rows, status, updated_at "
"FROM etl_checkpoint ORDER BY dataset, market, scope"
).fetchall()
return [
{
"dataset": r[0],
"market": r[1],
"scope": r[2],
"watermark": r[3],
"rows": r[4],
"status": r[5],
"updated_at": None if r[6] is None else r[6].isoformat(),
}
for r in rows
]
def failed_checkpoints(conn: duckdb.DuckDBPyConnection) -> list[dict]:
return [c for c in list_checkpoints(conn) if c["status"] != "ok"]
+142
View File
@@ -0,0 +1,142 @@
"""seed 导入(init --seed):把现成数据(如 stock_prediction 的 A股日K)搬进库。
- 支持列名:symbol, date, open, high, low, close, volume(=vol), amount(=amt),
turnover, adj_factor(后两者可选);
- 幂等:事务内 DELETE 覆盖区(同 symbol 的 [min,max] 日期)后 INSERT;
- parquet / csv 由扩展名路由。
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import date, datetime
from pathlib import Path
import duckdb
import pyarrow as pa
import pyarrow.csv as pacsv
import pyarrow.parquet as paparquet
from ..sources.classify import classify_cn
_COLUMN_ALIASES = {
"symbol": "symbol",
"date": "date",
"open": "open",
"high": "high",
"low": "low",
"close": "close",
"volume": "volume",
"vol": "volume",
"amount": "amount",
"amt": "amount",
"turnover": "turnover",
"adj_factor": "adj_factor",
}
_REQUIRED = ("symbol", "date", "open", "high", "low", "close")
class SeedError(ValueError):
pass
@dataclass(slots=True)
class SeedReport:
path: str
rows: int
symbols: list[str]
date_min: date | None
date_max: date | None
def _read_table(path: Path) -> pa.Table:
suffix = path.suffix.lower()
if suffix == ".parquet":
return paparquet.read_table(path)
if suffix == ".csv":
return pacsv.read_csv(path)
raise SeedError(f"seed 仅支持 .parquet/.csv,收到 {path.name}")
def _as_date(v) -> date:
if isinstance(v, datetime):
return v.date()
if isinstance(v, date):
return v
return date.fromisoformat(str(v)[:10])
def _rows_from_table(table: pa.Table) -> list[tuple]:
cols = {}
for name in table.column_names:
canonical = _COLUMN_ALIASES.get(name.strip().lower())
if canonical:
cols[canonical] = table.column(name).to_pylist()
missing = [c for c in _REQUIRED if c not in cols]
if missing:
raise SeedError(f"seed 缺必需列:{missing}(实际列:{table.column_names})")
n = len(cols["symbol"])
out: list[tuple] = []
for i in range(n):
symbol = str(cols["symbol"][i]).strip().lower()
if not symbol:
raise SeedError(f"seed 第 {i} 行 symbol 为空")
out.append(
(
symbol,
_as_date(cols["date"][i]),
float(cols["open"][i]),
float(cols["high"][i]),
float(cols["low"][i]),
float(cols["close"][i]),
float(cols["volume"][i] or 0),
float(cols["amount"][i] or 0),
None if cols.get("turnover", [None] * n)[i] is None else float(cols["turnover"][i]),
None if cols.get("adj_factor", [None] * n)[i] is None else float(cols["adj_factor"][i]),
)
)
return out
def import_seed(
conn: duckdb.DuckDBPyConnection,
seed_path: Path | str,
*,
market: str = "cn",
category: str | None = None,
) -> SeedReport:
"""导入 seed 到 bars_1d。category 缺省按 symbol 规则分类(A股)。"""
path = Path(seed_path)
if not path.exists():
raise SeedError(f"seed 文件不存在:{path}")
rows = _rows_from_table(_read_table(path))
if not rows:
raise SeedError(f"seed 无数据行:{path}")
conn.begin()
try:
symbols = sorted({r[0] for r in rows})
dates = [r[1] for r in rows]
for symbol in symbols:
sym_rows = [r for r in rows if r[0] == symbol]
cat = category or classify_cn(symbol)
conn.execute(
"DELETE FROM bars_1d WHERE market=? AND category=? AND symbol=? "
"AND date BETWEEN ? AND ?",
[market, cat, symbol, min(dates), max(dates)],
)
conn.executemany(
"INSERT INTO bars_1d (market, category, symbol, date, open, high, low, "
"close, volume, amount, turnover, adj_factor) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
[(market, cat, *r) for r in sym_rows],
)
conn.commit()
except Exception:
conn.rollback()
raise
return SeedReport(
path=str(path),
rows=len(rows),
symbols=symbols,
date_min=min(dates),
date_max=max(dates),
)
+166
View File
@@ -0,0 +1,166 @@
"""update 命令的 ETL 编排(设计 §6/§7.3)。
幂等模式:以 (market, symbol, 日期区间) 粒度整区替换(事务内 DELETE+INSERT),
重跑任意次数结果一致。CLI 一次性场景允许事件循环内同步 DuckDB 调用
(§7.6 的线程池隔离约束针对 serve 常驻场景,M2 接入调度器时迁移)。
"""
from __future__ import annotations
import asyncio
from datetime import date
import duckdb
from ..sources.base import SourceAdapter, UpdateReport, full_year_calendar
from ..sources.classify import classify_cn
from .checkpoint import upsert_checkpoint
_CHUNK = 400 # 每次 fetch 的 symbol 数(TDX 一次全拉列表太大时分块)
async def update_securities(
conn: duckdb.DuckDBPyConnection, market: str, adapter: SourceAdapter
) -> UpdateReport:
rows = await adapter.fetch_securities(market)
conn.begin()
try:
conn.execute("DELETE FROM securities WHERE market=?", [market])
conn.executemany(
"INSERT INTO securities (market, symbol, name, category, exchange, currency, lot_size) "
"VALUES (?,?,?,?,?,?,?)",
[
(market, r.symbol, r.name, r.category, r.exchange, r.currency, r.lot_size)
for r in rows
],
)
conn.commit()
except Exception:
conn.rollback()
raise
upsert_checkpoint(conn, "securities", market, "all", "", len(rows))
return UpdateReport(dataset="securities", market=market, symbols=[r.symbol for r in rows], rows=len(rows))
async def update_calendar(
conn: duckdb.DuckDBPyConnection, market: str, adapter: SourceAdapter, years: list[int]
) -> UpdateReport:
"""calendar 落库恒为「整年逐日」:adapter 只需给出交易日(§5.2)。
源(tdx/fake)给出的是交易日集合,本函数负责补全全年(非交易日 is_open=False),
保证 calendar 表的语义唯一:任意一天查得到、is_open 可信。
"""
total = 0
for year in years:
rows = await adapter.fetch_calendar(market, year)
trade_dates = {r.trade_date for r in rows if r.is_open}
full = full_year_calendar(year, trade_dates)
conn.begin()
try:
conn.execute(
"DELETE FROM calendar WHERE market=? AND trade_date BETWEEN ? AND ?",
[market, date(year, 1, 1), date(year, 12, 31)],
)
conn.executemany(
"INSERT INTO calendar (market, trade_date, is_open, note) VALUES (?,?,?,?)",
[(market, r.trade_date, r.is_open, r.note) for r in full],
)
conn.commit()
except Exception:
conn.rollback()
raise
total += len(full)
open_rows = sum(1 for r in full if r.is_open)
upsert_checkpoint(conn, "calendar", market, str(year), date(year, 12, 31).isoformat(), open_rows)
return UpdateReport(dataset="calendar", market=market, rows=total)
async def update_bars_1d(
conn: duckdb.DuckDBPyConnection,
market: str,
adapter: SourceAdapter,
*,
symbols: list[str] | None = None,
start: date | None = None,
end: date | None = None,
) -> UpdateReport:
if symbols is None:
rows = conn.execute(
"SELECT symbol FROM securities WHERE market=? "
"AND category IN ('stock','index','etf','cbond') ORDER BY symbol",
[market],
).fetchall()
symbols = [r[0] for r in rows]
if not symbols:
raise RuntimeError(
"securities 表为空:请先 update --dataset securities 建证券主数据"
)
total_rows = 0
seen_dates: list[date] = []
for i in range(0, len(symbols), _CHUNK):
chunk = symbols[i : i + _CHUNK]
bars = await adapter.fetch_bars_1d(market, chunk, start, end)
conn.begin()
try:
for symbol in chunk:
sym_bars = [b for b in bars if b.symbol == symbol]
if not sym_bars:
continue
category = classify_cn(symbol) if market == "cn" else "stock"
d_min, d_max = min(b.date for b in sym_bars), max(b.date for b in sym_bars)
conn.execute(
"DELETE FROM bars_1d WHERE market=? AND category=? AND symbol=? "
"AND date BETWEEN ? AND ?",
[market, category, symbol, d_min, d_max],
)
conn.executemany(
"INSERT INTO bars_1d (market, category, symbol, date, open, high, low, "
"close, volume, amount, turnover, adj_factor) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
[
(market, category, b.symbol, b.date, b.open, b.high, b.low, b.close,
b.volume, b.amount, b.turnover, b.adj_factor)
for b in sym_bars
],
)
total_rows += len(sym_bars)
seen_dates.extend([b.date for b in sym_bars])
conn.commit()
except Exception:
conn.rollback()
raise
watermark = max(seen_dates).isoformat() if seen_dates else ""
upsert_checkpoint(conn, "bars_1d", market, "daily", watermark, total_rows)
return UpdateReport(
dataset="bars_1d", market=market, symbols=symbols, rows=total_rows,
date_min=min(seen_dates) if seen_dates else None,
date_max=max(seen_dates) if seen_dates else None,
)
async def run_update(
conn: duckdb.DuckDBPyConnection,
market: str,
adapter: SourceAdapter,
*,
datasets: list[str],
symbols: list[str] | None = None,
start: date | None = None,
end: date | None = None,
years: list[int] | None = None,
) -> list[UpdateReport]:
"""按依赖顺序编排:securities → calendar → bars_1d。"""
reports: list[UpdateReport] = []
if "securities" in datasets:
reports.append(await update_securities(conn, market, adapter))
if "calendar" in datasets:
reports.append(await update_calendar(conn, market, adapter, years or [date.today().year]))
if "bars_1d" in datasets:
reports.append(
await update_bars_1d(conn, market, adapter, symbols=symbols, start=start, end=end)
)
return reports
def run_update_sync(conn: duckdb.DuckDBPyConnection, market: str, adapter: SourceAdapter, **kw) -> list[UpdateReport]:
"""CLI 同步入口。"""
return asyncio.run(run_update(conn, market, adapter, **kw))
+29
View File
@@ -0,0 +1,29 @@
"""导出:csv/parquet 落盘(pandas/polars 走 /v1/query fmt=arrow)。"""
from .files import (
EXPORT_FORMATS,
ExportFormatError,
ExportPathError,
ExportResult,
get_export_job,
new_job_id,
record_export_job,
resolve_export_target,
run_query_arrow,
spec_hash,
write_export,
)
__all__ = [
"EXPORT_FORMATS",
"ExportFormatError",
"ExportPathError",
"ExportResult",
"get_export_job",
"new_job_id",
"record_export_job",
"resolve_export_target",
"run_query_arrow",
"spec_hash",
"write_export",
]
+133
View File
@@ -0,0 +1,133 @@
"""导出落盘(设计 §9)。
- export 只管落盘格式:csv / parquet(pandas/polars 走 /v1/query fmt=arrow 消费);
- 路径安全(评审 P1):输出根恒为 exports/{job_id}/,名称字段白名单字符过滤 +
resolve 后必须仍在 exports/ 前缀内;杜绝 ../ 穿出 dbpath;
- job 结果记 export_jobs 表(spec hash 缓存 24h 的键在 M2 worker 化时接入)。
"""
from __future__ import annotations
import hashlib
import json
import re
import uuid
from dataclasses import dataclass
from pathlib import Path
import duckdb
import pyarrow as pa
import pyarrow.csv as pacsv
import pyarrow.parquet as paparquet
_JOB_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$")
_FILENAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
EXPORT_FORMATS: frozenset[str] = frozenset({"csv", "parquet"})
class ExportPathError(ValueError):
pass
class ExportFormatError(ValueError):
pass
def new_job_id() -> str:
return uuid.uuid4().hex[:12]
def spec_hash(spec: dict) -> str:
"""导出请求的稳定指纹(缓存键,§9)。"""
canonical = json.dumps(spec, sort_keys=True, ensure_ascii=False)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def resolve_export_target(exports_dir: Path, job_id: str, filename: str) -> Path:
"""exports/{job_id}/{filename};白名单 + resolve 前缀校验(§9 路径安全)。
纵深防御:``_JOB_ID_RE`` / ``_FILENAME_RE`` 已禁掉 ``/`` 与 ``..``,因此常规
输入下 resolve 后必然在 exports/ 内。最后的 startswith 校验是为**符号链接**
(exports/{job_id} 被替换成指向外部的软链)兜底——该分支在无软链的环境里
结构性不可达,保留为防线而非死代码。
"""
if not _JOB_ID_RE.match(job_id):
raise ExportPathError(f"非法 job_id:{job_id!r}")
if not _FILENAME_RE.match(filename):
raise ExportPathError(f"非法导出文件名:{filename!r}")
exports_dir = Path(exports_dir).resolve()
target = (exports_dir / job_id / filename).resolve()
root_str = str(exports_dir).rstrip("\\/") + "\\"
posix_root = str(exports_dir).rstrip("\\/") + "/"
if not (str(target).startswith(root_str) or str(target).startswith(posix_root)):
raise ExportPathError(f"导出路径越界:{target}")
return target
@dataclass
class ExportResult:
path: Path
rows: int
bytes_written: int
fmt: str
def run_query_arrow(
conn: duckdb.DuckDBPyConnection, sql: str, params: list | None = None
) -> pa.Table:
"""执行查询返回 Arrow 表(查询/导出共用;调用方负责 sqlgate 校验与线程池)。"""
result = conn.execute(sql, params or [])
return result.arrow().read_all()
def write_export(table: pa.Table, fmt: str, target: Path) -> ExportResult:
if fmt not in EXPORT_FORMATS:
raise ExportFormatError(f"export 仅支持落盘格式 {sorted(EXPORT_FORMATS)},收到 {fmt!r}")
target.parent.mkdir(parents=True, exist_ok=True)
if fmt == "csv":
pacsv.write_csv(table, target)
else:
paparquet.write_table(table, target, compression="zstd")
size = target.stat().st_size
return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt=fmt)
def record_export_job(
conn: duckdb.DuckDBPyConnection,
job_id: str,
*,
fmt: str,
path: str,
rows: int,
bytes_written: int,
state: str = "done",
spec_json: str = "{}",
) -> None:
conn.execute(
"INSERT OR REPLACE INTO export_jobs "
"(job_id, spec_json, fmt, path, rows, bytes, state, created_at, finished_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, now(), now())",
[job_id, spec_json, fmt, path, rows, bytes_written, state],
)
def get_export_job(conn: duckdb.DuckDBPyConnection, job_id: str) -> dict | None:
row = conn.execute(
"SELECT job_id, spec_json, fmt, path, rows, bytes, state, "
"created_at, finished_at FROM export_jobs WHERE job_id = ?",
[job_id],
).fetchone()
if row is None:
return None
return {
"job_id": row[0],
"spec_json": row[1],
"fmt": row[2],
"path": row[3],
"rows": row[4],
"bytes": row[5],
"state": row[6],
"created_at": None if row[7] is None else row[7].isoformat(),
"finished_at": None if row[8] is None else row[8].isoformat(),
}
+120
View File
@@ -0,0 +1,120 @@
"""单写者锁 quantdata.lock(设计 §11)。
判活 = PID + 进程 start time + boot_id 三元一致(防 PID 复用误判)。
发现陈旧锁:先备份 .lock.bak 再接管,并记日志。
"""
from __future__ import annotations
import json
import os
import shutil
import time
from dataclasses import dataclass
from pathlib import Path
import psutil
class LockHeldError(RuntimeError):
def __init__(self, info: "LockInfo") -> None:
self.info = info
super().__init__(
f"已有 quantdata 实例持有锁:PID={info.pid}( acquired_at={info.acquired_at})。"
f"如确认无实例在跑,可删除锁文件。"
)
@dataclass
class LockInfo:
pid: int
start_time: float
boot_id: float
acquired_at: str
cmd: str
def to_json(self) -> str:
return json.dumps(self.__dict__, ensure_ascii=False, indent=2)
@classmethod
def from_json(cls, text: str) -> "LockInfo":
d = json.loads(text)
return cls(
pid=int(d["pid"]),
start_time=float(d["start_time"]),
boot_id=float(d["boot_id"]),
acquired_at=str(d.get("acquired_at", "")),
cmd=str(d.get("cmd", "")),
)
def _boot_id() -> float:
"""boot 标识:psutil.boot_time() 秒级时间戳,Windows/Linux 通用。"""
return psutil.boot_time()
def _process_start_time(pid: int) -> float | None:
"""进程当前 start time;进程不存在返回 None。"""
try:
return psutil.Process(pid).create_time()
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
return None
def is_alive(info: LockInfo, *, now_boot_id: float | None = None) -> bool:
"""三元判活:进程存在 + start_time 一致 + boot_id 一致。"""
now_boot_id = _boot_id() if now_boot_id is None else now_boot_id
if info.boot_id != now_boot_id:
return False # 重启过系统 → 一律陈旧
start = _process_start_time(info.pid)
if start is None:
return False
# start_time 允许极小浮点误差
return abs(start - info.start_time) < 0.01
def read_lock(path: Path) -> LockInfo | None:
try:
return LockInfo.from_json(path.read_text(encoding="utf-8"))
except FileNotFoundError:
return None
except (json.JSONDecodeError, KeyError, ValueError):
# 损坏的锁文件视为陈旧(半写状态)
return None
def acquire(lock_path: Path, *, cmd: str = "", acquired_at: str | None = None) -> LockInfo:
"""拿锁;活锁持有 → LockHeldError;陈旧/损坏锁 → 备份后接管。"""
existing = read_lock(lock_path)
if existing is not None and is_alive(existing):
raise LockHeldError(existing)
if existing is not None:
# 陈旧锁:先备份再接管(§11)
backup = lock_path.with_suffix(lock_path.suffix + ".bak")
shutil.copy2(lock_path, backup)
try:
print(f"[lock] 发现陈旧锁(PID={existing.pid} 已不存在),备份到 {backup} 后接管")
except OSError: # stdout 关闭等极端场景不阻塞拿锁
pass
lock_path.parent.mkdir(parents=True, exist_ok=True)
me = psutil.Process(os.getpid())
info = LockInfo(
pid=me.pid,
start_time=me.create_time(),
boot_id=_boot_id(),
acquired_at=acquired_at or time.strftime("%Y-%m-%dT%H:%M:%S%z"),
cmd=cmd,
)
lock_path.write_text(info.to_json(), encoding="utf-8")
return info
def release(lock_path: Path, info: LockInfo) -> bool:
"""释放锁:仅当锁内仍是自己(防止误删后来者的锁)才删除。"""
current = read_lock(lock_path)
if current is None:
return False
if current.pid == info.pid and abs(current.start_time - info.start_time) < 0.01:
lock_path.unlink(missing_ok=True)
return True
return False
+77
View File
@@ -0,0 +1,77 @@
"""dbpath 目录规范(设计 §3.1)。一切状态收敛在 -d 指定的 dbpath。"""
from __future__ import annotations
import os
from pathlib import Path
class DbPathLayout:
"""dbpath 内固定相对位置的访问器。不负责创建目录(init 时 ensure)。"""
def __init__(self, root: Path | str) -> None:
self.root = Path(root).resolve()
@property
def config_file(self) -> Path:
return self.root / "config.json"
@property
def catalog(self) -> Path:
return self.root / "catalog.duckdb"
@property
def data_dir(self) -> Path:
return self.root / "data"
@property
def scripts_dir(self) -> Path:
return self.root / "scripts"
@property
def exports_dir(self) -> Path:
return self.root / "exports"
@property
def logs_dir(self) -> Path:
return self.root / "logs"
@property
def tmp_dir(self) -> Path:
return self.root / "tmp"
@property
def lock_file(self) -> Path:
return self.root / "quantdata.lock"
@property
def trash_dir(self) -> Path:
"""compaction 两阶段协议的延迟删除区(M2 使用,目录先规范)。"""
return self.root / "data" / ".trash"
def ensure_dirs(self) -> list[Path]:
"""init 用:建全部目录,返回已确保存在的目录列表。幂等。"""
made: list[Path] = []
for d in (
self.root,
self.data_dir,
self.scripts_dir,
self.exports_dir,
self.logs_dir,
self.tmp_dir,
self.trash_dir,
):
if not d.exists():
d.mkdir(parents=True)
made.append(d)
return made
def resolve_dbpath(explicit: str | None) -> Path:
"""-d 显式指定优先;否则环境变量 QUANTDATA_HOME 兜底;都没有则报错。"""
if explicit:
return Path(explicit).resolve()
env = os.environ.get("QUANTDATA_HOME")
if env:
return Path(env).resolve()
raise SystemExit("缺少 dbpath:请用 -d /xxx/dbpath 或设置环境变量 QUANTDATA_HOME")
+29
View File
@@ -0,0 +1,29 @@
"""serve 常驻:FastAPI 网关、市场状态、鉴权、运行时。"""
from .app import AppContext, QueryRequest, ExportRequest, create_app
from .auth import (
SecurityError,
assert_security,
is_loopback,
mask_query_string,
mask_token,
token_ok,
)
from .state import MARKET_TZ, SESSIONS, MarketState, market_state
__all__ = [
"AppContext",
"ExportRequest",
"MARKET_TZ",
"MarketState",
"QueryRequest",
"SESSIONS",
"SecurityError",
"assert_security",
"create_app",
"is_loopback",
"mask_query_string",
"mask_token",
"market_state",
"token_ok",
]
+395
View File
@@ -0,0 +1,395 @@
"""FastAPI 服务网关(设计 §8):REST /v1/query /v1/export /v1/meta /v1/stats /v1/state + 探针。
执行模型(§7.6):查询一律线程池执行(事件循环零阻塞);
超时 = ``connection.interrupt()`` 真中断;并发信号量 + 等待超时 429(§8.3)。
"""
from __future__ import annotations
import asyncio
import base64
import time
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import duckdb
import psutil
import pyarrow as pa
from fastapi import Depends, FastAPI, Header, HTTPException
from fastapi.responses import FileResponse, JSONResponse
from pydantic import BaseModel, Field
from zoneinfo import ZoneInfo
from .. import __version__
from ..config import AppConfig
from ..export.files import (
EXPORT_FORMATS,
ExportPathError,
get_export_job,
new_job_id,
record_export_job,
resolve_export_target,
write_export,
)
from ..sqlgate import SqlRejected, validate
from .auth import token_ok
from .state import MARKET_TZ, market_state
class ResultTooLargeError(ValueError):
pass
class QueryTimeoutError(RuntimeError):
pass
@dataclass
class AppContext:
"""app 依赖注入载体。cursor 由调用方提供(serve 从写连接派生;测试注入)。"""
dbpath: Path
config: AppConfig
cursor: duckdb.DuckDBPyConnection
schema_version: int = 0
class QueryRequest(BaseModel):
sql: str
params: list[Any] = Field(default_factory=list)
fmt: str = "json" # json | arrow
max_rows: int | None = None
timeout_ms: int | None = None
class ExportRequest(BaseModel):
fmt: str # csv | parquet
sql: str
params: list[Any] = Field(default_factory=list)
filename: str | None = None
sync: bool = True
def execute_query_blocking(
cursor: duckdb.DuckDBPyConnection,
sql: str,
params: list[Any],
max_rows: int,
max_arrow_bytes: int,
) -> tuple[pa.Table, bool]:
"""线程池内执行:fetch Arrow → 上限校验 → 截断。返回 (table, truncated)。"""
result = cursor.execute(sql, params or [])
tbl = result.arrow().read_all()
if tbl.nbytes > max_arrow_bytes:
raise ResultTooLargeError(
f"结果 {tbl.nbytes} 字节超过上限 {max_arrow_bytes}:请改走 /v1/export(§8.3)"
)
truncated = tbl.num_rows > max_rows
if truncated:
tbl = tbl.slice(0, max_rows)
return tbl, truncated
def arrow_stream_b64(tbl: pa.Table) -> str:
"""Arrow IPC **streaming format** → base64(附录 A 解码路径)。"""
sink = pa.BufferOutputStream()
with pa.ipc.new_stream(sink, tbl.schema) as writer:
writer.write_table(tbl)
return base64.b64encode(sink.getvalue().to_pybytes()).decode("ascii")
def table_to_rows(tbl: pa.Table) -> list[list[Any]]:
if tbl.num_columns == 0:
return []
cols = [c.to_pylist() for c in tbl.columns]
return [list(row) for row in zip(*cols)]
def utc_now_iso() -> str:
"""ISO8601 UTC 带 Z(§8.2 全文统一)。"""
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def query_response(
tbl: pa.Table, truncated: bool, elapsed_ms: int, fmt: str
) -> dict[str, Any]:
out: dict[str, Any] = {
"columns": tbl.schema.names,
"dtypes": [str(f.type) for f in tbl.schema],
"row_count": tbl.num_rows,
"elapsed_ms": elapsed_ms,
"truncated": truncated,
}
if fmt == "arrow":
out["b64_arrow"] = arrow_stream_b64(tbl)
else:
out["rows"] = table_to_rows(tbl)
return out
def disk_usage_bytes(root: Path) -> int:
return sum(p.stat().st_size for p in root.rglob("*") if p.is_file())
async def _run_with_timeout(
ctx: AppContext,
sql: str,
params: list[Any],
max_rows: int,
timeout_s: float,
) -> tuple[pa.Table, bool]:
"""线程池执行 + 超时真中断(§7.6)。query / export 共用同一执行语义。
不能用 ``asyncio.wait_for(to_thread(...))``:超时时 wait_for 会 cancel 那个
已成 future,随后收割阶段抛的 CancelledError 会逃逸出调用方的 except 分支
(Python 3.13 实测)。这里用 shield + 手动超时,超时后 interrupt 连接,
再把线程异常全部吞掉,只向外报 504。
"""
task = asyncio.ensure_future(
asyncio.to_thread(
execute_query_blocking, ctx.cursor, sql, params, max_rows, ctx.config.sql.max_arrow_bytes
)
)
done, _pending = await asyncio.wait({task}, timeout=timeout_s)
if not done:
ctx.cursor.interrupt() # 真中断:线程内查询被打断
try:
await asyncio.shield(task) # 收割线程异常,保证不悬空
except BaseException: # noqa: BLE001 — 中断/取消异常统一吞掉,对外只报 504
pass
raise HTTPException(
status_code=504, detail=f"查询超时(>{timeout_s:.1f}s),已 interrupt 中断"
)
return task.result()
def create_app(ctx: AppContext) -> FastAPI:
sql_cfg = ctx.config.sql
query_sem = asyncio.Semaphore(max(1, sql_cfg.max_concurrent))
def _validate_sql(sql: str) -> None:
try:
validate(sql, sql_cfg.allow_functions)
except SqlRejected as e:
raise HTTPException(status_code=400, detail=e.reason) from e
async def _acquire_query_slot() -> None:
try:
await asyncio.wait_for(query_sem.acquire(), timeout=sql_cfg.acquire_timeout_s)
except asyncio.TimeoutError as e:
raise HTTPException(
status_code=429,
detail=f"查询并发已达上限 {sql_cfg.max_concurrent},排队超时(§8.3)",
) from e
async def require_token(
authorization: str | None = Header(default=None),
) -> None:
provided = None
if authorization and authorization.startswith("Bearer "):
provided = authorization[len("Bearer ") :].strip()
if not token_ok(provided, ctx.config.http.token):
raise HTTPException(status_code=401, detail="invalid or missing token")
app = FastAPI(title="quantdata", version=__version__, docs_url="/docs")
# ---- 探针 ---------------------------------------------------------------
@app.get("/healthz")
def healthz() -> dict:
return {"status": "ok"}
@app.get("/v1/ready")
def ready() -> Any:
checks: dict[str, Any] = {}
try:
ctx.cursor.execute("SELECT 1")
checks["catalog"] = "ok"
except duckdb.Error as e:
checks["catalog"] = f"error: {e}"
checks["schema_version"] = ctx.schema_version
checks["schema"] = "ok" if ctx.schema_version >= 1 else "not_migrated"
free_pct: float | None = None
if ctx.dbpath.exists():
usage = psutil.disk_usage(str(ctx.dbpath))
free_pct = round(100.0 - usage.percent, 2)
checks["disk_free_pct"] = free_pct
disk_ok = free_pct is None or free_pct > 5.0 # <5% 硬阈值(§13)
if checks["catalog"] == "ok" and checks["schema"] == "ok" and disk_ok:
return {"ready": True, "checks": checks}
return JSONResponse(status_code=503, content={"ready": False, "checks": checks})
# ---- query ---------------------------------------------------------------
@app.post("/v1/query", dependencies=[Depends(require_token)])
async def query(body: QueryRequest) -> dict[str, Any]:
if body.fmt not in ("json", "arrow"):
raise HTTPException(status_code=422, detail=f"fmt 仅支持 json|arrow,收到 {body.fmt!r}")
_validate_sql(body.sql)
max_rows = min(body.max_rows or sql_cfg.max_rows, sql_cfg.max_rows)
timeout_s = (body.timeout_ms if body.timeout_ms is not None else sql_cfg.timeout_ms) / 1000
started = time.monotonic()
await _acquire_query_slot()
try:
tbl, truncated = await _run_with_timeout(
ctx, body.sql, body.params, max_rows, timeout_s
)
except ResultTooLargeError as e:
raise HTTPException(status_code=413, detail=str(e)) from e
except duckdb.Error as e:
if "interrupt" in str(e).lower():
raise HTTPException(status_code=504, detail=f"查询被中断:{e}") from e
raise HTTPException(status_code=400, detail=f"执行失败:{e}") from e
elapsed = int((time.monotonic() - started) * 1000)
return query_response(tbl, truncated, elapsed, body.fmt)
# ---- export(M1 同步路径:小导出直回文件流;job 化在 M2)--------------------
@app.post("/v1/export", dependencies=[Depends(require_token)])
async def export(body: ExportRequest) -> FileResponse:
if body.fmt not in EXPORT_FORMATS:
raise HTTPException(
status_code=422,
detail=(
f"export 仅支持落盘格式 {sorted(EXPORT_FORMATS)};"
f"pandas/polars 走 /v1/query fmt=arrow(§9 双语义归位)"
),
)
_validate_sql(body.sql)
timeout_s = sql_cfg.timeout_ms / 1000
await _acquire_query_slot()
try:
tbl, _truncated = await _run_with_timeout(
ctx, body.sql, body.params, sql_cfg.max_rows, timeout_s
)
except duckdb.Error as e:
raise HTTPException(status_code=400, detail=f"执行失败:{e}") from e
job_id = new_job_id()
filename = body.filename or f"export.{body.fmt}"
try:
target = resolve_export_target(
ctx.dbpath / ctx.config.export.out_dir, job_id, filename
)
except ExportPathError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
result = await asyncio.to_thread(write_export, tbl, body.fmt, target)
record_export_job(
ctx.cursor,
job_id,
fmt=body.fmt,
path=str(target.relative_to(ctx.dbpath)),
rows=result.rows,
bytes_written=result.bytes_written,
)
media = "text/csv" if body.fmt == "csv" else "application/octet-stream"
return FileResponse(target, media_type=media, filename=filename)
@app.get("/v1/export/{job_id}", dependencies=[Depends(require_token)])
def export_job(job_id: str) -> dict:
job = get_export_job(ctx.cursor, job_id)
if job is None:
raise HTTPException(status_code=404, detail=f"导出任务不存在:{job_id}")
return job
# ---- meta / stats / state -------------------------------------------------
@app.get("/v1/meta", dependencies=[Depends(require_token)])
def meta() -> dict:
names = [
r[0]
for r in ctx.cursor.execute(
"SELECT table_name FROM information_schema.tables "
"WHERE table_schema='main' AND table_type='VIEW' ORDER BY table_name"
).fetchall()
]
views = []
for name in names:
info: dict[str, Any] = {"name": name}
info["columns"] = [
r[0]
for r in ctx.cursor.execute(
"SELECT column_name FROM information_schema.columns "
"WHERE table_schema='main' AND table_name=? ORDER BY ordinal_position",
[name],
).fetchall()
]
info["row_count"] = ctx.cursor.execute(f"SELECT count(*) FROM {name}").fetchone()[0]
cols = set(info["columns"])
if "date" in cols:
row = ctx.cursor.execute(
f"SELECT min(date), max(date) FROM {name}"
).fetchone()
info["date_min"] = None if row[0] is None else row[0].isoformat()
info["date_max"] = None if row[1] is None else row[1].isoformat()
if "symbol" in cols:
info["sample_symbols"] = [
r[0]
for r in ctx.cursor.execute(
f"SELECT DISTINCT symbol FROM {name} WHERE symbol IS NOT NULL LIMIT 3"
).fetchall()
]
views.append(info)
return {
"version": __version__,
"schema_version": ctx.schema_version,
"views": views,
}
@app.get("/v1/stats", dependencies=[Depends(require_token)])
def stats() -> dict:
cps = ctx.cursor.execute(
"SELECT dataset, market, scope, watermark, rows, status, updated_at "
"FROM etl_checkpoint ORDER BY dataset, market, scope"
).fetchall()
jobs = ctx.cursor.execute(
"SELECT state, count(*) FROM export_jobs GROUP BY state"
).fetchall()
usage = None
if ctx.dbpath.exists():
usage = {"bytes": disk_usage_bytes(ctx.dbpath)}
try:
du = psutil.disk_usage(str(ctx.dbpath))
usage["free_pct"] = round(100.0 - du.percent, 2)
except OSError:
usage["free_pct"] = None
return {
"checkpoints": [
{
"dataset": r[0], "market": r[1], "scope": r[2], "watermark": r[3],
"rows": r[4], "status": r[5],
"updated_at": None if r[6] is None else r[6].isoformat(),
}
for r in cps
],
"export_jobs": {state: n for state, n in jobs},
"disk": usage,
"http": {"ws_conns": 0}, # WS 在 M3;指标占位(§13)
}
@app.get("/v1/state/{market}", dependencies=[Depends(require_token)])
def market_state_endpoint(market: str) -> dict:
tz = MARKET_TZ.get(market)
if tz is None:
raise HTTPException(status_code=404, detail=f"未知市场:{market}")
now_local = datetime.now(ZoneInfo(tz))
row = ctx.cursor.execute(
"SELECT is_open FROM calendar WHERE market=? AND trade_date=?",
[market, now_local.date()],
).fetchone()
is_trading_day = bool(row[0]) if row else False
if row is None:
reason = "no_calendar_data"
else:
reason = None
hhmm = now_local.hour * 100 + now_local.minute
ms = market_state(market, hhmm, is_trading_day=is_trading_day)
if reason:
ms.reason = reason
return {
"market": market,
"state": ms.state,
"ts": utc_now_iso(),
"next_at": ms.next_at,
"session_tz": ms.session_tz,
"reason": ms.reason,
}
return app
+51
View File
@@ -0,0 +1,51 @@
"""鉴权与日志脱敏(设计 §8.4)。
- token 非空时:REST ``Authorization: Bearer <token>``;WS(M3)优先握手 Header;
- 非本机且 token 为空 → 拒绝启动(fail-fast,--insecure 显式放行);
- 日志对 token 统一脱敏(只留前 4 位)。
"""
from __future__ import annotations
import re
_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"})
_TOKEN_IN_URL = re.compile(r"(token=)([^&\s]{4})[^&\s]*")
def mask_token(token: str) -> str:
"""token 脱敏:留前 4 位 + ***;空串原样。"""
if not token:
return ""
return token[:4] + "***"
def mask_query_string(qs: str) -> str:
"""日志里的 ``?token=...`` / ``&token=...`` 统一脱敏(含 uvicorn access log 场景)。"""
return _TOKEN_IN_URL.sub(r"\1\2***", qs)
def is_loopback(host: str) -> bool:
return host in _LOOPBACK_HOSTS
def assert_security(host: str, token: str, *, insecure: bool) -> None:
"""启动强制校验(§8.4):非本机 + 空 token + 未显式 --insecure → 拒绝启动。"""
if is_loopback(host) or token or insecure:
return
raise SecurityError(
f"拒绝启动:监听地址 {host} 非 loopback 且 http.token 为空;"
f"请设置 token 或显式使用 --insecure(§8.4 fail-fast)"
)
class SecurityError(RuntimeError):
pass
def token_ok(provided: str | None, expected: str) -> bool:
"""expected 为空 = 不校验;否则必须精确匹配。"""
if not expected:
return True
return bool(provided) and provided == expected
+124
View File
@@ -0,0 +1,124 @@
"""serve 常驻主循环(设计 §7.1 / §7.6)。
启动序:拿锁 → 开库 + migrate → 安全校验 → 起 uvicorn(Server.serve() 挂同一
事件循环,不用 uvicorn.run() 自建循环)→ 前台常驻;SIGTERM/SIGINT 优雅退出。
bind 失败(端口被占)→ 打印占用者 PID 后非零退出(交 supervisord 退避重试)。
"""
from __future__ import annotations
import asyncio
from pathlib import Path
import uvicorn
from ..config import AppConfig, load_config
from ..lockfile import LockHeldError, acquire, release
from ..paths import DbPathLayout
from ..store import SchemaTooNewError, migrate, open_catalog
from .app import AppContext, create_app
from .auth import assert_security
# 退出码:0 正常 / 2 锁被占 / 3 schema 版本不兼容 / 4 安全校验拒绝 / 5 端口被占
EXIT_OK = 0
EXIT_LOCK_HELD = 2
EXIT_SCHEMA_TOO_NEW = 3
EXIT_SECURITY = 4
EXIT_PORT_BUSY = 5
def find_listener_pid(port: int) -> int | None:
"""找占用端口的进程 PID(尽力而为;psutil 未装/权限不足返回 None)。"""
try:
import psutil
for conn in psutil.net_connections(kind="inet"):
if conn.status == psutil.CONN_LISTEN and conn.laddr and conn.laddr.port == port:
return conn.pid
except Exception: # noqa: BLE001 — psutil 缺失/平台权限差异都按“找不到”处理
return None
return None
async def _serve_no_http(stop_event: asyncio.Event) -> None:
"""--no-http:纯 CLI 批处理常驻(M1 无调度器,等信号)。"""
await stop_event.wait()
async def run_serve(
dbpath: Path,
*,
host: str | None = None,
port: int | None = None,
no_http: bool = False,
insecure: bool = False,
config: AppConfig | None = None,
stop_event: asyncio.Event | None = None,
check_port: bool = True,
) -> int:
"""serve 主流程,返回进程退出码。真网 uvicorn 启停段按白名单豁免覆盖率。"""
layout = DbPathLayout(dbpath)
if not layout.root.exists():
print(f"[serve] dbpath 不存在:{layout.root}(先 quantdata init)")
return 1
cfg = config or load_config(layout.config_file)
host = host or cfg.http.host
port = port or cfg.http.port
# 1. 单写者锁(§11)
try:
lock = acquire(layout.lock_file, cmd="quantdata serve")
except LockHeldError as e:
print(f"[serve] 拒绝启动:{e}")
return EXIT_LOCK_HELD
try:
# 2. 开库 + migrate(版本 > 代码已知最高版 → 拒启)
conn = open_catalog(layout.root)
try:
schema_version = migrate(conn)
except SchemaTooNewError as e:
print(f"[serve] {e}")
return EXIT_SCHEMA_TOO_NEW
# 4. 安全校验(§8.4 fail-fast)
try:
assert_security(host, cfg.http.token, insecure=insecure)
except Exception as e:
print(f"[serve] {e}")
return EXIT_SECURITY
# 3/4. 起网关(--no-http 跳过)
if no_http:
await _serve_no_http(stop_event or asyncio.Event())
return EXIT_OK
if check_port:
listener = find_listener_pid(port)
if listener is not None:
print(f"[serve] 端口 {port} 已被占用(PID={listener}),退出非零交 supervisord 重试")
return EXIT_PORT_BUSY
cursor = conn.cursor()
cursor.execute("SET autoinstall_known_extensions=false")
cursor.execute("SET autoload_known_extensions=false")
ctx = AppContext(
dbpath=layout.root, config=cfg, cursor=cursor, schema_version=schema_version
)
app = create_app(ctx)
uv_config = uvicorn.Config(
app, host=host, port=port, log_level=cfg.log.level.lower(),
timeout_graceful_shutdown=30, # 优雅退出预算 30s(§7.1)
)
server = uvicorn.Server(uv_config)
# 真 uvicorn 起服 / socket 层:§14 白名单豁免(平台相关,本机踩过 socketpair
# 死锁)。替代验证 = [CI] 验收链用 CLI 起真 serve + 真端口 curl(覆盖本函数
# 的编排层,只是不把 socket 段记入行覆盖)+ 每周 smoke workflow(DP-2 v0.7)。
await server.serve() # pragma: no cover # 挂同一事件循环(§7.6 第 3 条)
return EXIT_OK
finally:
try:
conn.close()
except Exception:
pass
release(layout.lock_file, lock)
+64
View File
@@ -0,0 +1,64 @@
"""市场状态机纯函数(设计 §10.2 的 M1 简版:日历 + 时段推导,无网络依赖)。
纯本地确定性推导:is_trading_day 由 calendar 表提供;时段规则表内置。
StateEngine 完整事件机(迁移推流)在 M2 接入 serve 调度器。
"""
from __future__ import annotations
from dataclasses import dataclass
MARKET_TZ: dict[str, str] = {
"cn": "Asia/Shanghai",
"us": "America/New_York",
"hk": "Asia/Hong_Kong",
}
# (开, 收) 本地时间,24h 制;M1 不含美股盘前盘后(可扩展)
SESSIONS: dict[str, tuple[tuple[str, str], ...]] = {
"cn": (("09:30", "11:30"), ("13:00", "15:00")),
"hk": (("09:30", "12:00"), ("13:00", "16:00")),
"us": (("09:30", "16:00"),),
}
_AUCTION_START = "09:15" # A股集合竞价
_MORNING_OPEN = "09:30"
@dataclass(slots=True)
class MarketState:
market: str
state: str # CLOSED / PRE_OPEN_AUCTION / OPEN / LUNCH_BREAK
session_tz: str
reason: str
next_at: str | None = None # 本地时间 "HH:MM"
def _hhmm(s: str) -> int:
"""'HH:MM' → HHMM 整数(与 market_state 入参口径一致,如 '09:30' → 930)。"""
h, m = s.split(":")
return int(h) * 100 + int(m)
def market_state(market: str, now_local_hhmm: int, *, is_trading_day: bool) -> MarketState:
"""给定交易所本地 ``HHMM`` 整数(如 930、1301)与是否交易日,确定性推导状态。
参数化时间点便于测试全部边界(开盘/午休/收盘/竞价/节假日)。
"""
tz = MARKET_TZ.get(market, "Asia/Shanghai")
if market not in SESSIONS:
raise KeyError(f"未知市场:{market}")
if not is_trading_day:
return MarketState(market, "CLOSED", tz, "non_trading_day")
sessions = SESSIONS[market]
if market == "cn" and _hhmm(_AUCTION_START) <= now_local_hhmm < _hhmm(_MORNING_OPEN):
return MarketState(market, "PRE_OPEN_AUCTION", tz, "session_rule", _MORNING_OPEN)
for open_s, close_s in sessions:
if _hhmm(open_s) <= now_local_hhmm < _hhmm(close_s):
return MarketState(market, "OPEN", tz, "session_rule", close_s)
# 午休:第一段收后到第二段开前(单段市场无午休)
first_close = _hhmm(sessions[0][1])
second_open = _hhmm(sessions[1][0]) if len(sessions) > 1 else None
if second_open is not None and first_close <= now_local_hhmm < second_open:
return MarketState(market, "LUNCH_BREAK", tz, "session_rule", sessions[1][0])
return MarketState(market, "CLOSED", tz, "session_rule", sessions[0][0])
+24
View File
@@ -0,0 +1,24 @@
"""数据源适配层:SourceAdapter 协议、A股分类、pytdxdata 适配器。"""
from .base import (
Bar1dRow,
CalendarRow,
FakeAdapter,
SecurityRow,
SourceAdapter,
UpdateReport,
)
from .classify import classify_cn, exchange_of
from .tdx import TdxAdapter
__all__ = [
"Bar1dRow",
"CalendarRow",
"FakeAdapter",
"SecurityRow",
"SourceAdapter",
"TdxAdapter",
"UpdateReport",
"classify_cn",
"exchange_of",
]
+124
View File
@@ -0,0 +1,124 @@
"""SourceAdapter 协议与行模型(设计 §5)。
适配层产出统一行模型,ETL 只认行模型;真实源(tdx)与测试源(fake)同构。
fixtures 录制回放 + 故障注入的宿主也在这一层(M2 完善限速/退避)。
"""
from __future__ import annotations
import calendar as _calendar
from dataclasses import dataclass, field
from datetime import date
from typing import Protocol, runtime_checkable
@dataclass(slots=True)
class SecurityRow:
symbol: str
name: str
category: str
exchange: str
currency: str = "CNY"
lot_size: int | None = None
@dataclass(slots=True)
class CalendarRow:
trade_date: date
is_open: bool
note: str = ""
@dataclass(slots=True)
class Bar1dRow:
symbol: str
date: date
open: float
high: float
low: float
close: float
volume: float = 0.0
amount: float = 0.0
turnover: float | None = None
adj_factor: float | None = None
@dataclass(slots=True)
class UpdateReport:
dataset: str
market: str
symbols: list[str] = field(default_factory=list)
rows: int = 0
date_min: date | None = None
date_max: date | None = None
def full_year_calendar(year: int, trade_dates: set[date]) -> list[CalendarRow]:
"""把「交易日集合」补全为「整年逐日」日历(§5.2)。
源只负责说清哪些天开市;补全全年是 ETL 的职责,保证 calendar 表语义唯一:
任意一天都查得到,is_open 可信。
"""
rows: list[CalendarRow] = []
for month in range(1, 13):
for day in range(1, _calendar.monthrange(year, month)[1] + 1):
d = date(year, month, day)
rows.append(CalendarRow(trade_date=d, is_open=d in trade_dates))
return rows
@runtime_checkable
class SourceAdapter(Protocol):
name: str
capabilities: frozenset[str]
async def fetch_securities(self, market: str) -> list[SecurityRow]: ...
async def fetch_bars_1d(
self, market: str, symbols: list[str], start: date | None, end: date | None
) -> list[Bar1dRow]: ...
async def fetch_calendar(self, market: str, year: int) -> list[CalendarRow]: ...
class FakeAdapter:
"""确定性测试源:可注入任意行;不打真网([CI] 层数据走 seed,本源供单测与 CLI 替身)。"""
name = "fake"
capabilities = frozenset({"securities", "bars_1d", "calendar"})
def __init__(
self,
securities: list[SecurityRow] | None = None,
bars: list[Bar1dRow] | None = None,
calendar: dict[int, list[CalendarRow]] | None = None,
) -> None:
self.securities = securities or []
self.bars = bars or []
self.calendar = calendar or {}
self.calls: list[tuple] = []
async def open(self) -> None:
self.calls.append(("open",))
async def close(self) -> None:
self.calls.append(("close",))
async def fetch_securities(self, market: str) -> list[SecurityRow]:
self.calls.append(("securities", market))
return list(self.securities)
async def fetch_bars_1d(
self, market: str, symbols: list[str], start: date | None, end: date | None
) -> list[Bar1dRow]:
self.calls.append(("bars_1d", market, tuple(symbols), start, end))
sym_set = set(symbols)
return [
b
for b in self.bars
if b.symbol in sym_set and (start is None or b.date >= start) and (end is None or b.date <= end)
]
async def fetch_calendar(self, market: str, year: int) -> list[CalendarRow]:
self.calls.append(("calendar", market, year))
return list(self.calendar.get(year, []))
+31
View File
@@ -0,0 +1,31 @@
"""A股证券分类(纯函数,表驱动)。symbol 规范 = pytdxdata 形态:sh600000 / sz000001 / bj430047。"""
from __future__ import annotations
# (category, 前缀元组) —— 顺序敏感:先长/特例后短/泛化
_CLASSIFY_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
("index", ("sh000", "sh899", "sz399", "sz395")),
("reits", ("sh508",)),
("etf", ("sh50", "sh51", "sh56", "sh58", "sz15", "sz16", "sz18")),
("cbond", ("sh10", "sh11", "sh12", "sz12")),
("stock", ("sh60", "sh68", "sz00", "sz30", "sz20")),
("stock", ("bj",)),
)
def classify_cn(symbol: str) -> str:
"""按 symbol 前缀归类 A股品类;未匹配归 other(绝不猜)。"""
s = symbol.strip().lower()
for category, prefixes in _CLASSIFY_RULES:
if any(s.startswith(p) for p in prefixes):
return category
return "other"
def exchange_of(symbol: str) -> str:
"""sh/sz/bj 前缀即交易所;无法识别返回空串。"""
s = symbol.strip().lower()
for ex in ("sh", "sz", "bj"):
if s.startswith(ex):
return ex
return ""
+116
View File
@@ -0,0 +1,116 @@
"""pytdxdata 适配器(主源,设计 §5.2)。
pytdxdata 是纯 asyncio 客户端(动态连接池),本适配器只做模型映射与调度编排;
TDX 断连/坏节点重连由 pytdxdata 内建(评审 P2),这里不重复造。
"""
from __future__ import annotations
from datetime import date, datetime
from pytdxdata import Adjust, KlinePeriod, TdxData
from pytdxdata.models import SecurityBar, SecurityInfo
from .base import Bar1dRow, CalendarRow, SecurityRow, full_year_calendar
from .classify import classify_cn, exchange_of
_MARKET_PREFIXES = ("sh", "sz", "bj")
def info_to_row(info: SecurityInfo) -> SecurityRow:
"""SecurityInfo → 统一行(category 用规则分类,kind 仅作参考不直接入库)。"""
symbol = info.symbol or ""
return SecurityRow(
symbol=symbol,
name=info.name,
category=classify_cn(symbol),
exchange=exchange_of(symbol),
currency="CNY",
lot_size=int(info.volunit) if info.volunit else None,
)
def bar_to_row(bar: SecurityBar) -> Bar1dRow:
return Bar1dRow(
symbol=bar.symbol,
date=date(bar.year, bar.month, bar.day),
open=bar.open,
high=bar.high,
low=bar.low,
close=bar.close,
volume=bar.vol,
amount=bar.amount,
)
def year_calendar_rows(year: int, trade_dates: set[date]) -> list[CalendarRow]:
"""从交易日集合合成全年日历(复用 base.full_year_calendar,§5.2)。"""
return full_year_calendar(year, trade_dates)
class TdxAdapter:
name = "tdx"
capabilities = frozenset({"securities", "bars_1d", "calendar"})
def __init__(self, client: TdxData | None = None) -> None:
self._client = client
self._owns_client = client is None
async def open(self) -> None:
if self._client is None:
self._client = TdxData()
if self._owns_client:
await self._client.start()
async def close(self) -> None:
if self._client is not None and self._owns_client:
await self._client.close()
self._owns_client = False
def _require(self) -> TdxData:
if self._client is None:
raise RuntimeError("TdxAdapter 未打开:先 await adapter.open()(或注入 client)")
return self._client
async def fetch_securities(self, market: str) -> list[SecurityRow]:
client = self._require()
if market != "cn":
return []
rows: list[SecurityRow] = []
seen: set[str] = set()
for prefix in _MARKET_PREFIXES:
for info in await client.get_universe(prefix):
if info.symbol and info.symbol not in seen:
seen.add(info.symbol)
rows.append(info_to_row(info))
return rows
async def fetch_bars_1d(
self, market: str, symbols: list[str], start: date | None, end: date | None
) -> list[Bar1dRow]:
client = self._require()
if market != "cn" or not symbols:
return []
bars = await client.get_bars(
symbols,
KlinePeriod.DAY,
adjust=Adjust.NONE, # 存不复权原始价(§4.3)
start_date=datetime(start.year, start.month, start.day) if start else None,
end_date=datetime(end.year, end.month, end.day) if end else None,
)
return [bar_to_row(b) for b in bars if b.symbol]
async def fetch_calendar(self, market: str, year: int) -> list[CalendarRow]:
"""TDX 交易日回调校验:用上证指数日K日期序列当交易日全集(§5.2)。"""
client = self._require()
if market != "cn":
return []
bars = await client.get_bars(
"sh000001",
KlinePeriod.DAY,
adjust=Adjust.NONE,
start_date=datetime(year, 1, 1),
end_date=datetime(year, 12, 31),
)
trade_dates = {date(b.year, b.month, b.day) for b in bars}
return year_calendar_rows(year, trade_dates)
+5
View File
@@ -0,0 +1,5 @@
"""SQL 闸:query_sql 安全约束(§8.3)。"""
from .gate import FORBIDDEN_FUNCTIONS, SqlRejected, validate
__all__ = ["FORBIDDEN_FUNCTIONS", "SqlRejected", "validate"]
+119
View File
@@ -0,0 +1,119 @@
"""query_sql 安全闸(设计 §8.3,评审 P0/P1 多条)。
六条闸:
1. 只读连接 + 扩展自动加载关闭(store/db.configure_query_conn);
2. 语句白名单:sqlglot fail-closed(解析失败一律拒)、dialect=duckdb、
AST 根节点判定(非字符串前缀)、多语句拒绝(parse_all 长度必须 =1);
3. 函数/表函数默认拒绝:显式拒 read_csv / read_parquet / *_scan / glob /
getenv / current_setting 等文件·网络·环境函数;FROM/JOIN 出现表函数直接拒;
``allow_functions`` 语义 = 额外放行清单(不是清空即全放行的总开关);
4. 结果上限(执行层 max_rows / arrow bytes);
5. 超时与并发闸(执行层线程池 + interrupt + 信号量);
6. 防御参数(SET max_expression_depth)。
sqlglot 进 lock pin 版本(pyproject >=,< 由 lock 冻结),升级须过本模块回归测试。
"""
from __future__ import annotations
import sqlglot
from sqlglot import exp
_DIALECT = "duckdb"
# 显式拒绝的函数(文件/网络/环境注入面),小写;`*_scan` 走后缀规则
FORBIDDEN_FUNCTIONS: frozenset[str] = frozenset(
{
"read_csv",
"read_csv_auto",
"read_parquet",
"read_json",
"read_json_auto",
"read_ndjson",
"read_text",
"glob",
"getenv",
"current_setting",
"iceberg_scan",
"delta_scan",
}
)
_SCAN_SUFFIX = "_scan"
# 允许的 AST 根节点:纯读取查询(CTE 附着在 Select 上)
_ALLOWED_ROOTS: tuple[type[exp.Expression], ...] = (
exp.Select,
exp.Union,
exp.Intersect,
exp.Except,
exp.Subquery,
)
class SqlRejected(ValueError):
"""SQL 被安全闸拒绝。reason 面向调用方,可直接进 HTTP 400 detail。"""
def __init__(self, reason: str) -> None:
self.reason = reason
super().__init__(reason)
def _func_name(node: exp.Expression) -> str | None:
if isinstance(node, exp.Anonymous):
return str(node.name).lower()
if isinstance(node, exp.Func):
return node.sql_name().lower()
return None
def _check_root(stmt: exp.Expression) -> None:
if not isinstance(stmt, _ALLOWED_ROOTS):
raise SqlRejected(f"仅允许只读查询(SELECT/UNION),收到:{type(stmt).__name__}")
def _check_from_tables(stmt: exp.Expression) -> None:
"""FROM/JOIN 出现函数(表函数)一律拒绝:库内视图都是普通表引用。"""
for table in stmt.find_all(exp.Table):
if not isinstance(table.this, exp.Identifier):
raise SqlRejected(
f"FROM/JOIN 仅允许库内普通表或视图,发现表函数:{table.sql(dialect=_DIALECT)}"
)
def _check_functions(stmt: exp.Expression, allow_functions: list[str]) -> None:
allowed = {a.lower().strip() for a in allow_functions if a.strip()}
for node in stmt.walk():
name = _func_name(node)
if name is None:
continue
if name in FORBIDDEN_FUNCTIONS and name not in allowed:
raise SqlRejected(
f"函数默认拒绝(文件/网络/环境注入面):{name};"
f"确需放行请在 config sql.allow_functions 显式加入(§8.3)"
)
if name.endswith(_SCAN_SUFFIX) and name not in allowed:
raise SqlRejected(f"函数默认拒绝(*_scan 表函数):{name}")
def parse_one(sql: str) -> exp.Expression:
"""fail-closed 解析:任何解析失败/空语句/多语句 → SqlRejected。"""
if not sql or not sql.strip():
raise SqlRejected("空 SQL")
try:
statements = sqlglot.parse(sql, read=_DIALECT)
except Exception as e: # sqlglot.errors.ParseError 及一切解析期异常
raise SqlRejected(f"SQL 解析失败(fail-closed):{e}") from e
if len(statements) != 1 or statements[0] is None:
raise SqlRejected(f"必须且只能包含一条语句,收到 {len(statements)} 条(多语句拒绝)")
return statements[0]
def validate(sql: str, allow_functions: list[str] | None = None) -> exp.Expression:
"""通过全部闸则返回解析后的 AST;否则抛 SqlRejected。"""
allow_functions = allow_functions or []
stmt = parse_one(sql)
_check_root(stmt)
_check_from_tables(stmt)
_check_functions(stmt, allow_functions)
return stmt
+23
View File
@@ -0,0 +1,23 @@
"""存储层:DuckDB 连接、migrations、联邦视图。"""
from .db import CatalogNotInitialized, configure_query_conn, open_catalog
from .migrations import (
KNOWN_MAX_VERSION,
MIGRATIONS,
SchemaTooNewError,
applied_versions,
current_version,
migrate,
)
__all__ = [
"CatalogNotInitialized",
"KNOWN_MAX_VERSION",
"MIGRATIONS",
"SchemaTooNewError",
"applied_versions",
"configure_query_conn",
"current_version",
"migrate",
"open_catalog",
]
+31
View File
@@ -0,0 +1,31 @@
"""DuckDB 连接管理:写连接(init/ETL/serve)与只读连接(查询/导出)分离(§8.3)。"""
from __future__ import annotations
from pathlib import Path
import duckdb
class CatalogNotInitialized(RuntimeError):
pass
def open_catalog(dbpath: Path, *, read_only: bool = False) -> duckdb.DuckDBPyConnection:
"""打开 catalog.duckdb。只读模式要求库已存在(未初始化给清晰报错)。"""
catalog = Path(dbpath) / "catalog.duckdb"
if read_only and not catalog.exists():
raise CatalogNotInitialized(
f"{catalog} 不存在:库未初始化,请先执行 quantdata init -d {dbpath}"
)
return duckdb.connect(str(catalog), read_only=read_only)
def configure_query_conn(
conn: duckdb.DuckDBPyConnection, *, memory_limit: str = "2GB"
) -> None:
"""查询连接安全预设(§8.3):断扩展自动加载(堵 httpfs 出站)+ 内存上限。"""
conn.execute("SET autoinstall_known_extensions=false")
conn.execute("SET autoload_known_extensions=false")
conn.execute(f"SET memory_limit='{memory_limit}'")
conn.execute("SET max_expression_depth=1000")
+259
View File
@@ -0,0 +1,259 @@
"""schema_migrations 版本管理(设计 §11 / §3.3)。
- 每个迁移在事务内执行(失败回滚);
- 库版本 > 代码已知最高版 → 报错退出(不带病跑);
- 视图 DDL 纳入迁移(评审 P1)。
"""
from __future__ import annotations
import duckdb
from .views import all_day_views
MIGRATION_0001_STATEMENTS: tuple[str, ...] = (
# ---- 参考数据 -------------------------------------------------------
"""
CREATE TABLE exchanges (
exchange_code VARCHAR PRIMARY KEY,
name VARCHAR NOT NULL,
market VARCHAR NOT NULL,
timezone VARCHAR NOT NULL,
session_json VARCHAR NOT NULL DEFAULT '{}'
)
""",
"""
CREATE TABLE securities (
market VARCHAR NOT NULL,
symbol VARCHAR NOT NULL,
name VARCHAR NOT NULL DEFAULT '',
category VARCHAR NOT NULL,
exchange VARCHAR NOT NULL DEFAULT '',
list_date DATE,
delist_date DATE,
currency VARCHAR NOT NULL DEFAULT '',
lot_size BIGINT,
underlying VARCHAR,
strike DOUBLE,
maturity DATE,
PRIMARY KEY (market, symbol)
)
""",
"""
CREATE TABLE calendar (
market VARCHAR NOT NULL,
trade_date DATE NOT NULL,
is_open BOOLEAN NOT NULL,
note VARCHAR NOT NULL DEFAULT '',
PRIMARY KEY (market, trade_date)
)
""",
"""
CREATE TABLE corp_actions (
market VARCHAR NOT NULL,
symbol VARCHAR NOT NULL,
ex_date DATE NOT NULL,
type VARCHAR NOT NULL,
ratio DOUBLE,
cash_div DOUBLE,
PRIMARY KEY (market, symbol, ex_date, type)
)
""",
# 申万 SCD2:官方发新版/追溯修订 → 开新 version(评审 P2)
"""
CREATE TABLE sw_industry (
version VARCHAR NOT NULL,
symbol VARCHAR NOT NULL,
l1 VARCHAR NOT NULL DEFAULT '',
l2 VARCHAR NOT NULL DEFAULT '',
l3 VARCHAR NOT NULL DEFAULT '',
effective_from DATE,
effective_to DATE
)
""",
"""
CREATE TABLE stock_status (
symbol VARCHAR NOT NULL,
date DATE NOT NULL,
is_st BOOLEAN NOT NULL DEFAULT FALSE,
st_reason VARCHAR NOT NULL DEFAULT '',
is_suspended BOOLEAN NOT NULL DEFAULT FALSE,
limit_up DOUBLE,
limit_down DOUBLE,
pct_limit DOUBLE,
source VARCHAR NOT NULL DEFAULT '',
PRIMARY KEY (symbol, date)
)
""",
"""
CREATE TABLE futures_meta (
symbol VARCHAR PRIMARY KEY,
contract VARCHAR NOT NULL DEFAULT '',
multiplier DOUBLE,
expiry DATE,
daily_limit_rule VARCHAR NOT NULL DEFAULT ''
)
""",
# ---- 行情(日K;1m 在 parquet,M2)-----------------------------------
"""
CREATE TABLE bars_1d (
market VARCHAR NOT NULL,
category VARCHAR NOT NULL,
symbol VARCHAR NOT NULL,
date DATE NOT NULL,
open DOUBLE NOT NULL,
high DOUBLE NOT NULL,
low DOUBLE NOT NULL,
close DOUBLE NOT NULL,
volume DOUBLE NOT NULL DEFAULT 0,
amount DOUBLE NOT NULL DEFAULT 0,
turnover DOUBLE,
adj_factor DOUBLE,
PRIMARY KEY (market, category, symbol, date)
)
""",
# ---- 信息流 ----------------------------------------------------------
"""
CREATE TABLE announcements (
id VARCHAR PRIMARY KEY,
symbol VARCHAR NOT NULL DEFAULT '',
title VARCHAR NOT NULL DEFAULT '',
category VARCHAR NOT NULL DEFAULT '',
publish_ts TIMESTAMPTZ,
url VARCHAR NOT NULL DEFAULT '',
source VARCHAR NOT NULL DEFAULT '',
hash VARCHAR NOT NULL DEFAULT ''
)
""",
"""
CREATE TABLE reports (
id VARCHAR PRIMARY KEY,
symbol VARCHAR NOT NULL DEFAULT '',
org VARCHAR NOT NULL DEFAULT '',
analyst VARCHAR NOT NULL DEFAULT '',
rating VARCHAR NOT NULL DEFAULT '',
title VARCHAR NOT NULL DEFAULT '',
publish_ts TIMESTAMPTZ,
url VARCHAR NOT NULL DEFAULT '',
source VARCHAR NOT NULL DEFAULT ''
)
""",
# 分钟新闻(NewsBridge 落库;断线补偿游标,评审 P1 补 DDL)
"""
CREATE TABLE news (
id VARCHAR PRIMARY KEY,
ts TIMESTAMPTZ NOT NULL,
source VARCHAR NOT NULL DEFAULT '',
market VARCHAR NOT NULL DEFAULT '',
symbols_json VARCHAR NOT NULL DEFAULT '[]',
level VARCHAR NOT NULL DEFAULT 'normal',
title VARCHAR NOT NULL DEFAULT '',
url VARCHAR NOT NULL DEFAULT '',
hash VARCHAR NOT NULL DEFAULT ''
)
""",
# ---- 运行时元数据 ------------------------------------------------------
"""
CREATE TABLE etl_checkpoint (
dataset VARCHAR NOT NULL,
market VARCHAR NOT NULL,
scope VARCHAR NOT NULL,
watermark VARCHAR NOT NULL,
rows BIGINT NOT NULL DEFAULT 0,
status VARCHAR NOT NULL DEFAULT 'ok',
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (dataset, market, scope)
)
""",
"""
CREATE TABLE export_jobs (
job_id VARCHAR PRIMARY KEY,
spec_json VARCHAR NOT NULL DEFAULT '{}',
fmt VARCHAR NOT NULL,
path VARCHAR NOT NULL DEFAULT '',
rows BIGINT,
bytes BIGINT,
state VARCHAR NOT NULL DEFAULT 'queued',
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
finished_at TIMESTAMPTZ
)
""",
"""
CREATE TABLE quality_reports (
date DATE NOT NULL,
market VARCHAR NOT NULL,
check_name VARCHAR NOT NULL,
result VARCHAR NOT NULL,
detail_json VARCHAR NOT NULL DEFAULT '{}',
PRIMARY KEY (date, market, check_name)
)
""",
*all_day_views(),
)
# (版本, 名称, 语句元组) —— 追加新迁移时只往列表尾部加。
# DuckDB 的 execute() 一次只吃一条语句,故迁移以「语句元组」为单位逐条执行。
MIGRATIONS: tuple[tuple[int, str, tuple[str, ...]], ...] = (
(1, "0001_initial", MIGRATION_0001_STATEMENTS),
)
KNOWN_MAX_VERSION = MIGRATIONS[-1][0]
class SchemaTooNewError(RuntimeError):
"""库 schema 版本 > 代码已知最高版:拒绝启动(§11,不带病跑)。"""
def __init__(self, db_version: int, known_max: int) -> None:
self.db_version = db_version
self.known_max = known_max
super().__init__(
f"catalog.duckdb schema 版本 {db_version} 高于代码已知最高版 {known_max}:"
f"请升级 pyquantdata 后再启动(§11 不带病跑)"
)
def _ensure_migrations_table(conn: duckdb.DuckDBPyConnection) -> None:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
"version INTEGER PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())"
)
def current_version(conn: duckdb.DuckDBPyConnection) -> int:
"""当前库版本;空库(无 schema_migrations 表)返回 0。"""
has_table = conn.execute(
"SELECT count(*) FROM information_schema.tables "
"WHERE table_schema='main' AND table_name='schema_migrations'"
).fetchone()
if not has_table or has_table[0] == 0:
return 0
row = conn.execute("SELECT max(version) FROM schema_migrations").fetchone()
return int(row[0]) if row and row[0] is not None else 0
def migrate(conn: duckdb.DuckDBPyConnection) -> int:
"""把库升到代码已知最高版,返回最终版本。幂等:已应用过则跳过。"""
_ensure_migrations_table(conn)
db_version = current_version(conn)
if db_version > KNOWN_MAX_VERSION:
raise SchemaTooNewError(db_version, KNOWN_MAX_VERSION)
for version, _name, statements in MIGRATIONS:
if version <= db_version:
continue
conn.begin()
try:
for stmt in statements:
conn.execute(stmt)
conn.execute("INSERT INTO schema_migrations (version) VALUES (?)", [version])
conn.commit()
except Exception:
conn.rollback()
raise
return current_version(conn)
def applied_versions(conn: duckdb.DuckDBPyConnection) -> list[int]:
_ensure_migrations_table(conn)
rows = conn.execute("SELECT version FROM schema_migrations ORDER BY version").fetchall()
return [int(r[0]) for r in rows]
+36
View File
@@ -0,0 +1,36 @@
"""联邦视图 DDL(设计 §3.3)。视图全部纳入 schema_migrations(评审 P1)。
规则:
- 显式列名,禁 ``*``(union/视图列错位不报错的隐患直接堵死);
- bars_1m 系视图 M2 加(1m 数据 M2 回填;视图 DDL 跟 migration 版本走)。
"""
from __future__ import annotations
_BAR_COLUMNS = (
"market, category, symbol, date, open, high, low, close, volume, amount, turnover, adj_factor"
)
# (view_name, market, category) —— M1 只做日K视图(数据在 bars_1d 物理表)
DAY_VIEWS: tuple[tuple[str, str, str], ...] = (
("cn_stock_1d", "cn", "stock"),
("cn_index_1d", "cn", "index"),
("cn_etf_1d", "cn", "etf"),
("cn_cbond_1d", "cn", "cbond"),
("us_stock_1d", "us", "stock"),
("us_index_1d", "us", "index"),
("hk_stock_1d", "hk", "stock"),
("hk_index_1d", "hk", "index"),
)
def day_view_sql(name: str, market: str, category: str) -> str:
return (
f"CREATE OR REPLACE VIEW {name} AS "
f"SELECT {_BAR_COLUMNS} FROM bars_1d "
f"WHERE market = '{market}' AND category = '{category}'"
)
def all_day_views() -> tuple[str, ...]:
return tuple(day_view_sql(*v) for v in DAY_VIEWS)