M1 最小闭环:CLI + FastAPI 网关 + SQL 闸 + 100% 覆盖率验收链
- CLI:init/serve/update/query/export/status(typer + rich),单写者锁三元判活 - 存储:DuckDB schema migrations(14 表 + 8 联邦视图,语句元组逐条执行) - ETL:A股证券/日历/日K,整区替换幂等;日历全年补全(calendar 语义唯一) - 网关:POST /v1/query(线程池 + interrupt 超时 504)、/v1/export、meta/stats/state、healthz/ready - 安全:SQL 闸 sqlglot fail-closed(文件/网络表函数默认拒)、非 loopback 空 token 拒启、导出路径双判 - 验收:254 测试全绿、1265 语句 100% 覆盖、[CI] 真起服验收链、smoke_m1.py 交付自检 - 文档:设计文档 v0.7(圆桌 45 条采纳 + 3 条实施校准),README 重写 - 发布:release.yml Trusted Publishing(OIDC 免密钥),action refs 已核验
This commit is contained in:
commit
d37c697bbe
59 files changed
+7586
No files matched your search
@@ -0,0 +1,3 @@
|
||||
"""pyquantdata: quantdata CLI —— 单目录自包含的量化数据中台。"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,6 @@
|
||||
"""python -m pyquantdata 入口。"""
|
||||
|
||||
from .cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,339 @@
|
||||
"""quantdata CLI(设计 §6):init / serve / update / query / export / status。
|
||||
|
||||
约定:``-d`` 可用环境变量 QUANTDATA_HOME 兜底;输出对齐「简洁分阶段」偏好;
|
||||
CLI 一次性命令可离线跑(serve --no-http 纯批处理)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from . import __version__
|
||||
from .config import AppConfig, default_config, load_config, write_config
|
||||
from .etl import SeedError, import_seed
|
||||
from .export.files import (
|
||||
EXPORT_FORMATS,
|
||||
write_export,
|
||||
)
|
||||
from .paths import DbPathLayout, resolve_dbpath
|
||||
from .sqlgate import SqlRejected, validate
|
||||
from .store import CatalogNotInitialized, migrate, open_catalog
|
||||
|
||||
app = typer.Typer(add_completion=False, help="quantdata:单目录自包含的量化数据中台")
|
||||
console = Console()
|
||||
err = Console(stderr=True)
|
||||
|
||||
# 容量估算表(设计 §3.4,GB):数据终态 + buffer ≥200GB ≈ 600GB 预算口径
|
||||
_CAPACITY_GB = {"cn": (150, 200), "us": (80, 90), "hk": (40, 45)}
|
||||
_BUFFER_GB = 200
|
||||
|
||||
|
||||
def estimate_capacity(markets: list[str]) -> dict:
|
||||
"""init --estimate 只算不写(§3.4)。"""
|
||||
per = {}
|
||||
lo_total = hi_total = 0
|
||||
for m in markets:
|
||||
lo, hi = _CAPACITY_GB.get(m, (0, 0))
|
||||
per[m] = {"gb_low": lo, "gb_high": hi}
|
||||
lo_total += lo
|
||||
hi_total += hi
|
||||
return {
|
||||
"per_market_gb": per,
|
||||
"data_gb": [lo_total, hi_total],
|
||||
"buffer_gb": _BUFFER_GB,
|
||||
"budget_gb": [lo_total + _BUFFER_GB, hi_total + _BUFFER_GB],
|
||||
}
|
||||
|
||||
|
||||
def _open_ro(dbpath: Path):
|
||||
"""只读打开;未初始化时打印可操作提示并 exit(1)(所有只读命令共用)。"""
|
||||
try:
|
||||
return open_catalog(dbpath, read_only=True)
|
||||
except CatalogNotInitialized as e:
|
||||
err.print(f"[red]未初始化:[/red]{e}")
|
||||
raise typer.Exit(1) from e
|
||||
|
||||
|
||||
@app.command()
|
||||
def init(
|
||||
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME", help="库目录"),
|
||||
markets: str = typer.Option("cn,us,hk", "--markets", help="启用市场,逗号分隔"),
|
||||
seed: Optional[Path] = typer.Option(None, "--seed", help="现成数据文件(parquet/csv)导入日K"),
|
||||
skip_history: bool = typer.Option(False, "--skip-history", help="跳过历史回填([CI] 层)"),
|
||||
estimate: bool = typer.Option(False, "--estimate", help="只估算磁盘预算,不写库"),
|
||||
) -> None:
|
||||
"""建目录 + config.json + schema(migrations)+ seed 导入 + 自检。幂等:重跑=续跑。"""
|
||||
dbpath = resolve_dbpath(d)
|
||||
layout = DbPathLayout(dbpath)
|
||||
market_list = [m.strip() for m in markets.split(",") if m.strip()]
|
||||
if estimate:
|
||||
console.print_json(json.dumps(estimate_capacity(market_list)))
|
||||
raise typer.Exit(0)
|
||||
|
||||
console.print(f"[bold]quantdata init[/bold] → {dbpath}")
|
||||
console.print(" [1/4] 目录结构")
|
||||
layout.ensure_dirs()
|
||||
config_file = layout.config_file
|
||||
if config_file.exists():
|
||||
cfg = load_config(config_file)
|
||||
console.print(" [2/4] config.json 已存在,保留(幂等)")
|
||||
else:
|
||||
cfg = default_config(market_list)
|
||||
write_config(cfg, config_file)
|
||||
console.print(f" [2/4] config.json 生成(markets={market_list})")
|
||||
|
||||
conn = open_catalog(layout.root)
|
||||
try:
|
||||
console.print(" [3/4] schema migrations")
|
||||
version = migrate(conn)
|
||||
console.print(f" schema 版本 = {version}")
|
||||
|
||||
console.print(" [4/4] 自检")
|
||||
seed_report = None
|
||||
if seed is not None:
|
||||
try:
|
||||
seed_report = import_seed(conn, seed)
|
||||
except SeedError as e:
|
||||
err.print(f"[red]seed 导入失败:[/red]{e}")
|
||||
raise typer.Exit(1) from e
|
||||
_print_selfcheck(conn, layout, seed_report)
|
||||
finally:
|
||||
conn.close()
|
||||
if skip_history:
|
||||
console.print("[dim]--skip-history:历史回填跳过([CI] 层验收口径)[/dim]")
|
||||
console.print("[green]init 完成[/green]")
|
||||
|
||||
|
||||
def _print_selfcheck(conn, layout: DbPathLayout, seed_report) -> None:
|
||||
tables = conn.execute(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_schema='main' "
|
||||
"AND table_type='BASE TABLE'"
|
||||
).fetchone()[0]
|
||||
views = conn.execute(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_schema='main' "
|
||||
"AND table_type='VIEW'"
|
||||
).fetchone()[0]
|
||||
bars_rows = conn.execute("SELECT count(*) FROM bars_1d").fetchone()[0]
|
||||
console.print(f" tables={tables} views={views} bars_1d rows={bars_rows}")
|
||||
if seed_report is not None:
|
||||
console.print(
|
||||
f" seed: {seed_report.rows} 行 / {len(seed_report.symbols)} symbols "
|
||||
f"[{seed_report.date_min} ~ {seed_report.date_max}]"
|
||||
)
|
||||
|
||||
|
||||
@app.command()
|
||||
def serve(
|
||||
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME"),
|
||||
host: Optional[str] = typer.Option(None, "--host", help="监听地址(默认 config http.host)"),
|
||||
port: Optional[int] = typer.Option(None, "--port", help="监听端口(默认 config http.port)"),
|
||||
no_http: bool = typer.Option(False, "--no-http", help="纯 CLI 批处理,不起网关"),
|
||||
insecure: bool = typer.Option(False, "--insecure", help="显式允许非 loopback 且无 token(§8.4)"),
|
||||
) -> None:
|
||||
"""常驻:单写者锁 + 调度 + FastAPI 网关(恒前台,Linux 交 supervisord)。"""
|
||||
from .serve.runtime import run_serve
|
||||
|
||||
dbpath = resolve_dbpath(d)
|
||||
code = asyncio.run(
|
||||
run_serve(dbpath, host=host, port=port, no_http=no_http, insecure=insecure)
|
||||
)
|
||||
raise typer.Exit(code)
|
||||
|
||||
|
||||
@app.command()
|
||||
def update(
|
||||
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME"),
|
||||
dataset: str = typer.Option("all", "--dataset", help="securities,calendar,bars_1d 或 all"),
|
||||
market: str = typer.Option("cn", "--market"),
|
||||
symbols: Optional[str] = typer.Option(None, "--symbols", help="逗号分隔,缺省=securities 全量"),
|
||||
start: Optional[str] = typer.Option(None, "--start", help="日K 起始 YYYY-MM-DD"),
|
||||
end: Optional[str] = typer.Option(None, "--end", help="日K 结束 YYYY-MM-DD"),
|
||||
) -> None:
|
||||
"""单次更新(cron 兜底):主源 pytdxdata。幂等,重跑安全。"""
|
||||
from .sources import TdxAdapter
|
||||
|
||||
dbpath = resolve_dbpath(d)
|
||||
layout = DbPathLayout(dbpath)
|
||||
conn = open_catalog(layout.root)
|
||||
try:
|
||||
migrate(conn)
|
||||
datasets = (
|
||||
["securities", "calendar", "bars_1d"] if dataset == "all" else
|
||||
[x.strip() for x in dataset.split(",") if x.strip()]
|
||||
)
|
||||
sym_list = [s.strip() for s in symbols.split(",")] if symbols else None
|
||||
adapter = TdxAdapter()
|
||||
console.print(f"[bold]quantdata update[/bold] market={market} datasets={datasets}")
|
||||
|
||||
# 适配器 open/close 与 ETL 必须在同一事件循环(pytdxdata 纯 asyncio)
|
||||
from .etl.update import run_update as _run_update
|
||||
|
||||
async def _run():
|
||||
await adapter.open()
|
||||
try:
|
||||
return await _run_update(
|
||||
conn, market, adapter,
|
||||
datasets=datasets, symbols=sym_list,
|
||||
start=date.fromisoformat(start) if start else None,
|
||||
end=date.fromisoformat(end) if end else None,
|
||||
)
|
||||
finally:
|
||||
await adapter.close()
|
||||
|
||||
reports = asyncio.run(_run())
|
||||
for r in reports:
|
||||
console.print(
|
||||
f" {r.dataset}: rows={r.rows} symbols={len(r.symbols)} "
|
||||
f"range=[{r.date_min} ~ {r.date_max}]"
|
||||
)
|
||||
console.print("[green]update 完成[/green]")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@app.command()
|
||||
def query(
|
||||
sql: str = typer.Argument(..., help="SQL(仅只读查询)"),
|
||||
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME"),
|
||||
fmt: str = typer.Option("table", "--fmt", help="table|json|csv|parquet|arrow"),
|
||||
o: Optional[Path] = typer.Option(None, "-o", help="csv/parquet/arrow 输出文件"),
|
||||
) -> None:
|
||||
"""本地直查(只读连接,不开服务也可用)。"""
|
||||
if fmt not in ("table", "json", "csv", "parquet", "arrow"):
|
||||
err.print(f"[red]--fmt 不支持:{fmt}[/red]")
|
||||
raise typer.Exit(1)
|
||||
if fmt in ("csv", "parquet", "arrow") and o is None:
|
||||
err.print(f"[red]--fmt {fmt} 需要 -o 输出文件[/red]")
|
||||
raise typer.Exit(1)
|
||||
dbpath = resolve_dbpath(d)
|
||||
conn = _open_ro(dbpath)
|
||||
try:
|
||||
from .store.db import configure_query_conn
|
||||
|
||||
configure_query_conn(conn)
|
||||
try:
|
||||
validate(sql) # 本地 CLI 也过闸:与 HTTP 同规则(§8.3)
|
||||
except SqlRejected as e:
|
||||
err.print(f"[red]SQL 被拒绝:[/red]{e.reason}")
|
||||
raise typer.Exit(1) from e
|
||||
tbl = conn.execute(sql).arrow().read_all()
|
||||
if fmt == "table":
|
||||
table = Table(*tbl.schema.names)
|
||||
for row in table_to_rows(tbl):
|
||||
table.add_row(*[str(v) for v in row])
|
||||
console.print(table)
|
||||
console.print(f"[dim]{tbl.num_rows} rows[/dim]")
|
||||
elif fmt == "json":
|
||||
# 纯 stdout JSON(无 rich 前缀/换行噪声),便于 `| jq` 与程序消费
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"columns": tbl.schema.names,
|
||||
"dtypes": [str(f.type) for f in tbl.schema],
|
||||
"rows": table_to_rows(tbl),
|
||||
"row_count": tbl.num_rows,
|
||||
},
|
||||
default=str,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
else:
|
||||
result = write_export(tbl, "csv" if fmt == "csv" else fmt, Path(o))
|
||||
console.print(f"[green]已写出[/green] {result.path}({result.rows} 行 / {result.bytes_written} 字节)")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def table_to_rows(tbl):
|
||||
if tbl.num_columns == 0:
|
||||
return []
|
||||
cols = [c.to_pylist() for c in tbl.columns]
|
||||
return [list(row) for row in zip(*cols)]
|
||||
|
||||
|
||||
@app.command()
|
||||
def export(
|
||||
sql: str = typer.Option(..., "--sql", help="导出查询"),
|
||||
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME"),
|
||||
fmt: str = typer.Option("csv", "--fmt", help="csv|parquet"),
|
||||
o: Path = typer.Option(..., "-o", help="输出文件"),
|
||||
) -> None:
|
||||
"""导出落盘格式(csv/parquet);pandas/polars 走 query --fmt arrow(§9)。"""
|
||||
if fmt not in EXPORT_FORMATS:
|
||||
err.print(f"[red]export 仅支持 {sorted(EXPORT_FORMATS)};pandas/polars 走 query --fmt arrow[/red]")
|
||||
raise typer.Exit(1)
|
||||
dbpath = resolve_dbpath(d)
|
||||
conn = _open_ro(dbpath)
|
||||
try:
|
||||
from .store.db import configure_query_conn
|
||||
|
||||
configure_query_conn(conn)
|
||||
try:
|
||||
validate(sql)
|
||||
except SqlRejected as e:
|
||||
err.print(f"[red]SQL 被拒绝:[/red]{e.reason}")
|
||||
raise typer.Exit(1) from e
|
||||
tbl = conn.execute(sql).arrow().read_all()
|
||||
result = write_export(tbl, fmt, Path(o))
|
||||
console.print(f"[green]导出完成[/green] {result.path}({result.rows} 行 / {result.bytes_written} 字节)")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@app.command()
|
||||
def status(
|
||||
d: Optional[str] = typer.Option(None, "--dbpath", "-d", envvar="QUANTDATA_HOME"),
|
||||
json_out: bool = typer.Option(False, "--json", help="结构化 JSON 输出(脚本化)"),
|
||||
) -> None:
|
||||
"""水位、锁、HTTP 配置、磁盘占用(运维入口)。"""
|
||||
from .lockfile import is_alive, read_lock
|
||||
from .store import current_version
|
||||
|
||||
dbpath = resolve_dbpath(d)
|
||||
layout = DbPathLayout(dbpath)
|
||||
info: dict = {"dbpath": str(layout.root)}
|
||||
lock = read_lock(layout.lock_file)
|
||||
info["lock"] = (
|
||||
{"held": True, "pid": lock.pid, "alive": is_alive(lock)} if lock else {"held": False}
|
||||
)
|
||||
info["http"] = {"enabled": True, "host": "127.0.0.1", "port": 8765, "note": "config 段 http"}
|
||||
conn = _open_ro(layout.root)
|
||||
try:
|
||||
info["schema_version"] = current_version(conn)
|
||||
cps = conn.execute(
|
||||
"SELECT dataset, market, scope, watermark, rows, status FROM etl_checkpoint "
|
||||
"ORDER BY dataset, market, scope"
|
||||
).fetchall()
|
||||
info["checkpoints"] = [
|
||||
{"dataset": r[0], "market": r[1], "scope": r[2], "watermark": r[3], "rows": r[4], "status": r[5]}
|
||||
for r in cps
|
||||
]
|
||||
info["bars_1d_rows"] = conn.execute("SELECT count(*) FROM bars_1d").fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
import psutil
|
||||
|
||||
du = psutil.disk_usage(str(layout.root))
|
||||
info["disk"] = {"used_pct": round(du.percent, 1), "free_gb": round(du.free / 1e9, 1)}
|
||||
except Exception: # noqa: BLE001 — 磁盘信息尽力而为
|
||||
info["disk"] = None
|
||||
# --json 与默认都输出同一份 JSON(默认人类可读,--json 供程序消费;M1 共用渲染)
|
||||
console.print_json(json.dumps(info, default=str, ensure_ascii=False))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
app()
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
main()
|
||||
@@ -0,0 +1,190 @@
|
||||
"""config.json 模型与读写(设计 §12)。JSON 格式;extra=forbid 防拼错字段静默失效。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
_ENV_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
||||
|
||||
|
||||
def expand_env(value: str) -> str:
|
||||
"""展开 ``${ENV_VAR}``(设计 §12:http.token 支持环境变量展开,不明文入库)。"""
|
||||
return _ENV_PATTERN.sub(lambda m: os_environ_get(m.group(1)), value)
|
||||
|
||||
|
||||
def os_environ_get(name: str) -> str:
|
||||
import os
|
||||
|
||||
return os.environ.get(name, "")
|
||||
|
||||
|
||||
class MarketConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
enabled: bool = True
|
||||
history_from: str
|
||||
history_from_bars_1d: str | None = None # null 时跟随 history_from(§3.4)
|
||||
|
||||
|
||||
class HttpConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
enabled: bool = True
|
||||
host: str = "127.0.0.1"
|
||||
port: int = 8765
|
||||
token: str = ""
|
||||
max_ws_conns: int = 64
|
||||
ws_queue_len: int = 256
|
||||
|
||||
|
||||
class SourcesConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
bars: str = "tdx"
|
||||
realtime: str = "westock"
|
||||
reference: list[str] = Field(default_factory=lambda: ["tdx", "westock"])
|
||||
futures: list[str] = Field(default_factory=lambda: ["tdx", "yfinance"])
|
||||
news: list[str] = Field(default_factory=lambda: ["scripts/fetch_news.py"])
|
||||
fallback: list[str] = Field(default_factory=lambda: ["akshare_port"])
|
||||
cross_check: bool = True
|
||||
cross_check_ratio: float = 0.01
|
||||
|
||||
|
||||
class ProxyConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
global_: str = Field(default="", alias="global")
|
||||
per_source: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class FetchConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
rate_limit_rps: dict[str, float] = Field(default_factory=dict)
|
||||
timeout_ms: int = 10000
|
||||
max_retry: int = 3
|
||||
|
||||
|
||||
class DailyChainConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
cn: str = "15:05"
|
||||
hk: str = "16:10"
|
||||
us: str = "04:30"
|
||||
|
||||
|
||||
class ScheduleConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
timezone: str = "Asia/Shanghai" # 显式绑定,不读系统 TZ(评审 P1)
|
||||
cn_intraday_interval_s: int = 90
|
||||
snapshot_interval_s: int = 3
|
||||
daily_chain: DailyChainConfig = Field(default_factory=DailyChainConfig)
|
||||
compaction: str = "monthly"
|
||||
|
||||
|
||||
class QualityConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
enabled: bool = True
|
||||
checks: list[str] = Field(
|
||||
default_factory=lambda: ["gap", "ohlc", "limit", "adj", "rowcount", "dualsource"]
|
||||
)
|
||||
|
||||
|
||||
class SqlConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
max_rows: int = 1_000_000
|
||||
timeout_ms: int = 30_000
|
||||
max_arrow_bytes: int = 64 * 1024 * 1024
|
||||
max_concurrent: int = 4
|
||||
acquire_timeout_s: float = 5.0 # 信号量等待超过即 429(§8.3)
|
||||
memory_limit: str = "2GB"
|
||||
allow_functions: list[str] = Field(default_factory=list) # 额外放行清单(§8.3)
|
||||
|
||||
|
||||
class ExportConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
out_dir: str = "exports"
|
||||
cache_ttl_h: int = 24
|
||||
keep_days: int = 7
|
||||
formats: list[str] = Field(default_factory=lambda: ["csv", "parquet", "qlib", "backtrader"])
|
||||
|
||||
|
||||
class LogConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
level: str = "INFO"
|
||||
keep_days: int = 30
|
||||
|
||||
|
||||
class AppConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
version: int = 1
|
||||
dbpath_created: str
|
||||
markets: dict[str, MarketConfig]
|
||||
categories: dict[str, list[str]]
|
||||
sources: SourcesConfig = Field(default_factory=SourcesConfig)
|
||||
proxy: ProxyConfig = Field(default_factory=ProxyConfig)
|
||||
fetch: FetchConfig = Field(default_factory=FetchConfig)
|
||||
http: HttpConfig = Field(default_factory=HttpConfig)
|
||||
schedule: ScheduleConfig = Field(default_factory=ScheduleConfig)
|
||||
quality: QualityConfig = Field(default_factory=QualityConfig)
|
||||
sql: SqlConfig = Field(default_factory=SqlConfig)
|
||||
export: ExportConfig = Field(default_factory=ExportConfig)
|
||||
log: LogConfig = Field(default_factory=LogConfig)
|
||||
|
||||
|
||||
_DEFAULT_CATEGORIES = {
|
||||
"cn": ["index", "stock", "future", "cbond", "etf", "reits", "option"],
|
||||
"us": ["index", "stock", "future", "etf", "reits"],
|
||||
"hk": ["index", "stock", "future", "etf", "reits"],
|
||||
}
|
||||
|
||||
|
||||
def default_config(markets: list[str] | None = None, now: datetime | None = None) -> AppConfig:
|
||||
"""init 生成的默认 config(§12)。markets 默认三市场全开。"""
|
||||
markets = markets or ["cn", "us", "hk"]
|
||||
defaults = {
|
||||
"cn": ("2004-01-01", None),
|
||||
"us": ("2020-01-01", None),
|
||||
"hk": ("2020-01-01", None),
|
||||
}
|
||||
now = now or datetime.now(timezone.utc)
|
||||
return AppConfig(
|
||||
dbpath_created=now.isoformat(),
|
||||
markets={
|
||||
m: MarketConfig(enabled=True, history_from=defaults[m][0], history_from_bars_1d=None)
|
||||
for m in markets
|
||||
},
|
||||
categories={m: list(_DEFAULT_CATEGORIES[m]) for m in markets},
|
||||
)
|
||||
|
||||
|
||||
def write_config(config: AppConfig, path: Path) -> None:
|
||||
path.write_text(config.model_dump_json(indent=2, by_alias=True), encoding="utf-8")
|
||||
|
||||
|
||||
def _expand_strings(obj: Any) -> Any:
|
||||
if isinstance(obj, str):
|
||||
return expand_env(obj)
|
||||
if isinstance(obj, list):
|
||||
return [_expand_strings(v) for v in obj]
|
||||
if isinstance(obj, dict):
|
||||
return {k: _expand_strings(v) for k, v in obj.items()}
|
||||
return obj
|
||||
|
||||
|
||||
def load_config(path: Path) -> AppConfig:
|
||||
"""读 config.json 并展开 ${ENV};字段拼错/多余直接报错(extra=forbid)。"""
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
return AppConfig.model_validate(_expand_strings(raw))
|
||||
@@ -0,0 +1,19 @@
|
||||
"""ETL:checkpoint、seed 导入、update 编排。"""
|
||||
|
||||
from .checkpoint import failed_checkpoints, list_checkpoints, upsert_checkpoint
|
||||
from .seed import SeedError, SeedReport, import_seed
|
||||
from .update import run_update, run_update_sync, update_bars_1d, update_calendar, update_securities
|
||||
|
||||
__all__ = [
|
||||
"SeedError",
|
||||
"SeedReport",
|
||||
"failed_checkpoints",
|
||||
"import_seed",
|
||||
"list_checkpoints",
|
||||
"run_update",
|
||||
"run_update_sync",
|
||||
"update_bars_1d",
|
||||
"update_calendar",
|
||||
"update_securities",
|
||||
"upsert_checkpoint",
|
||||
]
|
||||
@@ -0,0 +1,45 @@
|
||||
"""etl_checkpoint 断点与水位(设计 §7.4)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import duckdb
|
||||
|
||||
|
||||
def upsert_checkpoint(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
dataset: str,
|
||||
market: str,
|
||||
scope: str,
|
||||
watermark: str,
|
||||
rows: int,
|
||||
status: str = "ok",
|
||||
) -> None:
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO etl_checkpoint "
|
||||
"(dataset, market, scope, watermark, rows, status, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, now())",
|
||||
[dataset, market, scope, watermark, rows, status],
|
||||
)
|
||||
|
||||
|
||||
def list_checkpoints(conn: duckdb.DuckDBPyConnection) -> list[dict]:
|
||||
rows = conn.execute(
|
||||
"SELECT dataset, market, scope, watermark, rows, status, updated_at "
|
||||
"FROM etl_checkpoint ORDER BY dataset, market, scope"
|
||||
).fetchall()
|
||||
return [
|
||||
{
|
||||
"dataset": r[0],
|
||||
"market": r[1],
|
||||
"scope": r[2],
|
||||
"watermark": r[3],
|
||||
"rows": r[4],
|
||||
"status": r[5],
|
||||
"updated_at": None if r[6] is None else r[6].isoformat(),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def failed_checkpoints(conn: duckdb.DuckDBPyConnection) -> list[dict]:
|
||||
return [c for c in list_checkpoints(conn) if c["status"] != "ok"]
|
||||
@@ -0,0 +1,142 @@
|
||||
"""seed 导入(init --seed):把现成数据(如 stock_prediction 的 A股日K)搬进库。
|
||||
|
||||
- 支持列名:symbol, date, open, high, low, close, volume(=vol), amount(=amt),
|
||||
turnover, adj_factor(后两者可选);
|
||||
- 幂等:事务内 DELETE 覆盖区(同 symbol 的 [min,max] 日期)后 INSERT;
|
||||
- parquet / csv 由扩展名路由。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import pyarrow as pa
|
||||
import pyarrow.csv as pacsv
|
||||
import pyarrow.parquet as paparquet
|
||||
|
||||
from ..sources.classify import classify_cn
|
||||
|
||||
_COLUMN_ALIASES = {
|
||||
"symbol": "symbol",
|
||||
"date": "date",
|
||||
"open": "open",
|
||||
"high": "high",
|
||||
"low": "low",
|
||||
"close": "close",
|
||||
"volume": "volume",
|
||||
"vol": "volume",
|
||||
"amount": "amount",
|
||||
"amt": "amount",
|
||||
"turnover": "turnover",
|
||||
"adj_factor": "adj_factor",
|
||||
}
|
||||
_REQUIRED = ("symbol", "date", "open", "high", "low", "close")
|
||||
|
||||
|
||||
class SeedError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SeedReport:
|
||||
path: str
|
||||
rows: int
|
||||
symbols: list[str]
|
||||
date_min: date | None
|
||||
date_max: date | None
|
||||
|
||||
|
||||
def _read_table(path: Path) -> pa.Table:
|
||||
suffix = path.suffix.lower()
|
||||
if suffix == ".parquet":
|
||||
return paparquet.read_table(path)
|
||||
if suffix == ".csv":
|
||||
return pacsv.read_csv(path)
|
||||
raise SeedError(f"seed 仅支持 .parquet/.csv,收到 {path.name}")
|
||||
|
||||
|
||||
def _as_date(v) -> date:
|
||||
if isinstance(v, datetime):
|
||||
return v.date()
|
||||
if isinstance(v, date):
|
||||
return v
|
||||
return date.fromisoformat(str(v)[:10])
|
||||
|
||||
|
||||
def _rows_from_table(table: pa.Table) -> list[tuple]:
|
||||
cols = {}
|
||||
for name in table.column_names:
|
||||
canonical = _COLUMN_ALIASES.get(name.strip().lower())
|
||||
if canonical:
|
||||
cols[canonical] = table.column(name).to_pylist()
|
||||
missing = [c for c in _REQUIRED if c not in cols]
|
||||
if missing:
|
||||
raise SeedError(f"seed 缺必需列:{missing}(实际列:{table.column_names})")
|
||||
n = len(cols["symbol"])
|
||||
out: list[tuple] = []
|
||||
for i in range(n):
|
||||
symbol = str(cols["symbol"][i]).strip().lower()
|
||||
if not symbol:
|
||||
raise SeedError(f"seed 第 {i} 行 symbol 为空")
|
||||
out.append(
|
||||
(
|
||||
symbol,
|
||||
_as_date(cols["date"][i]),
|
||||
float(cols["open"][i]),
|
||||
float(cols["high"][i]),
|
||||
float(cols["low"][i]),
|
||||
float(cols["close"][i]),
|
||||
float(cols["volume"][i] or 0),
|
||||
float(cols["amount"][i] or 0),
|
||||
None if cols.get("turnover", [None] * n)[i] is None else float(cols["turnover"][i]),
|
||||
None if cols.get("adj_factor", [None] * n)[i] is None else float(cols["adj_factor"][i]),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def import_seed(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
seed_path: Path | str,
|
||||
*,
|
||||
market: str = "cn",
|
||||
category: str | None = None,
|
||||
) -> SeedReport:
|
||||
"""导入 seed 到 bars_1d。category 缺省按 symbol 规则分类(A股)。"""
|
||||
path = Path(seed_path)
|
||||
if not path.exists():
|
||||
raise SeedError(f"seed 文件不存在:{path}")
|
||||
rows = _rows_from_table(_read_table(path))
|
||||
if not rows:
|
||||
raise SeedError(f"seed 无数据行:{path}")
|
||||
conn.begin()
|
||||
try:
|
||||
symbols = sorted({r[0] for r in rows})
|
||||
dates = [r[1] for r in rows]
|
||||
for symbol in symbols:
|
||||
sym_rows = [r for r in rows if r[0] == symbol]
|
||||
cat = category or classify_cn(symbol)
|
||||
conn.execute(
|
||||
"DELETE FROM bars_1d WHERE market=? AND category=? AND symbol=? "
|
||||
"AND date BETWEEN ? AND ?",
|
||||
[market, cat, symbol, min(dates), max(dates)],
|
||||
)
|
||||
conn.executemany(
|
||||
"INSERT INTO bars_1d (market, category, symbol, date, open, high, low, "
|
||||
"close, volume, amount, turnover, adj_factor) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
[(market, cat, *r) for r in sym_rows],
|
||||
)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
return SeedReport(
|
||||
path=str(path),
|
||||
rows=len(rows),
|
||||
symbols=symbols,
|
||||
date_min=min(dates),
|
||||
date_max=max(dates),
|
||||
)
|
||||
@@ -0,0 +1,166 @@
|
||||
"""update 命令的 ETL 编排(设计 §6/§7.3)。
|
||||
|
||||
幂等模式:以 (market, symbol, 日期区间) 粒度整区替换(事务内 DELETE+INSERT),
|
||||
重跑任意次数结果一致。CLI 一次性场景允许事件循环内同步 DuckDB 调用
|
||||
(§7.6 的线程池隔离约束针对 serve 常驻场景,M2 接入调度器时迁移)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import date
|
||||
|
||||
import duckdb
|
||||
|
||||
from ..sources.base import SourceAdapter, UpdateReport, full_year_calendar
|
||||
from ..sources.classify import classify_cn
|
||||
from .checkpoint import upsert_checkpoint
|
||||
|
||||
_CHUNK = 400 # 每次 fetch 的 symbol 数(TDX 一次全拉列表太大时分块)
|
||||
|
||||
|
||||
async def update_securities(
|
||||
conn: duckdb.DuckDBPyConnection, market: str, adapter: SourceAdapter
|
||||
) -> UpdateReport:
|
||||
rows = await adapter.fetch_securities(market)
|
||||
conn.begin()
|
||||
try:
|
||||
conn.execute("DELETE FROM securities WHERE market=?", [market])
|
||||
conn.executemany(
|
||||
"INSERT INTO securities (market, symbol, name, category, exchange, currency, lot_size) "
|
||||
"VALUES (?,?,?,?,?,?,?)",
|
||||
[
|
||||
(market, r.symbol, r.name, r.category, r.exchange, r.currency, r.lot_size)
|
||||
for r in rows
|
||||
],
|
||||
)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
upsert_checkpoint(conn, "securities", market, "all", "", len(rows))
|
||||
return UpdateReport(dataset="securities", market=market, symbols=[r.symbol for r in rows], rows=len(rows))
|
||||
|
||||
|
||||
async def update_calendar(
|
||||
conn: duckdb.DuckDBPyConnection, market: str, adapter: SourceAdapter, years: list[int]
|
||||
) -> UpdateReport:
|
||||
"""calendar 落库恒为「整年逐日」:adapter 只需给出交易日(§5.2)。
|
||||
|
||||
源(tdx/fake)给出的是交易日集合,本函数负责补全全年(非交易日 is_open=False),
|
||||
保证 calendar 表的语义唯一:任意一天查得到、is_open 可信。
|
||||
"""
|
||||
total = 0
|
||||
for year in years:
|
||||
rows = await adapter.fetch_calendar(market, year)
|
||||
trade_dates = {r.trade_date for r in rows if r.is_open}
|
||||
full = full_year_calendar(year, trade_dates)
|
||||
conn.begin()
|
||||
try:
|
||||
conn.execute(
|
||||
"DELETE FROM calendar WHERE market=? AND trade_date BETWEEN ? AND ?",
|
||||
[market, date(year, 1, 1), date(year, 12, 31)],
|
||||
)
|
||||
conn.executemany(
|
||||
"INSERT INTO calendar (market, trade_date, is_open, note) VALUES (?,?,?,?)",
|
||||
[(market, r.trade_date, r.is_open, r.note) for r in full],
|
||||
)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
total += len(full)
|
||||
open_rows = sum(1 for r in full if r.is_open)
|
||||
upsert_checkpoint(conn, "calendar", market, str(year), date(year, 12, 31).isoformat(), open_rows)
|
||||
return UpdateReport(dataset="calendar", market=market, rows=total)
|
||||
|
||||
|
||||
async def update_bars_1d(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
market: str,
|
||||
adapter: SourceAdapter,
|
||||
*,
|
||||
symbols: list[str] | None = None,
|
||||
start: date | None = None,
|
||||
end: date | None = None,
|
||||
) -> UpdateReport:
|
||||
if symbols is None:
|
||||
rows = conn.execute(
|
||||
"SELECT symbol FROM securities WHERE market=? "
|
||||
"AND category IN ('stock','index','etf','cbond') ORDER BY symbol",
|
||||
[market],
|
||||
).fetchall()
|
||||
symbols = [r[0] for r in rows]
|
||||
if not symbols:
|
||||
raise RuntimeError(
|
||||
"securities 表为空:请先 update --dataset securities 建证券主数据"
|
||||
)
|
||||
total_rows = 0
|
||||
seen_dates: list[date] = []
|
||||
for i in range(0, len(symbols), _CHUNK):
|
||||
chunk = symbols[i : i + _CHUNK]
|
||||
bars = await adapter.fetch_bars_1d(market, chunk, start, end)
|
||||
conn.begin()
|
||||
try:
|
||||
for symbol in chunk:
|
||||
sym_bars = [b for b in bars if b.symbol == symbol]
|
||||
if not sym_bars:
|
||||
continue
|
||||
category = classify_cn(symbol) if market == "cn" else "stock"
|
||||
d_min, d_max = min(b.date for b in sym_bars), max(b.date for b in sym_bars)
|
||||
conn.execute(
|
||||
"DELETE FROM bars_1d WHERE market=? AND category=? AND symbol=? "
|
||||
"AND date BETWEEN ? AND ?",
|
||||
[market, category, symbol, d_min, d_max],
|
||||
)
|
||||
conn.executemany(
|
||||
"INSERT INTO bars_1d (market, category, symbol, date, open, high, low, "
|
||||
"close, volume, amount, turnover, adj_factor) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
[
|
||||
(market, category, b.symbol, b.date, b.open, b.high, b.low, b.close,
|
||||
b.volume, b.amount, b.turnover, b.adj_factor)
|
||||
for b in sym_bars
|
||||
],
|
||||
)
|
||||
total_rows += len(sym_bars)
|
||||
seen_dates.extend([b.date for b in sym_bars])
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
watermark = max(seen_dates).isoformat() if seen_dates else ""
|
||||
upsert_checkpoint(conn, "bars_1d", market, "daily", watermark, total_rows)
|
||||
return UpdateReport(
|
||||
dataset="bars_1d", market=market, symbols=symbols, rows=total_rows,
|
||||
date_min=min(seen_dates) if seen_dates else None,
|
||||
date_max=max(seen_dates) if seen_dates else None,
|
||||
)
|
||||
|
||||
|
||||
async def run_update(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
market: str,
|
||||
adapter: SourceAdapter,
|
||||
*,
|
||||
datasets: list[str],
|
||||
symbols: list[str] | None = None,
|
||||
start: date | None = None,
|
||||
end: date | None = None,
|
||||
years: list[int] | None = None,
|
||||
) -> list[UpdateReport]:
|
||||
"""按依赖顺序编排:securities → calendar → bars_1d。"""
|
||||
reports: list[UpdateReport] = []
|
||||
if "securities" in datasets:
|
||||
reports.append(await update_securities(conn, market, adapter))
|
||||
if "calendar" in datasets:
|
||||
reports.append(await update_calendar(conn, market, adapter, years or [date.today().year]))
|
||||
if "bars_1d" in datasets:
|
||||
reports.append(
|
||||
await update_bars_1d(conn, market, adapter, symbols=symbols, start=start, end=end)
|
||||
)
|
||||
return reports
|
||||
|
||||
|
||||
def run_update_sync(conn: duckdb.DuckDBPyConnection, market: str, adapter: SourceAdapter, **kw) -> list[UpdateReport]:
|
||||
"""CLI 同步入口。"""
|
||||
return asyncio.run(run_update(conn, market, adapter, **kw))
|
||||
@@ -0,0 +1,29 @@
|
||||
"""导出:csv/parquet 落盘(pandas/polars 走 /v1/query fmt=arrow)。"""
|
||||
|
||||
from .files import (
|
||||
EXPORT_FORMATS,
|
||||
ExportFormatError,
|
||||
ExportPathError,
|
||||
ExportResult,
|
||||
get_export_job,
|
||||
new_job_id,
|
||||
record_export_job,
|
||||
resolve_export_target,
|
||||
run_query_arrow,
|
||||
spec_hash,
|
||||
write_export,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"EXPORT_FORMATS",
|
||||
"ExportFormatError",
|
||||
"ExportPathError",
|
||||
"ExportResult",
|
||||
"get_export_job",
|
||||
"new_job_id",
|
||||
"record_export_job",
|
||||
"resolve_export_target",
|
||||
"run_query_arrow",
|
||||
"spec_hash",
|
||||
"write_export",
|
||||
]
|
||||
@@ -0,0 +1,133 @@
|
||||
"""导出落盘(设计 §9)。
|
||||
|
||||
- export 只管落盘格式:csv / parquet(pandas/polars 走 /v1/query fmt=arrow 消费);
|
||||
- 路径安全(评审 P1):输出根恒为 exports/{job_id}/,名称字段白名单字符过滤 +
|
||||
resolve 后必须仍在 exports/ 前缀内;杜绝 ../ 穿出 dbpath;
|
||||
- job 结果记 export_jobs 表(spec hash 缓存 24h 的键在 M2 worker 化时接入)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import pyarrow as pa
|
||||
import pyarrow.csv as pacsv
|
||||
import pyarrow.parquet as paparquet
|
||||
|
||||
_JOB_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$")
|
||||
_FILENAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
|
||||
EXPORT_FORMATS: frozenset[str] = frozenset({"csv", "parquet"})
|
||||
|
||||
|
||||
class ExportPathError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ExportFormatError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def new_job_id() -> str:
|
||||
return uuid.uuid4().hex[:12]
|
||||
|
||||
|
||||
def spec_hash(spec: dict) -> str:
|
||||
"""导出请求的稳定指纹(缓存键,§9)。"""
|
||||
canonical = json.dumps(spec, sort_keys=True, ensure_ascii=False)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def resolve_export_target(exports_dir: Path, job_id: str, filename: str) -> Path:
|
||||
"""exports/{job_id}/{filename};白名单 + resolve 前缀校验(§9 路径安全)。
|
||||
|
||||
纵深防御:``_JOB_ID_RE`` / ``_FILENAME_RE`` 已禁掉 ``/`` 与 ``..``,因此常规
|
||||
输入下 resolve 后必然在 exports/ 内。最后的 startswith 校验是为**符号链接**
|
||||
(exports/{job_id} 被替换成指向外部的软链)兜底——该分支在无软链的环境里
|
||||
结构性不可达,保留为防线而非死代码。
|
||||
"""
|
||||
if not _JOB_ID_RE.match(job_id):
|
||||
raise ExportPathError(f"非法 job_id:{job_id!r}")
|
||||
if not _FILENAME_RE.match(filename):
|
||||
raise ExportPathError(f"非法导出文件名:{filename!r}")
|
||||
exports_dir = Path(exports_dir).resolve()
|
||||
target = (exports_dir / job_id / filename).resolve()
|
||||
root_str = str(exports_dir).rstrip("\\/") + "\\"
|
||||
posix_root = str(exports_dir).rstrip("\\/") + "/"
|
||||
if not (str(target).startswith(root_str) or str(target).startswith(posix_root)):
|
||||
raise ExportPathError(f"导出路径越界:{target}")
|
||||
return target
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExportResult:
|
||||
path: Path
|
||||
rows: int
|
||||
bytes_written: int
|
||||
fmt: str
|
||||
|
||||
|
||||
def run_query_arrow(
|
||||
conn: duckdb.DuckDBPyConnection, sql: str, params: list | None = None
|
||||
) -> pa.Table:
|
||||
"""执行查询返回 Arrow 表(查询/导出共用;调用方负责 sqlgate 校验与线程池)。"""
|
||||
result = conn.execute(sql, params or [])
|
||||
return result.arrow().read_all()
|
||||
|
||||
|
||||
def write_export(table: pa.Table, fmt: str, target: Path) -> ExportResult:
|
||||
if fmt not in EXPORT_FORMATS:
|
||||
raise ExportFormatError(f"export 仅支持落盘格式 {sorted(EXPORT_FORMATS)},收到 {fmt!r}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if fmt == "csv":
|
||||
pacsv.write_csv(table, target)
|
||||
else:
|
||||
paparquet.write_table(table, target, compression="zstd")
|
||||
size = target.stat().st_size
|
||||
return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt=fmt)
|
||||
|
||||
|
||||
def record_export_job(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
job_id: str,
|
||||
*,
|
||||
fmt: str,
|
||||
path: str,
|
||||
rows: int,
|
||||
bytes_written: int,
|
||||
state: str = "done",
|
||||
spec_json: str = "{}",
|
||||
) -> None:
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO export_jobs "
|
||||
"(job_id, spec_json, fmt, path, rows, bytes, state, created_at, finished_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, now(), now())",
|
||||
[job_id, spec_json, fmt, path, rows, bytes_written, state],
|
||||
)
|
||||
|
||||
|
||||
def get_export_job(conn: duckdb.DuckDBPyConnection, job_id: str) -> dict | None:
|
||||
row = conn.execute(
|
||||
"SELECT job_id, spec_json, fmt, path, rows, bytes, state, "
|
||||
"created_at, finished_at FROM export_jobs WHERE job_id = ?",
|
||||
[job_id],
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"job_id": row[0],
|
||||
"spec_json": row[1],
|
||||
"fmt": row[2],
|
||||
"path": row[3],
|
||||
"rows": row[4],
|
||||
"bytes": row[5],
|
||||
"state": row[6],
|
||||
"created_at": None if row[7] is None else row[7].isoformat(),
|
||||
"finished_at": None if row[8] is None else row[8].isoformat(),
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
"""单写者锁 quantdata.lock(设计 §11)。
|
||||
|
||||
判活 = PID + 进程 start time + boot_id 三元一致(防 PID 复用误判)。
|
||||
发现陈旧锁:先备份 .lock.bak 再接管,并记日志。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import psutil
|
||||
|
||||
|
||||
class LockHeldError(RuntimeError):
|
||||
def __init__(self, info: "LockInfo") -> None:
|
||||
self.info = info
|
||||
super().__init__(
|
||||
f"已有 quantdata 实例持有锁:PID={info.pid}( acquired_at={info.acquired_at})。"
|
||||
f"如确认无实例在跑,可删除锁文件。"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LockInfo:
|
||||
pid: int
|
||||
start_time: float
|
||||
boot_id: float
|
||||
acquired_at: str
|
||||
cmd: str
|
||||
|
||||
def to_json(self) -> str:
|
||||
return json.dumps(self.__dict__, ensure_ascii=False, indent=2)
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, text: str) -> "LockInfo":
|
||||
d = json.loads(text)
|
||||
return cls(
|
||||
pid=int(d["pid"]),
|
||||
start_time=float(d["start_time"]),
|
||||
boot_id=float(d["boot_id"]),
|
||||
acquired_at=str(d.get("acquired_at", "")),
|
||||
cmd=str(d.get("cmd", "")),
|
||||
)
|
||||
|
||||
|
||||
def _boot_id() -> float:
|
||||
"""boot 标识:psutil.boot_time() 秒级时间戳,Windows/Linux 通用。"""
|
||||
return psutil.boot_time()
|
||||
|
||||
|
||||
def _process_start_time(pid: int) -> float | None:
|
||||
"""进程当前 start time;进程不存在返回 None。"""
|
||||
try:
|
||||
return psutil.Process(pid).create_time()
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
||||
return None
|
||||
|
||||
|
||||
def is_alive(info: LockInfo, *, now_boot_id: float | None = None) -> bool:
|
||||
"""三元判活:进程存在 + start_time 一致 + boot_id 一致。"""
|
||||
now_boot_id = _boot_id() if now_boot_id is None else now_boot_id
|
||||
if info.boot_id != now_boot_id:
|
||||
return False # 重启过系统 → 一律陈旧
|
||||
start = _process_start_time(info.pid)
|
||||
if start is None:
|
||||
return False
|
||||
# start_time 允许极小浮点误差
|
||||
return abs(start - info.start_time) < 0.01
|
||||
|
||||
|
||||
def read_lock(path: Path) -> LockInfo | None:
|
||||
try:
|
||||
return LockInfo.from_json(path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except (json.JSONDecodeError, KeyError, ValueError):
|
||||
# 损坏的锁文件视为陈旧(半写状态)
|
||||
return None
|
||||
|
||||
|
||||
def acquire(lock_path: Path, *, cmd: str = "", acquired_at: str | None = None) -> LockInfo:
|
||||
"""拿锁;活锁持有 → LockHeldError;陈旧/损坏锁 → 备份后接管。"""
|
||||
existing = read_lock(lock_path)
|
||||
if existing is not None and is_alive(existing):
|
||||
raise LockHeldError(existing)
|
||||
if existing is not None:
|
||||
# 陈旧锁:先备份再接管(§11)
|
||||
backup = lock_path.with_suffix(lock_path.suffix + ".bak")
|
||||
shutil.copy2(lock_path, backup)
|
||||
try:
|
||||
print(f"[lock] 发现陈旧锁(PID={existing.pid} 已不存在),备份到 {backup} 后接管")
|
||||
except OSError: # stdout 关闭等极端场景不阻塞拿锁
|
||||
pass
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
me = psutil.Process(os.getpid())
|
||||
info = LockInfo(
|
||||
pid=me.pid,
|
||||
start_time=me.create_time(),
|
||||
boot_id=_boot_id(),
|
||||
acquired_at=acquired_at or time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
cmd=cmd,
|
||||
)
|
||||
lock_path.write_text(info.to_json(), encoding="utf-8")
|
||||
return info
|
||||
|
||||
|
||||
def release(lock_path: Path, info: LockInfo) -> bool:
|
||||
"""释放锁:仅当锁内仍是自己(防止误删后来者的锁)才删除。"""
|
||||
current = read_lock(lock_path)
|
||||
if current is None:
|
||||
return False
|
||||
if current.pid == info.pid and abs(current.start_time - info.start_time) < 0.01:
|
||||
lock_path.unlink(missing_ok=True)
|
||||
return True
|
||||
return False
|
||||
@@ -0,0 +1,77 @@
|
||||
"""dbpath 目录规范(设计 §3.1)。一切状态收敛在 -d 指定的 dbpath。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class DbPathLayout:
|
||||
"""dbpath 内固定相对位置的访问器。不负责创建目录(init 时 ensure)。"""
|
||||
|
||||
def __init__(self, root: Path | str) -> None:
|
||||
self.root = Path(root).resolve()
|
||||
|
||||
@property
|
||||
def config_file(self) -> Path:
|
||||
return self.root / "config.json"
|
||||
|
||||
@property
|
||||
def catalog(self) -> Path:
|
||||
return self.root / "catalog.duckdb"
|
||||
|
||||
@property
|
||||
def data_dir(self) -> Path:
|
||||
return self.root / "data"
|
||||
|
||||
@property
|
||||
def scripts_dir(self) -> Path:
|
||||
return self.root / "scripts"
|
||||
|
||||
@property
|
||||
def exports_dir(self) -> Path:
|
||||
return self.root / "exports"
|
||||
|
||||
@property
|
||||
def logs_dir(self) -> Path:
|
||||
return self.root / "logs"
|
||||
|
||||
@property
|
||||
def tmp_dir(self) -> Path:
|
||||
return self.root / "tmp"
|
||||
|
||||
@property
|
||||
def lock_file(self) -> Path:
|
||||
return self.root / "quantdata.lock"
|
||||
|
||||
@property
|
||||
def trash_dir(self) -> Path:
|
||||
"""compaction 两阶段协议的延迟删除区(M2 使用,目录先规范)。"""
|
||||
return self.root / "data" / ".trash"
|
||||
|
||||
def ensure_dirs(self) -> list[Path]:
|
||||
"""init 用:建全部目录,返回已确保存在的目录列表。幂等。"""
|
||||
made: list[Path] = []
|
||||
for d in (
|
||||
self.root,
|
||||
self.data_dir,
|
||||
self.scripts_dir,
|
||||
self.exports_dir,
|
||||
self.logs_dir,
|
||||
self.tmp_dir,
|
||||
self.trash_dir,
|
||||
):
|
||||
if not d.exists():
|
||||
d.mkdir(parents=True)
|
||||
made.append(d)
|
||||
return made
|
||||
|
||||
|
||||
def resolve_dbpath(explicit: str | None) -> Path:
|
||||
"""-d 显式指定优先;否则环境变量 QUANTDATA_HOME 兜底;都没有则报错。"""
|
||||
if explicit:
|
||||
return Path(explicit).resolve()
|
||||
env = os.environ.get("QUANTDATA_HOME")
|
||||
if env:
|
||||
return Path(env).resolve()
|
||||
raise SystemExit("缺少 dbpath:请用 -d /xxx/dbpath 或设置环境变量 QUANTDATA_HOME")
|
||||
@@ -0,0 +1,29 @@
|
||||
"""serve 常驻:FastAPI 网关、市场状态、鉴权、运行时。"""
|
||||
|
||||
from .app import AppContext, QueryRequest, ExportRequest, create_app
|
||||
from .auth import (
|
||||
SecurityError,
|
||||
assert_security,
|
||||
is_loopback,
|
||||
mask_query_string,
|
||||
mask_token,
|
||||
token_ok,
|
||||
)
|
||||
from .state import MARKET_TZ, SESSIONS, MarketState, market_state
|
||||
|
||||
__all__ = [
|
||||
"AppContext",
|
||||
"ExportRequest",
|
||||
"MARKET_TZ",
|
||||
"MarketState",
|
||||
"QueryRequest",
|
||||
"SESSIONS",
|
||||
"SecurityError",
|
||||
"assert_security",
|
||||
"create_app",
|
||||
"is_loopback",
|
||||
"mask_query_string",
|
||||
"mask_token",
|
||||
"market_state",
|
||||
"token_ok",
|
||||
]
|
||||
@@ -0,0 +1,395 @@
|
||||
"""FastAPI 服务网关(设计 §8):REST /v1/query /v1/export /v1/meta /v1/stats /v1/state + 探针。
|
||||
|
||||
执行模型(§7.6):查询一律线程池执行(事件循环零阻塞);
|
||||
超时 = ``connection.interrupt()`` 真中断;并发信号量 + 等待超时 429(§8.3)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import duckdb
|
||||
import psutil
|
||||
import pyarrow as pa
|
||||
from fastapi import Depends, FastAPI, Header, HTTPException
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from .. import __version__
|
||||
from ..config import AppConfig
|
||||
from ..export.files import (
|
||||
EXPORT_FORMATS,
|
||||
ExportPathError,
|
||||
get_export_job,
|
||||
new_job_id,
|
||||
record_export_job,
|
||||
resolve_export_target,
|
||||
write_export,
|
||||
)
|
||||
from ..sqlgate import SqlRejected, validate
|
||||
from .auth import token_ok
|
||||
from .state import MARKET_TZ, market_state
|
||||
|
||||
|
||||
class ResultTooLargeError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class QueryTimeoutError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class AppContext:
|
||||
"""app 依赖注入载体。cursor 由调用方提供(serve 从写连接派生;测试注入)。"""
|
||||
|
||||
dbpath: Path
|
||||
config: AppConfig
|
||||
cursor: duckdb.DuckDBPyConnection
|
||||
schema_version: int = 0
|
||||
|
||||
|
||||
class QueryRequest(BaseModel):
|
||||
sql: str
|
||||
params: list[Any] = Field(default_factory=list)
|
||||
fmt: str = "json" # json | arrow
|
||||
max_rows: int | None = None
|
||||
timeout_ms: int | None = None
|
||||
|
||||
|
||||
class ExportRequest(BaseModel):
|
||||
fmt: str # csv | parquet
|
||||
sql: str
|
||||
params: list[Any] = Field(default_factory=list)
|
||||
filename: str | None = None
|
||||
sync: bool = True
|
||||
|
||||
|
||||
def execute_query_blocking(
|
||||
cursor: duckdb.DuckDBPyConnection,
|
||||
sql: str,
|
||||
params: list[Any],
|
||||
max_rows: int,
|
||||
max_arrow_bytes: int,
|
||||
) -> tuple[pa.Table, bool]:
|
||||
"""线程池内执行:fetch Arrow → 上限校验 → 截断。返回 (table, truncated)。"""
|
||||
result = cursor.execute(sql, params or [])
|
||||
tbl = result.arrow().read_all()
|
||||
if tbl.nbytes > max_arrow_bytes:
|
||||
raise ResultTooLargeError(
|
||||
f"结果 {tbl.nbytes} 字节超过上限 {max_arrow_bytes}:请改走 /v1/export(§8.3)"
|
||||
)
|
||||
truncated = tbl.num_rows > max_rows
|
||||
if truncated:
|
||||
tbl = tbl.slice(0, max_rows)
|
||||
return tbl, truncated
|
||||
|
||||
|
||||
def arrow_stream_b64(tbl: pa.Table) -> str:
|
||||
"""Arrow IPC **streaming format** → base64(附录 A 解码路径)。"""
|
||||
sink = pa.BufferOutputStream()
|
||||
with pa.ipc.new_stream(sink, tbl.schema) as writer:
|
||||
writer.write_table(tbl)
|
||||
return base64.b64encode(sink.getvalue().to_pybytes()).decode("ascii")
|
||||
|
||||
|
||||
def table_to_rows(tbl: pa.Table) -> list[list[Any]]:
|
||||
if tbl.num_columns == 0:
|
||||
return []
|
||||
cols = [c.to_pylist() for c in tbl.columns]
|
||||
return [list(row) for row in zip(*cols)]
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
"""ISO8601 UTC 带 Z(§8.2 全文统一)。"""
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def query_response(
|
||||
tbl: pa.Table, truncated: bool, elapsed_ms: int, fmt: str
|
||||
) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {
|
||||
"columns": tbl.schema.names,
|
||||
"dtypes": [str(f.type) for f in tbl.schema],
|
||||
"row_count": tbl.num_rows,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"truncated": truncated,
|
||||
}
|
||||
if fmt == "arrow":
|
||||
out["b64_arrow"] = arrow_stream_b64(tbl)
|
||||
else:
|
||||
out["rows"] = table_to_rows(tbl)
|
||||
return out
|
||||
|
||||
|
||||
def disk_usage_bytes(root: Path) -> int:
|
||||
return sum(p.stat().st_size for p in root.rglob("*") if p.is_file())
|
||||
|
||||
|
||||
async def _run_with_timeout(
|
||||
ctx: AppContext,
|
||||
sql: str,
|
||||
params: list[Any],
|
||||
max_rows: int,
|
||||
timeout_s: float,
|
||||
) -> tuple[pa.Table, bool]:
|
||||
"""线程池执行 + 超时真中断(§7.6)。query / export 共用同一执行语义。
|
||||
|
||||
不能用 ``asyncio.wait_for(to_thread(...))``:超时时 wait_for 会 cancel 那个
|
||||
已成 future,随后收割阶段抛的 CancelledError 会逃逸出调用方的 except 分支
|
||||
(Python 3.13 实测)。这里用 shield + 手动超时,超时后 interrupt 连接,
|
||||
再把线程异常全部吞掉,只向外报 504。
|
||||
"""
|
||||
task = asyncio.ensure_future(
|
||||
asyncio.to_thread(
|
||||
execute_query_blocking, ctx.cursor, sql, params, max_rows, ctx.config.sql.max_arrow_bytes
|
||||
)
|
||||
)
|
||||
done, _pending = await asyncio.wait({task}, timeout=timeout_s)
|
||||
if not done:
|
||||
ctx.cursor.interrupt() # 真中断:线程内查询被打断
|
||||
try:
|
||||
await asyncio.shield(task) # 收割线程异常,保证不悬空
|
||||
except BaseException: # noqa: BLE001 — 中断/取消异常统一吞掉,对外只报 504
|
||||
pass
|
||||
raise HTTPException(
|
||||
status_code=504, detail=f"查询超时(>{timeout_s:.1f}s),已 interrupt 中断"
|
||||
)
|
||||
return task.result()
|
||||
|
||||
|
||||
def create_app(ctx: AppContext) -> FastAPI:
|
||||
sql_cfg = ctx.config.sql
|
||||
query_sem = asyncio.Semaphore(max(1, sql_cfg.max_concurrent))
|
||||
|
||||
def _validate_sql(sql: str) -> None:
|
||||
try:
|
||||
validate(sql, sql_cfg.allow_functions)
|
||||
except SqlRejected as e:
|
||||
raise HTTPException(status_code=400, detail=e.reason) from e
|
||||
|
||||
async def _acquire_query_slot() -> None:
|
||||
try:
|
||||
await asyncio.wait_for(query_sem.acquire(), timeout=sql_cfg.acquire_timeout_s)
|
||||
except asyncio.TimeoutError as e:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"查询并发已达上限 {sql_cfg.max_concurrent},排队超时(§8.3)",
|
||||
) from e
|
||||
|
||||
async def require_token(
|
||||
authorization: str | None = Header(default=None),
|
||||
) -> None:
|
||||
provided = None
|
||||
if authorization and authorization.startswith("Bearer "):
|
||||
provided = authorization[len("Bearer ") :].strip()
|
||||
if not token_ok(provided, ctx.config.http.token):
|
||||
raise HTTPException(status_code=401, detail="invalid or missing token")
|
||||
|
||||
app = FastAPI(title="quantdata", version=__version__, docs_url="/docs")
|
||||
|
||||
# ---- 探针 ---------------------------------------------------------------
|
||||
@app.get("/healthz")
|
||||
def healthz() -> dict:
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/v1/ready")
|
||||
def ready() -> Any:
|
||||
checks: dict[str, Any] = {}
|
||||
try:
|
||||
ctx.cursor.execute("SELECT 1")
|
||||
checks["catalog"] = "ok"
|
||||
except duckdb.Error as e:
|
||||
checks["catalog"] = f"error: {e}"
|
||||
checks["schema_version"] = ctx.schema_version
|
||||
checks["schema"] = "ok" if ctx.schema_version >= 1 else "not_migrated"
|
||||
free_pct: float | None = None
|
||||
if ctx.dbpath.exists():
|
||||
usage = psutil.disk_usage(str(ctx.dbpath))
|
||||
free_pct = round(100.0 - usage.percent, 2)
|
||||
checks["disk_free_pct"] = free_pct
|
||||
disk_ok = free_pct is None or free_pct > 5.0 # <5% 硬阈值(§13)
|
||||
if checks["catalog"] == "ok" and checks["schema"] == "ok" and disk_ok:
|
||||
return {"ready": True, "checks": checks}
|
||||
return JSONResponse(status_code=503, content={"ready": False, "checks": checks})
|
||||
|
||||
# ---- query ---------------------------------------------------------------
|
||||
@app.post("/v1/query", dependencies=[Depends(require_token)])
|
||||
async def query(body: QueryRequest) -> dict[str, Any]:
|
||||
if body.fmt not in ("json", "arrow"):
|
||||
raise HTTPException(status_code=422, detail=f"fmt 仅支持 json|arrow,收到 {body.fmt!r}")
|
||||
_validate_sql(body.sql)
|
||||
max_rows = min(body.max_rows or sql_cfg.max_rows, sql_cfg.max_rows)
|
||||
timeout_s = (body.timeout_ms if body.timeout_ms is not None else sql_cfg.timeout_ms) / 1000
|
||||
started = time.monotonic()
|
||||
await _acquire_query_slot()
|
||||
try:
|
||||
tbl, truncated = await _run_with_timeout(
|
||||
ctx, body.sql, body.params, max_rows, timeout_s
|
||||
)
|
||||
except ResultTooLargeError as e:
|
||||
raise HTTPException(status_code=413, detail=str(e)) from e
|
||||
except duckdb.Error as e:
|
||||
if "interrupt" in str(e).lower():
|
||||
raise HTTPException(status_code=504, detail=f"查询被中断:{e}") from e
|
||||
raise HTTPException(status_code=400, detail=f"执行失败:{e}") from e
|
||||
elapsed = int((time.monotonic() - started) * 1000)
|
||||
return query_response(tbl, truncated, elapsed, body.fmt)
|
||||
|
||||
# ---- export(M1 同步路径:小导出直回文件流;job 化在 M2)--------------------
|
||||
@app.post("/v1/export", dependencies=[Depends(require_token)])
|
||||
async def export(body: ExportRequest) -> FileResponse:
|
||||
if body.fmt not in EXPORT_FORMATS:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=(
|
||||
f"export 仅支持落盘格式 {sorted(EXPORT_FORMATS)};"
|
||||
f"pandas/polars 走 /v1/query fmt=arrow(§9 双语义归位)"
|
||||
),
|
||||
)
|
||||
_validate_sql(body.sql)
|
||||
timeout_s = sql_cfg.timeout_ms / 1000
|
||||
await _acquire_query_slot()
|
||||
try:
|
||||
tbl, _truncated = await _run_with_timeout(
|
||||
ctx, body.sql, body.params, sql_cfg.max_rows, timeout_s
|
||||
)
|
||||
except duckdb.Error as e:
|
||||
raise HTTPException(status_code=400, detail=f"执行失败:{e}") from e
|
||||
job_id = new_job_id()
|
||||
filename = body.filename or f"export.{body.fmt}"
|
||||
try:
|
||||
target = resolve_export_target(
|
||||
ctx.dbpath / ctx.config.export.out_dir, job_id, filename
|
||||
)
|
||||
except ExportPathError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
result = await asyncio.to_thread(write_export, tbl, body.fmt, target)
|
||||
record_export_job(
|
||||
ctx.cursor,
|
||||
job_id,
|
||||
fmt=body.fmt,
|
||||
path=str(target.relative_to(ctx.dbpath)),
|
||||
rows=result.rows,
|
||||
bytes_written=result.bytes_written,
|
||||
)
|
||||
media = "text/csv" if body.fmt == "csv" else "application/octet-stream"
|
||||
return FileResponse(target, media_type=media, filename=filename)
|
||||
|
||||
@app.get("/v1/export/{job_id}", dependencies=[Depends(require_token)])
|
||||
def export_job(job_id: str) -> dict:
|
||||
job = get_export_job(ctx.cursor, job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail=f"导出任务不存在:{job_id}")
|
||||
return job
|
||||
|
||||
# ---- meta / stats / state -------------------------------------------------
|
||||
@app.get("/v1/meta", dependencies=[Depends(require_token)])
|
||||
def meta() -> dict:
|
||||
names = [
|
||||
r[0]
|
||||
for r in ctx.cursor.execute(
|
||||
"SELECT table_name FROM information_schema.tables "
|
||||
"WHERE table_schema='main' AND table_type='VIEW' ORDER BY table_name"
|
||||
).fetchall()
|
||||
]
|
||||
views = []
|
||||
for name in names:
|
||||
info: dict[str, Any] = {"name": name}
|
||||
info["columns"] = [
|
||||
r[0]
|
||||
for r in ctx.cursor.execute(
|
||||
"SELECT column_name FROM information_schema.columns "
|
||||
"WHERE table_schema='main' AND table_name=? ORDER BY ordinal_position",
|
||||
[name],
|
||||
).fetchall()
|
||||
]
|
||||
info["row_count"] = ctx.cursor.execute(f"SELECT count(*) FROM {name}").fetchone()[0]
|
||||
cols = set(info["columns"])
|
||||
if "date" in cols:
|
||||
row = ctx.cursor.execute(
|
||||
f"SELECT min(date), max(date) FROM {name}"
|
||||
).fetchone()
|
||||
info["date_min"] = None if row[0] is None else row[0].isoformat()
|
||||
info["date_max"] = None if row[1] is None else row[1].isoformat()
|
||||
if "symbol" in cols:
|
||||
info["sample_symbols"] = [
|
||||
r[0]
|
||||
for r in ctx.cursor.execute(
|
||||
f"SELECT DISTINCT symbol FROM {name} WHERE symbol IS NOT NULL LIMIT 3"
|
||||
).fetchall()
|
||||
]
|
||||
views.append(info)
|
||||
return {
|
||||
"version": __version__,
|
||||
"schema_version": ctx.schema_version,
|
||||
"views": views,
|
||||
}
|
||||
|
||||
@app.get("/v1/stats", dependencies=[Depends(require_token)])
|
||||
def stats() -> dict:
|
||||
cps = ctx.cursor.execute(
|
||||
"SELECT dataset, market, scope, watermark, rows, status, updated_at "
|
||||
"FROM etl_checkpoint ORDER BY dataset, market, scope"
|
||||
).fetchall()
|
||||
jobs = ctx.cursor.execute(
|
||||
"SELECT state, count(*) FROM export_jobs GROUP BY state"
|
||||
).fetchall()
|
||||
usage = None
|
||||
if ctx.dbpath.exists():
|
||||
usage = {"bytes": disk_usage_bytes(ctx.dbpath)}
|
||||
try:
|
||||
du = psutil.disk_usage(str(ctx.dbpath))
|
||||
usage["free_pct"] = round(100.0 - du.percent, 2)
|
||||
except OSError:
|
||||
usage["free_pct"] = None
|
||||
return {
|
||||
"checkpoints": [
|
||||
{
|
||||
"dataset": r[0], "market": r[1], "scope": r[2], "watermark": r[3],
|
||||
"rows": r[4], "status": r[5],
|
||||
"updated_at": None if r[6] is None else r[6].isoformat(),
|
||||
}
|
||||
for r in cps
|
||||
],
|
||||
"export_jobs": {state: n for state, n in jobs},
|
||||
"disk": usage,
|
||||
"http": {"ws_conns": 0}, # WS 在 M3;指标占位(§13)
|
||||
}
|
||||
|
||||
@app.get("/v1/state/{market}", dependencies=[Depends(require_token)])
|
||||
def market_state_endpoint(market: str) -> dict:
|
||||
tz = MARKET_TZ.get(market)
|
||||
if tz is None:
|
||||
raise HTTPException(status_code=404, detail=f"未知市场:{market}")
|
||||
now_local = datetime.now(ZoneInfo(tz))
|
||||
row = ctx.cursor.execute(
|
||||
"SELECT is_open FROM calendar WHERE market=? AND trade_date=?",
|
||||
[market, now_local.date()],
|
||||
).fetchone()
|
||||
is_trading_day = bool(row[0]) if row else False
|
||||
if row is None:
|
||||
reason = "no_calendar_data"
|
||||
else:
|
||||
reason = None
|
||||
hhmm = now_local.hour * 100 + now_local.minute
|
||||
ms = market_state(market, hhmm, is_trading_day=is_trading_day)
|
||||
if reason:
|
||||
ms.reason = reason
|
||||
return {
|
||||
"market": market,
|
||||
"state": ms.state,
|
||||
"ts": utc_now_iso(),
|
||||
"next_at": ms.next_at,
|
||||
"session_tz": ms.session_tz,
|
||||
"reason": ms.reason,
|
||||
}
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,51 @@
|
||||
"""鉴权与日志脱敏(设计 §8.4)。
|
||||
|
||||
- token 非空时:REST ``Authorization: Bearer <token>``;WS(M3)优先握手 Header;
|
||||
- 非本机且 token 为空 → 拒绝启动(fail-fast,--insecure 显式放行);
|
||||
- 日志对 token 统一脱敏(只留前 4 位)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"})
|
||||
|
||||
_TOKEN_IN_URL = re.compile(r"(token=)([^&\s]{4})[^&\s]*")
|
||||
|
||||
|
||||
def mask_token(token: str) -> str:
|
||||
"""token 脱敏:留前 4 位 + ***;空串原样。"""
|
||||
if not token:
|
||||
return ""
|
||||
return token[:4] + "***"
|
||||
|
||||
|
||||
def mask_query_string(qs: str) -> str:
|
||||
"""日志里的 ``?token=...`` / ``&token=...`` 统一脱敏(含 uvicorn access log 场景)。"""
|
||||
return _TOKEN_IN_URL.sub(r"\1\2***", qs)
|
||||
|
||||
|
||||
def is_loopback(host: str) -> bool:
|
||||
return host in _LOOPBACK_HOSTS
|
||||
|
||||
|
||||
def assert_security(host: str, token: str, *, insecure: bool) -> None:
|
||||
"""启动强制校验(§8.4):非本机 + 空 token + 未显式 --insecure → 拒绝启动。"""
|
||||
if is_loopback(host) or token or insecure:
|
||||
return
|
||||
raise SecurityError(
|
||||
f"拒绝启动:监听地址 {host} 非 loopback 且 http.token 为空;"
|
||||
f"请设置 token 或显式使用 --insecure(§8.4 fail-fast)"
|
||||
)
|
||||
|
||||
|
||||
class SecurityError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def token_ok(provided: str | None, expected: str) -> bool:
|
||||
"""expected 为空 = 不校验;否则必须精确匹配。"""
|
||||
if not expected:
|
||||
return True
|
||||
return bool(provided) and provided == expected
|
||||
@@ -0,0 +1,124 @@
|
||||
"""serve 常驻主循环(设计 §7.1 / §7.6)。
|
||||
|
||||
启动序:拿锁 → 开库 + migrate → 安全校验 → 起 uvicorn(Server.serve() 挂同一
|
||||
事件循环,不用 uvicorn.run() 自建循环)→ 前台常驻;SIGTERM/SIGINT 优雅退出。
|
||||
bind 失败(端口被占)→ 打印占用者 PID 后非零退出(交 supervisord 退避重试)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
import uvicorn
|
||||
|
||||
from ..config import AppConfig, load_config
|
||||
from ..lockfile import LockHeldError, acquire, release
|
||||
from ..paths import DbPathLayout
|
||||
from ..store import SchemaTooNewError, migrate, open_catalog
|
||||
from .app import AppContext, create_app
|
||||
from .auth import assert_security
|
||||
|
||||
# 退出码:0 正常 / 2 锁被占 / 3 schema 版本不兼容 / 4 安全校验拒绝 / 5 端口被占
|
||||
EXIT_OK = 0
|
||||
EXIT_LOCK_HELD = 2
|
||||
EXIT_SCHEMA_TOO_NEW = 3
|
||||
EXIT_SECURITY = 4
|
||||
EXIT_PORT_BUSY = 5
|
||||
|
||||
|
||||
def find_listener_pid(port: int) -> int | None:
|
||||
"""找占用端口的进程 PID(尽力而为;psutil 未装/权限不足返回 None)。"""
|
||||
try:
|
||||
import psutil
|
||||
|
||||
for conn in psutil.net_connections(kind="inet"):
|
||||
if conn.status == psutil.CONN_LISTEN and conn.laddr and conn.laddr.port == port:
|
||||
return conn.pid
|
||||
except Exception: # noqa: BLE001 — psutil 缺失/平台权限差异都按“找不到”处理
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
async def _serve_no_http(stop_event: asyncio.Event) -> None:
|
||||
"""--no-http:纯 CLI 批处理常驻(M1 无调度器,等信号)。"""
|
||||
await stop_event.wait()
|
||||
|
||||
|
||||
async def run_serve(
|
||||
dbpath: Path,
|
||||
*,
|
||||
host: str | None = None,
|
||||
port: int | None = None,
|
||||
no_http: bool = False,
|
||||
insecure: bool = False,
|
||||
config: AppConfig | None = None,
|
||||
stop_event: asyncio.Event | None = None,
|
||||
check_port: bool = True,
|
||||
) -> int:
|
||||
"""serve 主流程,返回进程退出码。真网 uvicorn 启停段按白名单豁免覆盖率。"""
|
||||
layout = DbPathLayout(dbpath)
|
||||
if not layout.root.exists():
|
||||
print(f"[serve] dbpath 不存在:{layout.root}(先 quantdata init)")
|
||||
return 1
|
||||
cfg = config or load_config(layout.config_file)
|
||||
host = host or cfg.http.host
|
||||
port = port or cfg.http.port
|
||||
|
||||
# 1. 单写者锁(§11)
|
||||
try:
|
||||
lock = acquire(layout.lock_file, cmd="quantdata serve")
|
||||
except LockHeldError as e:
|
||||
print(f"[serve] 拒绝启动:{e}")
|
||||
return EXIT_LOCK_HELD
|
||||
try:
|
||||
# 2. 开库 + migrate(版本 > 代码已知最高版 → 拒启)
|
||||
conn = open_catalog(layout.root)
|
||||
try:
|
||||
schema_version = migrate(conn)
|
||||
except SchemaTooNewError as e:
|
||||
print(f"[serve] {e}")
|
||||
return EXIT_SCHEMA_TOO_NEW
|
||||
|
||||
# 4. 安全校验(§8.4 fail-fast)
|
||||
try:
|
||||
assert_security(host, cfg.http.token, insecure=insecure)
|
||||
except Exception as e:
|
||||
print(f"[serve] {e}")
|
||||
return EXIT_SECURITY
|
||||
|
||||
# 3/4. 起网关(--no-http 跳过)
|
||||
if no_http:
|
||||
await _serve_no_http(stop_event or asyncio.Event())
|
||||
return EXIT_OK
|
||||
|
||||
if check_port:
|
||||
listener = find_listener_pid(port)
|
||||
if listener is not None:
|
||||
print(f"[serve] 端口 {port} 已被占用(PID={listener}),退出非零交 supervisord 重试")
|
||||
return EXIT_PORT_BUSY
|
||||
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SET autoinstall_known_extensions=false")
|
||||
cursor.execute("SET autoload_known_extensions=false")
|
||||
ctx = AppContext(
|
||||
dbpath=layout.root, config=cfg, cursor=cursor, schema_version=schema_version
|
||||
)
|
||||
app = create_app(ctx)
|
||||
|
||||
uv_config = uvicorn.Config(
|
||||
app, host=host, port=port, log_level=cfg.log.level.lower(),
|
||||
timeout_graceful_shutdown=30, # 优雅退出预算 30s(§7.1)
|
||||
)
|
||||
server = uvicorn.Server(uv_config)
|
||||
# 真 uvicorn 起服 / socket 层:§14 白名单豁免(平台相关,本机踩过 socketpair
|
||||
# 死锁)。替代验证 = [CI] 验收链用 CLI 起真 serve + 真端口 curl(覆盖本函数
|
||||
# 的编排层,只是不把 socket 段记入行覆盖)+ 每周 smoke workflow(DP-2 v0.7)。
|
||||
await server.serve() # pragma: no cover # 挂同一事件循环(§7.6 第 3 条)
|
||||
return EXIT_OK
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
release(layout.lock_file, lock)
|
||||
@@ -0,0 +1,64 @@
|
||||
"""市场状态机纯函数(设计 §10.2 的 M1 简版:日历 + 时段推导,无网络依赖)。
|
||||
|
||||
纯本地确定性推导:is_trading_day 由 calendar 表提供;时段规则表内置。
|
||||
StateEngine 完整事件机(迁移推流)在 M2 接入 serve 调度器。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
MARKET_TZ: dict[str, str] = {
|
||||
"cn": "Asia/Shanghai",
|
||||
"us": "America/New_York",
|
||||
"hk": "Asia/Hong_Kong",
|
||||
}
|
||||
|
||||
# (开, 收) 本地时间,24h 制;M1 不含美股盘前盘后(可扩展)
|
||||
SESSIONS: dict[str, tuple[tuple[str, str], ...]] = {
|
||||
"cn": (("09:30", "11:30"), ("13:00", "15:00")),
|
||||
"hk": (("09:30", "12:00"), ("13:00", "16:00")),
|
||||
"us": (("09:30", "16:00"),),
|
||||
}
|
||||
|
||||
_AUCTION_START = "09:15" # A股集合竞价
|
||||
_MORNING_OPEN = "09:30"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class MarketState:
|
||||
market: str
|
||||
state: str # CLOSED / PRE_OPEN_AUCTION / OPEN / LUNCH_BREAK
|
||||
session_tz: str
|
||||
reason: str
|
||||
next_at: str | None = None # 本地时间 "HH:MM"
|
||||
|
||||
|
||||
def _hhmm(s: str) -> int:
|
||||
"""'HH:MM' → HHMM 整数(与 market_state 入参口径一致,如 '09:30' → 930)。"""
|
||||
h, m = s.split(":")
|
||||
return int(h) * 100 + int(m)
|
||||
|
||||
|
||||
def market_state(market: str, now_local_hhmm: int, *, is_trading_day: bool) -> MarketState:
|
||||
"""给定交易所本地 ``HHMM`` 整数(如 930、1301)与是否交易日,确定性推导状态。
|
||||
|
||||
参数化时间点便于测试全部边界(开盘/午休/收盘/竞价/节假日)。
|
||||
"""
|
||||
tz = MARKET_TZ.get(market, "Asia/Shanghai")
|
||||
if market not in SESSIONS:
|
||||
raise KeyError(f"未知市场:{market}")
|
||||
if not is_trading_day:
|
||||
return MarketState(market, "CLOSED", tz, "non_trading_day")
|
||||
sessions = SESSIONS[market]
|
||||
if market == "cn" and _hhmm(_AUCTION_START) <= now_local_hhmm < _hhmm(_MORNING_OPEN):
|
||||
return MarketState(market, "PRE_OPEN_AUCTION", tz, "session_rule", _MORNING_OPEN)
|
||||
for open_s, close_s in sessions:
|
||||
if _hhmm(open_s) <= now_local_hhmm < _hhmm(close_s):
|
||||
return MarketState(market, "OPEN", tz, "session_rule", close_s)
|
||||
# 午休:第一段收后到第二段开前(单段市场无午休)
|
||||
first_close = _hhmm(sessions[0][1])
|
||||
second_open = _hhmm(sessions[1][0]) if len(sessions) > 1 else None
|
||||
if second_open is not None and first_close <= now_local_hhmm < second_open:
|
||||
return MarketState(market, "LUNCH_BREAK", tz, "session_rule", sessions[1][0])
|
||||
return MarketState(market, "CLOSED", tz, "session_rule", sessions[0][0])
|
||||
@@ -0,0 +1,24 @@
|
||||
"""数据源适配层:SourceAdapter 协议、A股分类、pytdxdata 适配器。"""
|
||||
|
||||
from .base import (
|
||||
Bar1dRow,
|
||||
CalendarRow,
|
||||
FakeAdapter,
|
||||
SecurityRow,
|
||||
SourceAdapter,
|
||||
UpdateReport,
|
||||
)
|
||||
from .classify import classify_cn, exchange_of
|
||||
from .tdx import TdxAdapter
|
||||
|
||||
__all__ = [
|
||||
"Bar1dRow",
|
||||
"CalendarRow",
|
||||
"FakeAdapter",
|
||||
"SecurityRow",
|
||||
"SourceAdapter",
|
||||
"TdxAdapter",
|
||||
"UpdateReport",
|
||||
"classify_cn",
|
||||
"exchange_of",
|
||||
]
|
||||
@@ -0,0 +1,124 @@
|
||||
"""SourceAdapter 协议与行模型(设计 §5)。
|
||||
|
||||
适配层产出统一行模型,ETL 只认行模型;真实源(tdx)与测试源(fake)同构。
|
||||
fixtures 录制回放 + 故障注入的宿主也在这一层(M2 完善限速/退避)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar as _calendar
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SecurityRow:
|
||||
symbol: str
|
||||
name: str
|
||||
category: str
|
||||
exchange: str
|
||||
currency: str = "CNY"
|
||||
lot_size: int | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CalendarRow:
|
||||
trade_date: date
|
||||
is_open: bool
|
||||
note: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Bar1dRow:
|
||||
symbol: str
|
||||
date: date
|
||||
open: float
|
||||
high: float
|
||||
low: float
|
||||
close: float
|
||||
volume: float = 0.0
|
||||
amount: float = 0.0
|
||||
turnover: float | None = None
|
||||
adj_factor: float | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class UpdateReport:
|
||||
dataset: str
|
||||
market: str
|
||||
symbols: list[str] = field(default_factory=list)
|
||||
rows: int = 0
|
||||
date_min: date | None = None
|
||||
date_max: date | None = None
|
||||
|
||||
|
||||
def full_year_calendar(year: int, trade_dates: set[date]) -> list[CalendarRow]:
|
||||
"""把「交易日集合」补全为「整年逐日」日历(§5.2)。
|
||||
|
||||
源只负责说清哪些天开市;补全全年是 ETL 的职责,保证 calendar 表语义唯一:
|
||||
任意一天都查得到,is_open 可信。
|
||||
"""
|
||||
rows: list[CalendarRow] = []
|
||||
for month in range(1, 13):
|
||||
for day in range(1, _calendar.monthrange(year, month)[1] + 1):
|
||||
d = date(year, month, day)
|
||||
rows.append(CalendarRow(trade_date=d, is_open=d in trade_dates))
|
||||
return rows
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SourceAdapter(Protocol):
|
||||
name: str
|
||||
capabilities: frozenset[str]
|
||||
|
||||
async def fetch_securities(self, market: str) -> list[SecurityRow]: ...
|
||||
|
||||
async def fetch_bars_1d(
|
||||
self, market: str, symbols: list[str], start: date | None, end: date | None
|
||||
) -> list[Bar1dRow]: ...
|
||||
|
||||
async def fetch_calendar(self, market: str, year: int) -> list[CalendarRow]: ...
|
||||
|
||||
|
||||
class FakeAdapter:
|
||||
"""确定性测试源:可注入任意行;不打真网([CI] 层数据走 seed,本源供单测与 CLI 替身)。"""
|
||||
|
||||
name = "fake"
|
||||
capabilities = frozenset({"securities", "bars_1d", "calendar"})
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
securities: list[SecurityRow] | None = None,
|
||||
bars: list[Bar1dRow] | None = None,
|
||||
calendar: dict[int, list[CalendarRow]] | None = None,
|
||||
) -> None:
|
||||
self.securities = securities or []
|
||||
self.bars = bars or []
|
||||
self.calendar = calendar or {}
|
||||
self.calls: list[tuple] = []
|
||||
|
||||
async def open(self) -> None:
|
||||
self.calls.append(("open",))
|
||||
|
||||
async def close(self) -> None:
|
||||
self.calls.append(("close",))
|
||||
|
||||
async def fetch_securities(self, market: str) -> list[SecurityRow]:
|
||||
self.calls.append(("securities", market))
|
||||
return list(self.securities)
|
||||
|
||||
async def fetch_bars_1d(
|
||||
self, market: str, symbols: list[str], start: date | None, end: date | None
|
||||
) -> list[Bar1dRow]:
|
||||
self.calls.append(("bars_1d", market, tuple(symbols), start, end))
|
||||
sym_set = set(symbols)
|
||||
return [
|
||||
b
|
||||
for b in self.bars
|
||||
if b.symbol in sym_set and (start is None or b.date >= start) and (end is None or b.date <= end)
|
||||
]
|
||||
|
||||
async def fetch_calendar(self, market: str, year: int) -> list[CalendarRow]:
|
||||
self.calls.append(("calendar", market, year))
|
||||
return list(self.calendar.get(year, []))
|
||||
@@ -0,0 +1,31 @@
|
||||
"""A股证券分类(纯函数,表驱动)。symbol 规范 = pytdxdata 形态:sh600000 / sz000001 / bj430047。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# (category, 前缀元组) —— 顺序敏感:先长/特例后短/泛化
|
||||
_CLASSIFY_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
("index", ("sh000", "sh899", "sz399", "sz395")),
|
||||
("reits", ("sh508",)),
|
||||
("etf", ("sh50", "sh51", "sh56", "sh58", "sz15", "sz16", "sz18")),
|
||||
("cbond", ("sh10", "sh11", "sh12", "sz12")),
|
||||
("stock", ("sh60", "sh68", "sz00", "sz30", "sz20")),
|
||||
("stock", ("bj",)),
|
||||
)
|
||||
|
||||
|
||||
def classify_cn(symbol: str) -> str:
|
||||
"""按 symbol 前缀归类 A股品类;未匹配归 other(绝不猜)。"""
|
||||
s = symbol.strip().lower()
|
||||
for category, prefixes in _CLASSIFY_RULES:
|
||||
if any(s.startswith(p) for p in prefixes):
|
||||
return category
|
||||
return "other"
|
||||
|
||||
|
||||
def exchange_of(symbol: str) -> str:
|
||||
"""sh/sz/bj 前缀即交易所;无法识别返回空串。"""
|
||||
s = symbol.strip().lower()
|
||||
for ex in ("sh", "sz", "bj"):
|
||||
if s.startswith(ex):
|
||||
return ex
|
||||
return ""
|
||||
@@ -0,0 +1,116 @@
|
||||
"""pytdxdata 适配器(主源,设计 §5.2)。
|
||||
|
||||
pytdxdata 是纯 asyncio 客户端(动态连接池),本适配器只做模型映射与调度编排;
|
||||
TDX 断连/坏节点重连由 pytdxdata 内建(评审 P2),这里不重复造。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
|
||||
from pytdxdata import Adjust, KlinePeriod, TdxData
|
||||
from pytdxdata.models import SecurityBar, SecurityInfo
|
||||
|
||||
from .base import Bar1dRow, CalendarRow, SecurityRow, full_year_calendar
|
||||
from .classify import classify_cn, exchange_of
|
||||
|
||||
_MARKET_PREFIXES = ("sh", "sz", "bj")
|
||||
|
||||
|
||||
def info_to_row(info: SecurityInfo) -> SecurityRow:
|
||||
"""SecurityInfo → 统一行(category 用规则分类,kind 仅作参考不直接入库)。"""
|
||||
symbol = info.symbol or ""
|
||||
return SecurityRow(
|
||||
symbol=symbol,
|
||||
name=info.name,
|
||||
category=classify_cn(symbol),
|
||||
exchange=exchange_of(symbol),
|
||||
currency="CNY",
|
||||
lot_size=int(info.volunit) if info.volunit else None,
|
||||
)
|
||||
|
||||
|
||||
def bar_to_row(bar: SecurityBar) -> Bar1dRow:
|
||||
return Bar1dRow(
|
||||
symbol=bar.symbol,
|
||||
date=date(bar.year, bar.month, bar.day),
|
||||
open=bar.open,
|
||||
high=bar.high,
|
||||
low=bar.low,
|
||||
close=bar.close,
|
||||
volume=bar.vol,
|
||||
amount=bar.amount,
|
||||
)
|
||||
|
||||
|
||||
def year_calendar_rows(year: int, trade_dates: set[date]) -> list[CalendarRow]:
|
||||
"""从交易日集合合成全年日历(复用 base.full_year_calendar,§5.2)。"""
|
||||
return full_year_calendar(year, trade_dates)
|
||||
|
||||
|
||||
class TdxAdapter:
|
||||
name = "tdx"
|
||||
capabilities = frozenset({"securities", "bars_1d", "calendar"})
|
||||
|
||||
def __init__(self, client: TdxData | None = None) -> None:
|
||||
self._client = client
|
||||
self._owns_client = client is None
|
||||
|
||||
async def open(self) -> None:
|
||||
if self._client is None:
|
||||
self._client = TdxData()
|
||||
if self._owns_client:
|
||||
await self._client.start()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._client is not None and self._owns_client:
|
||||
await self._client.close()
|
||||
self._owns_client = False
|
||||
|
||||
def _require(self) -> TdxData:
|
||||
if self._client is None:
|
||||
raise RuntimeError("TdxAdapter 未打开:先 await adapter.open()(或注入 client)")
|
||||
return self._client
|
||||
|
||||
async def fetch_securities(self, market: str) -> list[SecurityRow]:
|
||||
client = self._require()
|
||||
if market != "cn":
|
||||
return []
|
||||
rows: list[SecurityRow] = []
|
||||
seen: set[str] = set()
|
||||
for prefix in _MARKET_PREFIXES:
|
||||
for info in await client.get_universe(prefix):
|
||||
if info.symbol and info.symbol not in seen:
|
||||
seen.add(info.symbol)
|
||||
rows.append(info_to_row(info))
|
||||
return rows
|
||||
|
||||
async def fetch_bars_1d(
|
||||
self, market: str, symbols: list[str], start: date | None, end: date | None
|
||||
) -> list[Bar1dRow]:
|
||||
client = self._require()
|
||||
if market != "cn" or not symbols:
|
||||
return []
|
||||
bars = await client.get_bars(
|
||||
symbols,
|
||||
KlinePeriod.DAY,
|
||||
adjust=Adjust.NONE, # 存不复权原始价(§4.3)
|
||||
start_date=datetime(start.year, start.month, start.day) if start else None,
|
||||
end_date=datetime(end.year, end.month, end.day) if end else None,
|
||||
)
|
||||
return [bar_to_row(b) for b in bars if b.symbol]
|
||||
|
||||
async def fetch_calendar(self, market: str, year: int) -> list[CalendarRow]:
|
||||
"""TDX 交易日回调校验:用上证指数日K日期序列当交易日全集(§5.2)。"""
|
||||
client = self._require()
|
||||
if market != "cn":
|
||||
return []
|
||||
bars = await client.get_bars(
|
||||
"sh000001",
|
||||
KlinePeriod.DAY,
|
||||
adjust=Adjust.NONE,
|
||||
start_date=datetime(year, 1, 1),
|
||||
end_date=datetime(year, 12, 31),
|
||||
)
|
||||
trade_dates = {date(b.year, b.month, b.day) for b in bars}
|
||||
return year_calendar_rows(year, trade_dates)
|
||||
@@ -0,0 +1,5 @@
|
||||
"""SQL 闸:query_sql 安全约束(§8.3)。"""
|
||||
|
||||
from .gate import FORBIDDEN_FUNCTIONS, SqlRejected, validate
|
||||
|
||||
__all__ = ["FORBIDDEN_FUNCTIONS", "SqlRejected", "validate"]
|
||||
@@ -0,0 +1,119 @@
|
||||
"""query_sql 安全闸(设计 §8.3,评审 P0/P1 多条)。
|
||||
|
||||
六条闸:
|
||||
1. 只读连接 + 扩展自动加载关闭(store/db.configure_query_conn);
|
||||
2. 语句白名单:sqlglot fail-closed(解析失败一律拒)、dialect=duckdb、
|
||||
AST 根节点判定(非字符串前缀)、多语句拒绝(parse_all 长度必须 =1);
|
||||
3. 函数/表函数默认拒绝:显式拒 read_csv / read_parquet / *_scan / glob /
|
||||
getenv / current_setting 等文件·网络·环境函数;FROM/JOIN 出现表函数直接拒;
|
||||
``allow_functions`` 语义 = 额外放行清单(不是清空即全放行的总开关);
|
||||
4. 结果上限(执行层 max_rows / arrow bytes);
|
||||
5. 超时与并发闸(执行层线程池 + interrupt + 信号量);
|
||||
6. 防御参数(SET max_expression_depth)。
|
||||
|
||||
sqlglot 进 lock pin 版本(pyproject >=,< 由 lock 冻结),升级须过本模块回归测试。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlglot
|
||||
from sqlglot import exp
|
||||
|
||||
_DIALECT = "duckdb"
|
||||
|
||||
# 显式拒绝的函数(文件/网络/环境注入面),小写;`*_scan` 走后缀规则
|
||||
FORBIDDEN_FUNCTIONS: frozenset[str] = frozenset(
|
||||
{
|
||||
"read_csv",
|
||||
"read_csv_auto",
|
||||
"read_parquet",
|
||||
"read_json",
|
||||
"read_json_auto",
|
||||
"read_ndjson",
|
||||
"read_text",
|
||||
"glob",
|
||||
"getenv",
|
||||
"current_setting",
|
||||
"iceberg_scan",
|
||||
"delta_scan",
|
||||
}
|
||||
)
|
||||
|
||||
_SCAN_SUFFIX = "_scan"
|
||||
|
||||
# 允许的 AST 根节点:纯读取查询(CTE 附着在 Select 上)
|
||||
_ALLOWED_ROOTS: tuple[type[exp.Expression], ...] = (
|
||||
exp.Select,
|
||||
exp.Union,
|
||||
exp.Intersect,
|
||||
exp.Except,
|
||||
exp.Subquery,
|
||||
)
|
||||
|
||||
|
||||
class SqlRejected(ValueError):
|
||||
"""SQL 被安全闸拒绝。reason 面向调用方,可直接进 HTTP 400 detail。"""
|
||||
|
||||
def __init__(self, reason: str) -> None:
|
||||
self.reason = reason
|
||||
super().__init__(reason)
|
||||
|
||||
|
||||
def _func_name(node: exp.Expression) -> str | None:
|
||||
if isinstance(node, exp.Anonymous):
|
||||
return str(node.name).lower()
|
||||
if isinstance(node, exp.Func):
|
||||
return node.sql_name().lower()
|
||||
return None
|
||||
|
||||
|
||||
def _check_root(stmt: exp.Expression) -> None:
|
||||
if not isinstance(stmt, _ALLOWED_ROOTS):
|
||||
raise SqlRejected(f"仅允许只读查询(SELECT/UNION),收到:{type(stmt).__name__}")
|
||||
|
||||
|
||||
def _check_from_tables(stmt: exp.Expression) -> None:
|
||||
"""FROM/JOIN 出现函数(表函数)一律拒绝:库内视图都是普通表引用。"""
|
||||
for table in stmt.find_all(exp.Table):
|
||||
if not isinstance(table.this, exp.Identifier):
|
||||
raise SqlRejected(
|
||||
f"FROM/JOIN 仅允许库内普通表或视图,发现表函数:{table.sql(dialect=_DIALECT)}"
|
||||
)
|
||||
|
||||
|
||||
def _check_functions(stmt: exp.Expression, allow_functions: list[str]) -> None:
|
||||
allowed = {a.lower().strip() for a in allow_functions if a.strip()}
|
||||
for node in stmt.walk():
|
||||
name = _func_name(node)
|
||||
if name is None:
|
||||
continue
|
||||
if name in FORBIDDEN_FUNCTIONS and name not in allowed:
|
||||
raise SqlRejected(
|
||||
f"函数默认拒绝(文件/网络/环境注入面):{name};"
|
||||
f"确需放行请在 config sql.allow_functions 显式加入(§8.3)"
|
||||
)
|
||||
if name.endswith(_SCAN_SUFFIX) and name not in allowed:
|
||||
raise SqlRejected(f"函数默认拒绝(*_scan 表函数):{name}")
|
||||
|
||||
|
||||
def parse_one(sql: str) -> exp.Expression:
|
||||
"""fail-closed 解析:任何解析失败/空语句/多语句 → SqlRejected。"""
|
||||
if not sql or not sql.strip():
|
||||
raise SqlRejected("空 SQL")
|
||||
try:
|
||||
statements = sqlglot.parse(sql, read=_DIALECT)
|
||||
except Exception as e: # sqlglot.errors.ParseError 及一切解析期异常
|
||||
raise SqlRejected(f"SQL 解析失败(fail-closed):{e}") from e
|
||||
if len(statements) != 1 or statements[0] is None:
|
||||
raise SqlRejected(f"必须且只能包含一条语句,收到 {len(statements)} 条(多语句拒绝)")
|
||||
return statements[0]
|
||||
|
||||
|
||||
def validate(sql: str, allow_functions: list[str] | None = None) -> exp.Expression:
|
||||
"""通过全部闸则返回解析后的 AST;否则抛 SqlRejected。"""
|
||||
allow_functions = allow_functions or []
|
||||
stmt = parse_one(sql)
|
||||
_check_root(stmt)
|
||||
_check_from_tables(stmt)
|
||||
_check_functions(stmt, allow_functions)
|
||||
return stmt
|
||||
@@ -0,0 +1,23 @@
|
||||
"""存储层:DuckDB 连接、migrations、联邦视图。"""
|
||||
|
||||
from .db import CatalogNotInitialized, configure_query_conn, open_catalog
|
||||
from .migrations import (
|
||||
KNOWN_MAX_VERSION,
|
||||
MIGRATIONS,
|
||||
SchemaTooNewError,
|
||||
applied_versions,
|
||||
current_version,
|
||||
migrate,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CatalogNotInitialized",
|
||||
"KNOWN_MAX_VERSION",
|
||||
"MIGRATIONS",
|
||||
"SchemaTooNewError",
|
||||
"applied_versions",
|
||||
"configure_query_conn",
|
||||
"current_version",
|
||||
"migrate",
|
||||
"open_catalog",
|
||||
]
|
||||
@@ -0,0 +1,31 @@
|
||||
"""DuckDB 连接管理:写连接(init/ETL/serve)与只读连接(查询/导出)分离(§8.3)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
|
||||
|
||||
class CatalogNotInitialized(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def open_catalog(dbpath: Path, *, read_only: bool = False) -> duckdb.DuckDBPyConnection:
|
||||
"""打开 catalog.duckdb。只读模式要求库已存在(未初始化给清晰报错)。"""
|
||||
catalog = Path(dbpath) / "catalog.duckdb"
|
||||
if read_only and not catalog.exists():
|
||||
raise CatalogNotInitialized(
|
||||
f"{catalog} 不存在:库未初始化,请先执行 quantdata init -d {dbpath}"
|
||||
)
|
||||
return duckdb.connect(str(catalog), read_only=read_only)
|
||||
|
||||
|
||||
def configure_query_conn(
|
||||
conn: duckdb.DuckDBPyConnection, *, memory_limit: str = "2GB"
|
||||
) -> None:
|
||||
"""查询连接安全预设(§8.3):断扩展自动加载(堵 httpfs 出站)+ 内存上限。"""
|
||||
conn.execute("SET autoinstall_known_extensions=false")
|
||||
conn.execute("SET autoload_known_extensions=false")
|
||||
conn.execute(f"SET memory_limit='{memory_limit}'")
|
||||
conn.execute("SET max_expression_depth=1000")
|
||||
@@ -0,0 +1,259 @@
|
||||
"""schema_migrations 版本管理(设计 §11 / §3.3)。
|
||||
|
||||
- 每个迁移在事务内执行(失败回滚);
|
||||
- 库版本 > 代码已知最高版 → 报错退出(不带病跑);
|
||||
- 视图 DDL 纳入迁移(评审 P1)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import duckdb
|
||||
|
||||
from .views import all_day_views
|
||||
|
||||
MIGRATION_0001_STATEMENTS: tuple[str, ...] = (
|
||||
# ---- 参考数据 -------------------------------------------------------
|
||||
"""
|
||||
CREATE TABLE exchanges (
|
||||
exchange_code VARCHAR PRIMARY KEY,
|
||||
name VARCHAR NOT NULL,
|
||||
market VARCHAR NOT NULL,
|
||||
timezone VARCHAR NOT NULL,
|
||||
session_json VARCHAR NOT NULL DEFAULT '{}'
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE securities (
|
||||
market VARCHAR NOT NULL,
|
||||
symbol VARCHAR NOT NULL,
|
||||
name VARCHAR NOT NULL DEFAULT '',
|
||||
category VARCHAR NOT NULL,
|
||||
exchange VARCHAR NOT NULL DEFAULT '',
|
||||
list_date DATE,
|
||||
delist_date DATE,
|
||||
currency VARCHAR NOT NULL DEFAULT '',
|
||||
lot_size BIGINT,
|
||||
underlying VARCHAR,
|
||||
strike DOUBLE,
|
||||
maturity DATE,
|
||||
PRIMARY KEY (market, symbol)
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE calendar (
|
||||
market VARCHAR NOT NULL,
|
||||
trade_date DATE NOT NULL,
|
||||
is_open BOOLEAN NOT NULL,
|
||||
note VARCHAR NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (market, trade_date)
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE corp_actions (
|
||||
market VARCHAR NOT NULL,
|
||||
symbol VARCHAR NOT NULL,
|
||||
ex_date DATE NOT NULL,
|
||||
type VARCHAR NOT NULL,
|
||||
ratio DOUBLE,
|
||||
cash_div DOUBLE,
|
||||
PRIMARY KEY (market, symbol, ex_date, type)
|
||||
)
|
||||
""",
|
||||
# 申万 SCD2:官方发新版/追溯修订 → 开新 version(评审 P2)
|
||||
"""
|
||||
CREATE TABLE sw_industry (
|
||||
version VARCHAR NOT NULL,
|
||||
symbol VARCHAR NOT NULL,
|
||||
l1 VARCHAR NOT NULL DEFAULT '',
|
||||
l2 VARCHAR NOT NULL DEFAULT '',
|
||||
l3 VARCHAR NOT NULL DEFAULT '',
|
||||
effective_from DATE,
|
||||
effective_to DATE
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE stock_status (
|
||||
symbol VARCHAR NOT NULL,
|
||||
date DATE NOT NULL,
|
||||
is_st BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
st_reason VARCHAR NOT NULL DEFAULT '',
|
||||
is_suspended BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
limit_up DOUBLE,
|
||||
limit_down DOUBLE,
|
||||
pct_limit DOUBLE,
|
||||
source VARCHAR NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (symbol, date)
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE futures_meta (
|
||||
symbol VARCHAR PRIMARY KEY,
|
||||
contract VARCHAR NOT NULL DEFAULT '',
|
||||
multiplier DOUBLE,
|
||||
expiry DATE,
|
||||
daily_limit_rule VARCHAR NOT NULL DEFAULT ''
|
||||
)
|
||||
""",
|
||||
# ---- 行情(日K;1m 在 parquet,M2)-----------------------------------
|
||||
"""
|
||||
CREATE TABLE bars_1d (
|
||||
market VARCHAR NOT NULL,
|
||||
category VARCHAR NOT NULL,
|
||||
symbol VARCHAR NOT NULL,
|
||||
date DATE NOT NULL,
|
||||
open DOUBLE NOT NULL,
|
||||
high DOUBLE NOT NULL,
|
||||
low DOUBLE NOT NULL,
|
||||
close DOUBLE NOT NULL,
|
||||
volume DOUBLE NOT NULL DEFAULT 0,
|
||||
amount DOUBLE NOT NULL DEFAULT 0,
|
||||
turnover DOUBLE,
|
||||
adj_factor DOUBLE,
|
||||
PRIMARY KEY (market, category, symbol, date)
|
||||
)
|
||||
""",
|
||||
# ---- 信息流 ----------------------------------------------------------
|
||||
"""
|
||||
CREATE TABLE announcements (
|
||||
id VARCHAR PRIMARY KEY,
|
||||
symbol VARCHAR NOT NULL DEFAULT '',
|
||||
title VARCHAR NOT NULL DEFAULT '',
|
||||
category VARCHAR NOT NULL DEFAULT '',
|
||||
publish_ts TIMESTAMPTZ,
|
||||
url VARCHAR NOT NULL DEFAULT '',
|
||||
source VARCHAR NOT NULL DEFAULT '',
|
||||
hash VARCHAR NOT NULL DEFAULT ''
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE reports (
|
||||
id VARCHAR PRIMARY KEY,
|
||||
symbol VARCHAR NOT NULL DEFAULT '',
|
||||
org VARCHAR NOT NULL DEFAULT '',
|
||||
analyst VARCHAR NOT NULL DEFAULT '',
|
||||
rating VARCHAR NOT NULL DEFAULT '',
|
||||
title VARCHAR NOT NULL DEFAULT '',
|
||||
publish_ts TIMESTAMPTZ,
|
||||
url VARCHAR NOT NULL DEFAULT '',
|
||||
source VARCHAR NOT NULL DEFAULT ''
|
||||
)
|
||||
""",
|
||||
# 分钟新闻(NewsBridge 落库;断线补偿游标,评审 P1 补 DDL)
|
||||
"""
|
||||
CREATE TABLE news (
|
||||
id VARCHAR PRIMARY KEY,
|
||||
ts TIMESTAMPTZ NOT NULL,
|
||||
source VARCHAR NOT NULL DEFAULT '',
|
||||
market VARCHAR NOT NULL DEFAULT '',
|
||||
symbols_json VARCHAR NOT NULL DEFAULT '[]',
|
||||
level VARCHAR NOT NULL DEFAULT 'normal',
|
||||
title VARCHAR NOT NULL DEFAULT '',
|
||||
url VARCHAR NOT NULL DEFAULT '',
|
||||
hash VARCHAR NOT NULL DEFAULT ''
|
||||
)
|
||||
""",
|
||||
# ---- 运行时元数据 ------------------------------------------------------
|
||||
"""
|
||||
CREATE TABLE etl_checkpoint (
|
||||
dataset VARCHAR NOT NULL,
|
||||
market VARCHAR NOT NULL,
|
||||
scope VARCHAR NOT NULL,
|
||||
watermark VARCHAR NOT NULL,
|
||||
rows BIGINT NOT NULL DEFAULT 0,
|
||||
status VARCHAR NOT NULL DEFAULT 'ok',
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (dataset, market, scope)
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE export_jobs (
|
||||
job_id VARCHAR PRIMARY KEY,
|
||||
spec_json VARCHAR NOT NULL DEFAULT '{}',
|
||||
fmt VARCHAR NOT NULL,
|
||||
path VARCHAR NOT NULL DEFAULT '',
|
||||
rows BIGINT,
|
||||
bytes BIGINT,
|
||||
state VARCHAR NOT NULL DEFAULT 'queued',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
finished_at TIMESTAMPTZ
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE quality_reports (
|
||||
date DATE NOT NULL,
|
||||
market VARCHAR NOT NULL,
|
||||
check_name VARCHAR NOT NULL,
|
||||
result VARCHAR NOT NULL,
|
||||
detail_json VARCHAR NOT NULL DEFAULT '{}',
|
||||
PRIMARY KEY (date, market, check_name)
|
||||
)
|
||||
""",
|
||||
*all_day_views(),
|
||||
)
|
||||
|
||||
|
||||
# (版本, 名称, 语句元组) —— 追加新迁移时只往列表尾部加。
|
||||
# DuckDB 的 execute() 一次只吃一条语句,故迁移以「语句元组」为单位逐条执行。
|
||||
MIGRATIONS: tuple[tuple[int, str, tuple[str, ...]], ...] = (
|
||||
(1, "0001_initial", MIGRATION_0001_STATEMENTS),
|
||||
)
|
||||
|
||||
KNOWN_MAX_VERSION = MIGRATIONS[-1][0]
|
||||
|
||||
|
||||
class SchemaTooNewError(RuntimeError):
|
||||
"""库 schema 版本 > 代码已知最高版:拒绝启动(§11,不带病跑)。"""
|
||||
|
||||
def __init__(self, db_version: int, known_max: int) -> None:
|
||||
self.db_version = db_version
|
||||
self.known_max = known_max
|
||||
super().__init__(
|
||||
f"catalog.duckdb schema 版本 {db_version} 高于代码已知最高版 {known_max}:"
|
||||
f"请升级 pyquantdata 后再启动(§11 不带病跑)"
|
||||
)
|
||||
|
||||
|
||||
def _ensure_migrations_table(conn: duckdb.DuckDBPyConnection) -> None:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
"version INTEGER PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())"
|
||||
)
|
||||
|
||||
|
||||
def current_version(conn: duckdb.DuckDBPyConnection) -> int:
|
||||
"""当前库版本;空库(无 schema_migrations 表)返回 0。"""
|
||||
has_table = conn.execute(
|
||||
"SELECT count(*) FROM information_schema.tables "
|
||||
"WHERE table_schema='main' AND table_name='schema_migrations'"
|
||||
).fetchone()
|
||||
if not has_table or has_table[0] == 0:
|
||||
return 0
|
||||
row = conn.execute("SELECT max(version) FROM schema_migrations").fetchone()
|
||||
return int(row[0]) if row and row[0] is not None else 0
|
||||
|
||||
|
||||
def migrate(conn: duckdb.DuckDBPyConnection) -> int:
|
||||
"""把库升到代码已知最高版,返回最终版本。幂等:已应用过则跳过。"""
|
||||
_ensure_migrations_table(conn)
|
||||
db_version = current_version(conn)
|
||||
if db_version > KNOWN_MAX_VERSION:
|
||||
raise SchemaTooNewError(db_version, KNOWN_MAX_VERSION)
|
||||
for version, _name, statements in MIGRATIONS:
|
||||
if version <= db_version:
|
||||
continue
|
||||
conn.begin()
|
||||
try:
|
||||
for stmt in statements:
|
||||
conn.execute(stmt)
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (?)", [version])
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
return current_version(conn)
|
||||
|
||||
|
||||
def applied_versions(conn: duckdb.DuckDBPyConnection) -> list[int]:
|
||||
_ensure_migrations_table(conn)
|
||||
rows = conn.execute("SELECT version FROM schema_migrations ORDER BY version").fetchall()
|
||||
return [int(r[0]) for r in rows]
|
||||
@@ -0,0 +1,36 @@
|
||||
"""联邦视图 DDL(设计 §3.3)。视图全部纳入 schema_migrations(评审 P1)。
|
||||
|
||||
规则:
|
||||
- 显式列名,禁 ``*``(union/视图列错位不报错的隐患直接堵死);
|
||||
- bars_1m 系视图 M2 加(1m 数据 M2 回填;视图 DDL 跟 migration 版本走)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
_BAR_COLUMNS = (
|
||||
"market, category, symbol, date, open, high, low, close, volume, amount, turnover, adj_factor"
|
||||
)
|
||||
|
||||
# (view_name, market, category) —— M1 只做日K视图(数据在 bars_1d 物理表)
|
||||
DAY_VIEWS: tuple[tuple[str, str, str], ...] = (
|
||||
("cn_stock_1d", "cn", "stock"),
|
||||
("cn_index_1d", "cn", "index"),
|
||||
("cn_etf_1d", "cn", "etf"),
|
||||
("cn_cbond_1d", "cn", "cbond"),
|
||||
("us_stock_1d", "us", "stock"),
|
||||
("us_index_1d", "us", "index"),
|
||||
("hk_stock_1d", "hk", "stock"),
|
||||
("hk_index_1d", "hk", "index"),
|
||||
)
|
||||
|
||||
|
||||
def day_view_sql(name: str, market: str, category: str) -> str:
|
||||
return (
|
||||
f"CREATE OR REPLACE VIEW {name} AS "
|
||||
f"SELECT {_BAR_COLUMNS} FROM bars_1d "
|
||||
f"WHERE market = '{market}' AND category = '{category}'"
|
||||
)
|
||||
|
||||
|
||||
def all_day_views() -> tuple[str, ...]:
|
||||
return tuple(day_view_sql(*v) for v in DAY_VIEWS)
|
||||
Reference in new issue
Block a user