Files
pyquantdata/src/pyquantdata/export/files.py
T
openbot b5f0ae3204 fix(cli): query --fmt arrow -o 落盘修复(Arrow IPC file 格式)
原实现把 arrow 透传给只认 csv/parquet 的 write_export → ExportFormatError
裸 traceback(0.1.0 实测 bug)。新增 write_arrow_file 写 IPC file 格式
(footer 完整可随机重开,与 HTTP b64_arrow 的 streaming 格式刻意区分),
EXPORT_FORMATS/HTTP/export 命令行为不变。补回归测试 255 passed / 100%。
2026-09-28 15:33:15 +08:00

149 lines
5.1 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""导出落盘(设计 §9)。
- export 只管落盘格式:csv / parquet(pandas/polars 走 /v1/query fmt=arrow 消费);
- 路径安全(评审 P1):输出根恒为 exports/{job_id}/,名称字段白名单字符过滤 +
resolve 后必须仍在 exports/ 前缀内;杜绝 ../ 穿出 dbpath;
- job 结果记 export_jobs 表(spec hash 缓存 24h 的键在 M2 worker 化时接入)。
"""
from __future__ import annotations
import hashlib
import json
import re
import uuid
from dataclasses import dataclass
from pathlib import Path
import duckdb
import pyarrow as pa
import pyarrow.csv as pacsv
import pyarrow.parquet as paparquet
_JOB_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$")
_FILENAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
EXPORT_FORMATS: frozenset[str] = frozenset({"csv", "parquet"})
class ExportPathError(ValueError):
pass
class ExportFormatError(ValueError):
pass
def new_job_id() -> str:
return uuid.uuid4().hex[:12]
def spec_hash(spec: dict) -> str:
"""导出请求的稳定指纹(缓存键,§9)。"""
canonical = json.dumps(spec, sort_keys=True, ensure_ascii=False)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def resolve_export_target(exports_dir: Path, job_id: str, filename: str) -> Path:
"""exports/{job_id}/{filename};白名单 + resolve 前缀校验(§9 路径安全)。
纵深防御:``_JOB_ID_RE`` / ``_FILENAME_RE`` 已禁掉 ``/`` 与 ``..``,因此常规
输入下 resolve 后必然在 exports/ 内。最后的 startswith 校验是为**符号链接**
(exports/{job_id} 被替换成指向外部的软链)兜底——该分支在无软链的环境里
结构性不可达,保留为防线而非死代码。
"""
if not _JOB_ID_RE.match(job_id):
raise ExportPathError(f"非法 job_id:{job_id!r}")
if not _FILENAME_RE.match(filename):
raise ExportPathError(f"非法导出文件名:{filename!r}")
exports_dir = Path(exports_dir).resolve()
target = (exports_dir / job_id / filename).resolve()
root_str = str(exports_dir).rstrip("\\/") + "\\"
posix_root = str(exports_dir).rstrip("\\/") + "/"
if not (str(target).startswith(root_str) or str(target).startswith(posix_root)):
raise ExportPathError(f"导出路径越界:{target}")
return target
@dataclass
class ExportResult:
path: Path
rows: int
bytes_written: int
fmt: str
def run_query_arrow(
conn: duckdb.DuckDBPyConnection, sql: str, params: list | None = None
) -> pa.Table:
"""执行查询返回 Arrow 表(查询/导出共用;调用方负责 sqlgate 校验与线程池)。"""
result = conn.execute(sql, params or [])
return result.arrow().read_all()
def write_export(table: pa.Table, fmt: str, target: Path) -> ExportResult:
if fmt not in EXPORT_FORMATS:
raise ExportFormatError(f"export 仅支持落盘格式 {sorted(EXPORT_FORMATS)},收到 {fmt!r}")
target.parent.mkdir(parents=True, exist_ok=True)
if fmt == "csv":
pacsv.write_csv(table, target)
else:
paparquet.write_table(table, target, compression="zstd")
size = target.stat().st_size
return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt=fmt)
def write_arrow_file(table: pa.Table, target: Path) -> ExportResult:
"""``query --fmt arrow -o`` 专用:Arrow IPC **file** 格式落盘。
与 HTTP ``/v1/query`` 的 ``b64_arrow``(streaming format)刻意区分:
落盘产物要给 ``polars.read_ipc`` / ``pyarrow.ipc.open_file`` 随时重开,
file 格式带 footer 与随机访问;streaming 只适合一次性管道传输。
不进 ``EXPORT_FORMATS`` —— 那是 export 命令 / HTTP export 的语义(csv/parquet)。
"""
target.parent.mkdir(parents=True, exist_ok=True)
with pa.ipc.new_file(target, table.schema) as writer:
writer.write_table(table)
size = target.stat().st_size
return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt="arrow")
def record_export_job(
conn: duckdb.DuckDBPyConnection,
job_id: str,
*,
fmt: str,
path: str,
rows: int,
bytes_written: int,
state: str = "done",
spec_json: str = "{}",
) -> None:
conn.execute(
"INSERT OR REPLACE INTO export_jobs "
"(job_id, spec_json, fmt, path, rows, bytes, state, created_at, finished_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, now(), now())",
[job_id, spec_json, fmt, path, rows, bytes_written, state],
)
def get_export_job(conn: duckdb.DuckDBPyConnection, job_id: str) -> dict | None:
row = conn.execute(
"SELECT job_id, spec_json, fmt, path, rows, bytes, state, "
"created_at, finished_at FROM export_jobs WHERE job_id = ?",
[job_id],
).fetchone()
if row is None:
return None
return {
"job_id": row[0],
"spec_json": row[1],
"fmt": row[2],
"path": row[3],
"rows": row[4],
"bytes": row[5],
"state": row[6],
"created_at": None if row[7] is None else row[7].isoformat(),
"finished_at": None if row[8] is None else row[8].isoformat(),
}