From b5f0ae3204f3bd49162a04d73f8fb31bc9a8607e Mon Sep 17 00:00:00 2001 From: openbot-coder Date: Mon, 28 Sep 2026 15:33:15 +0800 Subject: [PATCH] =?UTF-8?q?fix(cli):=20query=20--fmt=20arrow=20-o=20?= =?UTF-8?q?=E8=90=BD=E7=9B=98=E4=BF=AE=E5=A4=8D=EF=BC=88Arrow=20IPC=20file?= =?UTF-8?q?=20=E6=A0=BC=E5=BC=8F=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原实现把 arrow 透传给只认 csv/parquet 的 write_export → ExportFormatError 裸 traceback(0.1.0 实测 bug)。新增 write_arrow_file 写 IPC file 格式 (footer 完整可随机重开,与 HTTP b64_arrow 的 streaming 格式刻意区分), EXPORT_FORMATS/HTTP/export 命令行为不变。补回归测试 255 passed / 100%。 --- src/pyquantdata/cli.py | 9 ++++++++- src/pyquantdata/export/files.py | 15 +++++++++++++++ tests/test_cli.py | 23 +++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/pyquantdata/cli.py b/src/pyquantdata/cli.py index 88e8dcf..2b384c8 100644 --- a/src/pyquantdata/cli.py +++ b/src/pyquantdata/cli.py @@ -22,6 +22,7 @@ from .config import AppConfig, default_config, load_config, write_config from .etl import SeedError, import_seed from .export.files import ( EXPORT_FORMATS, + write_arrow_file, write_export, ) from .paths import DbPathLayout, resolve_dbpath @@ -247,7 +248,13 @@ def query( ) ) else: - result = write_export(tbl, "csv" if fmt == "csv" else fmt, Path(o)) + # arrow 单走 IPC file 落盘(write_export 仅 csv/parquet,直接调会抛 + # ExportFormatError —— 0.1.0 实测 bug,此处曾把 arrow 透传进去) + result = ( + write_arrow_file(tbl, Path(o)) + if fmt == "arrow" + else write_export(tbl, fmt, Path(o)) + ) console.print(f"[green]已写出[/green] {result.path}({result.rows} 行 / {result.bytes_written} 字节)") finally: conn.close() diff --git a/src/pyquantdata/export/files.py b/src/pyquantdata/export/files.py index 13b895d..7a9075f 100644 --- a/src/pyquantdata/export/files.py +++ b/src/pyquantdata/export/files.py @@ -93,6 +93,21 @@ def write_export(table: pa.Table, fmt: str, target: Path) -> ExportResult: return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt=fmt) +def write_arrow_file(table: pa.Table, target: Path) -> ExportResult: + """``query --fmt arrow -o`` 专用:Arrow IPC **file** 格式落盘。 + + 与 HTTP ``/v1/query`` 的 ``b64_arrow``(streaming format)刻意区分: + 落盘产物要给 ``polars.read_ipc`` / ``pyarrow.ipc.open_file`` 随时重开, + file 格式带 footer 与随机访问;streaming 只适合一次性管道传输。 + 不进 ``EXPORT_FORMATS`` —— 那是 export 命令 / HTTP export 的语义(csv/parquet)。 + """ + target.parent.mkdir(parents=True, exist_ok=True) + with pa.ipc.new_file(target, table.schema) as writer: + writer.write_table(table) + size = target.stat().st_size + return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt="arrow") + + def record_export_job( conn: duckdb.DuckDBPyConnection, job_id: str, diff --git a/tests/test_cli.py b/tests/test_cli.py index ebef5fb..b2c9503 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -92,6 +92,29 @@ def test_query_csv_to_file(initialized, tmp_path): assert out.exists() +def test_query_arrow_to_file(initialized, tmp_path): + """query --fmt arrow 落盘 = Arrow IPC **file** 格式(0.1.0 bug 回归)。 + + 原实现把 arrow 透传给只认 csv/parquet 的 write_export → ExportFormatError + 裸 traceback;修复后必须写出可被 open_file 随机重开的 footer 完整文件。 + """ + out = tmp_path / "q.arrow" + result = runner.invoke(app, [ + "query", "-d", str(initialized), "--fmt", "arrow", "-o", str(out), + "SELECT symbol, close FROM cn_stock_1d", + ]) + assert result.exit_code == 0, result.output + assert "已写出" in result.output + assert out.exists() and out.stat().st_size > 0 + import pyarrow as pa + import pyarrow.ipc as pai + + with pai.open_file(out) as reader: # file 格式(footer 随机访问),非 stream + tbl = reader.read_all() + assert tbl.num_rows == 10 + assert tbl.schema.names == ["symbol", "close"] + + def test_query_csv_without_out_exit_1(initialized): result = runner.invoke(app, [ "query", "-d", str(initialized), "--fmt", "csv", "SELECT 1",