diff --git a/src/pyquantdata/cli.py b/src/pyquantdata/cli.py index 88e8dcf..2b384c8 100644 --- a/src/pyquantdata/cli.py +++ b/src/pyquantdata/cli.py @@ -22,6 +22,7 @@ from .config import AppConfig, default_config, load_config, write_config from .etl import SeedError, import_seed from .export.files import ( EXPORT_FORMATS, + write_arrow_file, write_export, ) from .paths import DbPathLayout, resolve_dbpath @@ -247,7 +248,13 @@ def query( ) ) else: - result = write_export(tbl, "csv" if fmt == "csv" else fmt, Path(o)) + # arrow 单走 IPC file 落盘(write_export 仅 csv/parquet,直接调会抛 + # ExportFormatError —— 0.1.0 实测 bug,此处曾把 arrow 透传进去) + result = ( + write_arrow_file(tbl, Path(o)) + if fmt == "arrow" + else write_export(tbl, fmt, Path(o)) + ) console.print(f"[green]已写出[/green] {result.path}({result.rows} 行 / {result.bytes_written} 字节)") finally: conn.close() diff --git a/src/pyquantdata/export/files.py b/src/pyquantdata/export/files.py index 13b895d..7a9075f 100644 --- a/src/pyquantdata/export/files.py +++ b/src/pyquantdata/export/files.py @@ -93,6 +93,21 @@ def write_export(table: pa.Table, fmt: str, target: Path) -> ExportResult: return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt=fmt) +def write_arrow_file(table: pa.Table, target: Path) -> ExportResult: + """``query --fmt arrow -o`` 专用:Arrow IPC **file** 格式落盘。 + + 与 HTTP ``/v1/query`` 的 ``b64_arrow``(streaming format)刻意区分: + 落盘产物要给 ``polars.read_ipc`` / ``pyarrow.ipc.open_file`` 随时重开, + file 格式带 footer 与随机访问;streaming 只适合一次性管道传输。 + 不进 ``EXPORT_FORMATS`` —— 那是 export 命令 / HTTP export 的语义(csv/parquet)。 + """ + target.parent.mkdir(parents=True, exist_ok=True) + with pa.ipc.new_file(target, table.schema) as writer: + writer.write_table(table) + size = target.stat().st_size + return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt="arrow") + + def record_export_job( conn: duckdb.DuckDBPyConnection, job_id: str, diff --git a/tests/test_cli.py b/tests/test_cli.py index ebef5fb..b2c9503 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -92,6 +92,29 @@ def test_query_csv_to_file(initialized, tmp_path): assert out.exists() +def test_query_arrow_to_file(initialized, tmp_path): + """query --fmt arrow 落盘 = Arrow IPC **file** 格式(0.1.0 bug 回归)。 + + 原实现把 arrow 透传给只认 csv/parquet 的 write_export → ExportFormatError + 裸 traceback;修复后必须写出可被 open_file 随机重开的 footer 完整文件。 + """ + out = tmp_path / "q.arrow" + result = runner.invoke(app, [ + "query", "-d", str(initialized), "--fmt", "arrow", "-o", str(out), + "SELECT symbol, close FROM cn_stock_1d", + ]) + assert result.exit_code == 0, result.output + assert "已写出" in result.output + assert out.exists() and out.stat().st_size > 0 + import pyarrow as pa + import pyarrow.ipc as pai + + with pai.open_file(out) as reader: # file 格式(footer 随机访问),非 stream + tbl = reader.read_all() + assert tbl.num_rows == 10 + assert tbl.schema.names == ["symbol", "close"] + + def test_query_csv_without_out_exit_1(initialized): result = runner.invoke(app, [ "query", "-d", str(initialized), "--fmt", "csv", "SELECT 1",