fix(cli): query --fmt arrow -o 落盘修复(Arrow IPC file 格式)

原实现把 arrow 透传给只认 csv/parquet 的 write_export → ExportFormatError
裸 traceback(0.1.0 实测 bug)。新增 write_arrow_file 写 IPC file 格式
(footer 完整可随机重开,与 HTTP b64_arrow 的 streaming 格式刻意区分),
EXPORT_FORMATS/HTTP/export 命令行为不变。补回归测试 255 passed / 100%。
This commit is contained in:
openbot committed 2026-09-28 15:33:15 +08:00
1 parent a46ab0d677
commit b5f0ae3204
3 files changed
+46 -1

No files matched your search

+8 -1
View File
@@ -22,6 +22,7 @@ from .config import AppConfig, default_config, load_config, write_config
from .etl import SeedError, import_seed from .etl import SeedError, import_seed
from .export.files import ( from .export.files import (
EXPORT_FORMATS, EXPORT_FORMATS,
write_arrow_file,
write_export, write_export,
) )
from .paths import DbPathLayout, resolve_dbpath from .paths import DbPathLayout, resolve_dbpath
@@ -247,7 +248,13 @@ def query(
) )
) )
else: else:
result = write_export(tbl, "csv" if fmt == "csv" else fmt, Path(o)) # arrow 单走 IPC file 落盘(write_export 仅 csv/parquet,直接调会抛
# ExportFormatError —— 0.1.0 实测 bug,此处曾把 arrow 透传进去)
result = (
write_arrow_file(tbl, Path(o))
if fmt == "arrow"
else write_export(tbl, fmt, Path(o))
)
console.print(f"[green]已写出[/green] {result.path}({result.rows} 行 / {result.bytes_written} 字节)") console.print(f"[green]已写出[/green] {result.path}({result.rows} 行 / {result.bytes_written} 字节)")
finally: finally:
conn.close() conn.close()
+15
View File
@@ -93,6 +93,21 @@ def write_export(table: pa.Table, fmt: str, target: Path) -> ExportResult:
return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt=fmt) return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt=fmt)
def write_arrow_file(table: pa.Table, target: Path) -> ExportResult:
"""``query --fmt arrow -o`` 专用:Arrow IPC **file** 格式落盘。
与 HTTP ``/v1/query`` 的 ``b64_arrow``(streaming format)刻意区分:
落盘产物要给 ``polars.read_ipc`` / ``pyarrow.ipc.open_file`` 随时重开,
file 格式带 footer 与随机访问;streaming 只适合一次性管道传输。
不进 ``EXPORT_FORMATS`` —— 那是 export 命令 / HTTP export 的语义(csv/parquet)。
"""
target.parent.mkdir(parents=True, exist_ok=True)
with pa.ipc.new_file(target, table.schema) as writer:
writer.write_table(table)
size = target.stat().st_size
return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt="arrow")
def record_export_job( def record_export_job(
conn: duckdb.DuckDBPyConnection, conn: duckdb.DuckDBPyConnection,
job_id: str, job_id: str,
+23
View File
@@ -92,6 +92,29 @@ def test_query_csv_to_file(initialized, tmp_path):
assert out.exists() assert out.exists()
def test_query_arrow_to_file(initialized, tmp_path):
"""query --fmt arrow 落盘 = Arrow IPC **file** 格式(0.1.0 bug 回归)。
原实现把 arrow 透传给只认 csv/parquet 的 write_export → ExportFormatError
裸 traceback;修复后必须写出可被 open_file 随机重开的 footer 完整文件。
"""
out = tmp_path / "q.arrow"
result = runner.invoke(app, [
"query", "-d", str(initialized), "--fmt", "arrow", "-o", str(out),
"SELECT symbol, close FROM cn_stock_1d",
])
assert result.exit_code == 0, result.output
assert "已写出" in result.output
assert out.exists() and out.stat().st_size > 0
import pyarrow as pa
import pyarrow.ipc as pai
with pai.open_file(out) as reader: # file 格式(footer 随机访问),非 stream
tbl = reader.read_all()
assert tbl.num_rows == 10
assert tbl.schema.names == ["symbol", "close"]
def test_query_csv_without_out_exit_1(initialized): def test_query_csv_without_out_exit_1(initialized):
result = runner.invoke(app, [ result = runner.invoke(app, [
"query", "-d", str(initialized), "--fmt", "csv", "SELECT 1", "query", "-d", str(initialized), "--fmt", "csv", "SELECT 1",