fix(cli): query --fmt arrow -o 落盘修复(Arrow IPC file 格式)
原实现把 arrow 透传给只认 csv/parquet 的 write_export → ExportFormatError 裸 traceback(0.1.0 实测 bug)。新增 write_arrow_file 写 IPC file 格式 (footer 完整可随机重开,与 HTTP b64_arrow 的 streaming 格式刻意区分), EXPORT_FORMATS/HTTP/export 命令行为不变。补回归测试 255 passed / 100%。
This commit is contained in:
1 parent
a46ab0d677
commit
b5f0ae3204
3 files changed
+46
-1
No files matched your search
@@ -22,6 +22,7 @@ from .config import AppConfig, default_config, load_config, write_config
|
||||
from .etl import SeedError, import_seed
|
||||
from .export.files import (
|
||||
EXPORT_FORMATS,
|
||||
write_arrow_file,
|
||||
write_export,
|
||||
)
|
||||
from .paths import DbPathLayout, resolve_dbpath
|
||||
@@ -247,7 +248,13 @@ def query(
|
||||
)
|
||||
)
|
||||
else:
|
||||
result = write_export(tbl, "csv" if fmt == "csv" else fmt, Path(o))
|
||||
# arrow 单走 IPC file 落盘(write_export 仅 csv/parquet,直接调会抛
|
||||
# ExportFormatError —— 0.1.0 实测 bug,此处曾把 arrow 透传进去)
|
||||
result = (
|
||||
write_arrow_file(tbl, Path(o))
|
||||
if fmt == "arrow"
|
||||
else write_export(tbl, fmt, Path(o))
|
||||
)
|
||||
console.print(f"[green]已写出[/green] {result.path}({result.rows} 行 / {result.bytes_written} 字节)")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
@@ -93,6 +93,21 @@ def write_export(table: pa.Table, fmt: str, target: Path) -> ExportResult:
|
||||
return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt=fmt)
|
||||
|
||||
|
||||
def write_arrow_file(table: pa.Table, target: Path) -> ExportResult:
|
||||
"""``query --fmt arrow -o`` 专用:Arrow IPC **file** 格式落盘。
|
||||
|
||||
与 HTTP ``/v1/query`` 的 ``b64_arrow``(streaming format)刻意区分:
|
||||
落盘产物要给 ``polars.read_ipc`` / ``pyarrow.ipc.open_file`` 随时重开,
|
||||
file 格式带 footer 与随机访问;streaming 只适合一次性管道传输。
|
||||
不进 ``EXPORT_FORMATS`` —— 那是 export 命令 / HTTP export 的语义(csv/parquet)。
|
||||
"""
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with pa.ipc.new_file(target, table.schema) as writer:
|
||||
writer.write_table(table)
|
||||
size = target.stat().st_size
|
||||
return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt="arrow")
|
||||
|
||||
|
||||
def record_export_job(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
job_id: str,
|
||||
|
||||
@@ -92,6 +92,29 @@ def test_query_csv_to_file(initialized, tmp_path):
|
||||
assert out.exists()
|
||||
|
||||
|
||||
def test_query_arrow_to_file(initialized, tmp_path):
|
||||
"""query --fmt arrow 落盘 = Arrow IPC **file** 格式(0.1.0 bug 回归)。
|
||||
|
||||
原实现把 arrow 透传给只认 csv/parquet 的 write_export → ExportFormatError
|
||||
裸 traceback;修复后必须写出可被 open_file 随机重开的 footer 完整文件。
|
||||
"""
|
||||
out = tmp_path / "q.arrow"
|
||||
result = runner.invoke(app, [
|
||||
"query", "-d", str(initialized), "--fmt", "arrow", "-o", str(out),
|
||||
"SELECT symbol, close FROM cn_stock_1d",
|
||||
])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "已写出" in result.output
|
||||
assert out.exists() and out.stat().st_size > 0
|
||||
import pyarrow as pa
|
||||
import pyarrow.ipc as pai
|
||||
|
||||
with pai.open_file(out) as reader: # file 格式(footer 随机访问),非 stream
|
||||
tbl = reader.read_all()
|
||||
assert tbl.num_rows == 10
|
||||
assert tbl.schema.names == ["symbol", "close"]
|
||||
|
||||
|
||||
def test_query_csv_without_out_exit_1(initialized):
|
||||
result = runner.invoke(app, [
|
||||
"query", "-d", str(initialized), "--fmt", "csv", "SELECT 1",
|
||||
|
||||
Reference in new issue
Block a user