fix(cli): query --fmt arrow -o 落盘修复(Arrow IPC file 格式)
原实现把 arrow 透传给只认 csv/parquet 的 write_export → ExportFormatError 裸 traceback(0.1.0 实测 bug)。新增 write_arrow_file 写 IPC file 格式 (footer 完整可随机重开,与 HTTP b64_arrow 的 streaming 格式刻意区分), EXPORT_FORMATS/HTTP/export 命令行为不变。补回归测试 255 passed / 100%。
This commit is contained in:
1 parent
a46ab0d677
commit
b5f0ae3204
3 files changed
+46
-1
No files matched your search
@@ -22,6 +22,7 @@ from .config import AppConfig, default_config, load_config, write_config
|
|||||||
from .etl import SeedError, import_seed
|
from .etl import SeedError, import_seed
|
||||||
from .export.files import (
|
from .export.files import (
|
||||||
EXPORT_FORMATS,
|
EXPORT_FORMATS,
|
||||||
|
write_arrow_file,
|
||||||
write_export,
|
write_export,
|
||||||
)
|
)
|
||||||
from .paths import DbPathLayout, resolve_dbpath
|
from .paths import DbPathLayout, resolve_dbpath
|
||||||
@@ -247,7 +248,13 @@ def query(
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
result = write_export(tbl, "csv" if fmt == "csv" else fmt, Path(o))
|
# arrow 单走 IPC file 落盘(write_export 仅 csv/parquet,直接调会抛
|
||||||
|
# ExportFormatError —— 0.1.0 实测 bug,此处曾把 arrow 透传进去)
|
||||||
|
result = (
|
||||||
|
write_arrow_file(tbl, Path(o))
|
||||||
|
if fmt == "arrow"
|
||||||
|
else write_export(tbl, fmt, Path(o))
|
||||||
|
)
|
||||||
console.print(f"[green]已写出[/green] {result.path}({result.rows} 行 / {result.bytes_written} 字节)")
|
console.print(f"[green]已写出[/green] {result.path}({result.rows} 行 / {result.bytes_written} 字节)")
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|||||||
@@ -93,6 +93,21 @@ def write_export(table: pa.Table, fmt: str, target: Path) -> ExportResult:
|
|||||||
return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt=fmt)
|
return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt=fmt)
|
||||||
|
|
||||||
|
|
||||||
|
def write_arrow_file(table: pa.Table, target: Path) -> ExportResult:
|
||||||
|
"""``query --fmt arrow -o`` 专用:Arrow IPC **file** 格式落盘。
|
||||||
|
|
||||||
|
与 HTTP ``/v1/query`` 的 ``b64_arrow``(streaming format)刻意区分:
|
||||||
|
落盘产物要给 ``polars.read_ipc`` / ``pyarrow.ipc.open_file`` 随时重开,
|
||||||
|
file 格式带 footer 与随机访问;streaming 只适合一次性管道传输。
|
||||||
|
不进 ``EXPORT_FORMATS`` —— 那是 export 命令 / HTTP export 的语义(csv/parquet)。
|
||||||
|
"""
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with pa.ipc.new_file(target, table.schema) as writer:
|
||||||
|
writer.write_table(table)
|
||||||
|
size = target.stat().st_size
|
||||||
|
return ExportResult(path=target, rows=table.num_rows, bytes_written=size, fmt="arrow")
|
||||||
|
|
||||||
|
|
||||||
def record_export_job(
|
def record_export_job(
|
||||||
conn: duckdb.DuckDBPyConnection,
|
conn: duckdb.DuckDBPyConnection,
|
||||||
job_id: str,
|
job_id: str,
|
||||||
|
|||||||
@@ -92,6 +92,29 @@ def test_query_csv_to_file(initialized, tmp_path):
|
|||||||
assert out.exists()
|
assert out.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_query_arrow_to_file(initialized, tmp_path):
|
||||||
|
"""query --fmt arrow 落盘 = Arrow IPC **file** 格式(0.1.0 bug 回归)。
|
||||||
|
|
||||||
|
原实现把 arrow 透传给只认 csv/parquet 的 write_export → ExportFormatError
|
||||||
|
裸 traceback;修复后必须写出可被 open_file 随机重开的 footer 完整文件。
|
||||||
|
"""
|
||||||
|
out = tmp_path / "q.arrow"
|
||||||
|
result = runner.invoke(app, [
|
||||||
|
"query", "-d", str(initialized), "--fmt", "arrow", "-o", str(out),
|
||||||
|
"SELECT symbol, close FROM cn_stock_1d",
|
||||||
|
])
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "已写出" in result.output
|
||||||
|
assert out.exists() and out.stat().st_size > 0
|
||||||
|
import pyarrow as pa
|
||||||
|
import pyarrow.ipc as pai
|
||||||
|
|
||||||
|
with pai.open_file(out) as reader: # file 格式(footer 随机访问),非 stream
|
||||||
|
tbl = reader.read_all()
|
||||||
|
assert tbl.num_rows == 10
|
||||||
|
assert tbl.schema.names == ["symbol", "close"]
|
||||||
|
|
||||||
|
|
||||||
def test_query_csv_without_out_exit_1(initialized):
|
def test_query_csv_without_out_exit_1(initialized):
|
||||||
result = runner.invoke(app, [
|
result = runner.invoke(app, [
|
||||||
"query", "-d", str(initialized), "--fmt", "csv", "SELECT 1",
|
"query", "-d", str(initialized), "--fmt", "csv", "SELECT 1",
|
||||||
|
|||||||
Reference in new issue
Block a user