From 9183c3294f7f4c9db3f35b4f7279e138f9941dbf Mon Sep 17 00:00:00 2001 From: openbot-coder Date: Tue, 15 Sep 2026 11:38:44 +0800 Subject: [PATCH] =?UTF-8?q?v0.4.0:=20=E5=88=9D=E5=A7=8B=E5=BC=80=E6=BA=90?= =?UTF-8?q?=E5=8F=91=E5=B8=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 通达信(TDX)行情数据客户端 —— 动态连接池 + 双层TTL缓存 + 3连接并发分页, 纯 asyncio、零运行时依赖。一套接口覆盖 A股(沪/深/北) / 港股 / 美股 / 期货 / 期权。 主要能力: - 四层架构:api → pool(标准/MAC/EX 三池) → protocol(16 命令模块) → cache(内存+SQLite) - TdxData 提供 51 个公开异步方法;CLI(tdx) 提供 19 个子命令 - 服务器文件下载 + 解析:板块 block_*.dat / 行业 tdxhy.cfg / 历史财报 gpcw*.zip - 85 个离线测试(协议 / 连接池 / 缓存 / 分页 / API / 文件解析 / CLI) 协议层参照 easy-tdx 的字节级格式独立实现,不依赖 easy-tdx 包。 --- .gitignore | 30 + .python-version | 1 + LICENSE | 21 + README.md | 359 +++++ docs/API_REFERENCE.md | 238 +++ examples/quickstart.py | 48 + examples/server_files.py | 50 + pyproject.toml | 56 + src/__init__.py | 0 src/pytdxdata/__init__.py | 33 + src/pytdxdata/_binary.py | 20 + src/pytdxdata/api.py | 1291 +++++++++++++++++ src/pytdxdata/cache/__init__.py | 0 src/pytdxdata/cache/disk.py | 148 ++ src/pytdxdata/cache/key.py | 46 + src/pytdxdata/cache/manager.py | 125 ++ src/pytdxdata/cache/memory.py | 63 + src/pytdxdata/cache/ttl.py | 50 + src/pytdxdata/cli.py | 541 +++++++ src/pytdxdata/codec/__init__.py | 20 + src/pytdxdata/codec/block.py | 73 + src/pytdxdata/codec/financial.py | 102 ++ src/pytdxdata/codec/industry.py | 40 + src/pytdxdata/codec/payload.py | 16 + src/pytdxdata/config.py | 127 ++ src/pytdxdata/indicator.py | 136 ++ src/pytdxdata/models/__init__.py | 407 ++++++ src/pytdxdata/pool/__init__.py | 0 src/pytdxdata/pool/connection.py | 137 ++ src/pytdxdata/pool/ex_connection.py | 103 ++ src/pytdxdata/pool/ex_pool.py | 119 ++ src/pytdxdata/pool/health.py | 61 + src/pytdxdata/pool/pool.py | 328 +++++ src/pytdxdata/pool/probe_cache.py | 162 +++ src/pytdxdata/protocol/__init__.py | 0 src/pytdxdata/protocol/commands/__init__.py | 32 + src/pytdxdata/protocol/commands/base.py | 38 + src/pytdxdata/protocol/commands/ex_ext.py | 171 +++ src/pytdxdata/protocol/commands/ex_proto.py | 125 ++ src/pytdxdata/protocol/commands/finance.py | 64 + src/pytdxdata/protocol/commands/kline.py | 115 ++ src/pytdxdata/protocol/commands/mac.py | 80 + src/pytdxdata/protocol/commands/mac_ext.py | 508 +++++++ src/pytdxdata/protocol/commands/mac_tick.py | 71 + src/pytdxdata/protocol/commands/minute.py | 45 + src/pytdxdata/protocol/commands/quotes.py | 122 ++ .../protocol/commands/security_list.py | 60 + src/pytdxdata/protocol/commands/setup.py | 9 + src/pytdxdata/protocol/commands/std_ext.py | 195 +++ .../protocol/commands/transaction.py | 126 ++ src/pytdxdata/protocol/commands/xdxr.py | 85 ++ src/pytdxdata/protocol/datetime_.py | 28 + src/pytdxdata/protocol/frame.py | 43 + src/pytdxdata/protocol/mac_frame.py | 22 + src/pytdxdata/protocol/price.py | 67 + src/pytdxdata/protocol/volume.py | 46 + src/pytdxdata/router.py | 20 + src/pytdxdata/scheduler/__init__.py | 0 src/pytdxdata/scheduler/paginator.py | 155 ++ src/pytdxdata/server_probe.py | 148 ++ src/pytdxdata/servers.json | 488 +++++++ tests/test_api.py | 331 +++++ tests/test_cache.py | 144 ++ tests/test_cli_files.py | 158 ++ tests/test_codec_files.py | 319 ++++ tests/test_pool.py | 120 ++ tests/test_protocol.py | 178 +++ uv.lock | 185 +++ 68 files changed, 9249 insertions(+) create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 LICENSE create mode 100644 README.md create mode 100644 docs/API_REFERENCE.md create mode 100644 examples/quickstart.py create mode 100644 examples/server_files.py create mode 100644 pyproject.toml create mode 100644 src/__init__.py create mode 100644 src/pytdxdata/__init__.py create mode 100644 src/pytdxdata/_binary.py create mode 100644 src/pytdxdata/api.py create mode 100644 src/pytdxdata/cache/__init__.py create mode 100644 src/pytdxdata/cache/disk.py create mode 100644 src/pytdxdata/cache/key.py create mode 100644 src/pytdxdata/cache/manager.py create mode 100644 src/pytdxdata/cache/memory.py create mode 100644 src/pytdxdata/cache/ttl.py create mode 100644 src/pytdxdata/cli.py create mode 100644 src/pytdxdata/codec/__init__.py create mode 100644 src/pytdxdata/codec/block.py create mode 100644 src/pytdxdata/codec/financial.py create mode 100644 src/pytdxdata/codec/industry.py create mode 100644 src/pytdxdata/codec/payload.py create mode 100644 src/pytdxdata/config.py create mode 100644 src/pytdxdata/indicator.py create mode 100644 src/pytdxdata/models/__init__.py create mode 100644 src/pytdxdata/pool/__init__.py create mode 100644 src/pytdxdata/pool/connection.py create mode 100644 src/pytdxdata/pool/ex_connection.py create mode 100644 src/pytdxdata/pool/ex_pool.py create mode 100644 src/pytdxdata/pool/health.py create mode 100644 src/pytdxdata/pool/pool.py create mode 100644 src/pytdxdata/pool/probe_cache.py create mode 100644 src/pytdxdata/protocol/__init__.py create mode 100644 src/pytdxdata/protocol/commands/__init__.py create mode 100644 src/pytdxdata/protocol/commands/base.py create mode 100644 src/pytdxdata/protocol/commands/ex_ext.py create mode 100644 src/pytdxdata/protocol/commands/ex_proto.py create mode 100644 src/pytdxdata/protocol/commands/finance.py create mode 100644 src/pytdxdata/protocol/commands/kline.py create mode 100644 src/pytdxdata/protocol/commands/mac.py create mode 100644 src/pytdxdata/protocol/commands/mac_ext.py create mode 100644 src/pytdxdata/protocol/commands/mac_tick.py create mode 100644 src/pytdxdata/protocol/commands/minute.py create mode 100644 src/pytdxdata/protocol/commands/quotes.py create mode 100644 src/pytdxdata/protocol/commands/security_list.py create mode 100644 src/pytdxdata/protocol/commands/setup.py create mode 100644 src/pytdxdata/protocol/commands/std_ext.py create mode 100644 src/pytdxdata/protocol/commands/transaction.py create mode 100644 src/pytdxdata/protocol/commands/xdxr.py create mode 100644 src/pytdxdata/protocol/datetime_.py create mode 100644 src/pytdxdata/protocol/frame.py create mode 100644 src/pytdxdata/protocol/mac_frame.py create mode 100644 src/pytdxdata/protocol/price.py create mode 100644 src/pytdxdata/protocol/volume.py create mode 100644 src/pytdxdata/router.py create mode 100644 src/pytdxdata/scheduler/__init__.py create mode 100644 src/pytdxdata/scheduler/paginator.py create mode 100644 src/pytdxdata/server_probe.py create mode 100644 src/pytdxdata/servers.json create mode 100644 tests/test_api.py create mode 100644 tests/test_cache.py create mode 100644 tests/test_cli_files.py create mode 100644 tests/test_codec_files.py create mode 100644 tests/test_pool.py create mode 100644 tests/test_protocol.py create mode 100644 uv.lock diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ede30c6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Python-generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info +.pytest_cache/ +.ruff_cache/ + +# Virtual environments +.venv + +# IDE / AI assistant data +.codebuddy/ +.workbuddy/ + +# 代码图谱索引(本地生成) +.codegraph/ + +# 本地开发脚本与第三方源码归档(不随仓库发布) +examples/archive/ +examples/pytdxwrapper_src/ +examples/uvbuild_src/ + +# 根目录未使用的服务器清单副本(生效副本为 src/pytdxdata/servers.json) +/servers.json + +# 仓库备份 +_git_backup_*.bundle diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7f0c384 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 openbot-coder + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0137964 --- /dev/null +++ b/README.md @@ -0,0 +1,359 @@ +# pytdxdata + +[![PyPI version](https://img.shields.io/pypi/v/pytdxdata)](https://pypi.org/project/pytdxdata/) +[![Python](https://img.shields.io/pypi/pyversions/pytdxdata)](https://pypi.org/project/pytdxdata/) +[![License](https://img.shields.io/pypi/l/pytdxdata)](https://github.com/openbot-coder/pytdxdata/blob/main/LICENSE) +[![PyPI downloads](https://img.shields.io/pypi/dm/pytdxdata)](https://pypi.org/project/pytdxdata/) + +通达信(TongDaXin)行情数据客户端:**动态连接池 + 双层TTL缓存 + 3连接并发分页**(纯 asyncio,零运行时依赖)。 + +**A股(沪/深/北) + 港股 + 美股 + 期货 + 期权**,一套接口全市场。 + +参照 [easy-tdx](https://github.com/handsomejustin/easy_tdx) 重新实现,解决其三大短板(无连接池、串行分页、单连接单锁),协议层独立实现(不依赖 easy-tdx 包,仅参照其字节级格式)。 + +## 特性 + +| 特性 | 说明 | +|------|------| +| 🧩 动态连接池 | 启动时对服务器**真实K线探测**选优(剔除握手正常但数据返回空的坏节点),按等待队列/空闲时长动态扩缩容,健康分(失败×0.5/成功+0.2/冷却120s)+EWMA延迟选路,跨服务器故障转移 | +| 🗄️ 双层TTL缓存 | 内存LRU + 磁盘SQLite(WAL),TTL分级:报价5s / 分钟K线60s / 日线当天 / 复权K线1天 / **历史逐笔永久**;single-flight 并发同键去重(只回源一次);缓存"已解析记录"(重复请求0网络0解析) | +| ⚡ 并发分页 | 单请求拆N页(每页显式start偏移→无状态可并行),多连接并行拉取、按序拼接、失败换连接重试;1分钟K线走 MAC 单连接串行分页(避免多服务器数据空洞) | +| 🔀 双通道整合 | 标准通道(经典行情)+ MAC通道(逐笔/复权/1分钟K线),按命令自动选路 | +| 📦 零重型依赖 | 纯 asyncio + stdlib(socket/zlib/struct/sqlite3),返回 `list[dataclass]`,pandas 由调用方按需转换 | +| 🚀 批量下载 | `get_kline_batch` / `get_minute_batch` — 多股票自动并发,连接池分配不同服务器 | +| 📄 服务器文件解析 | 板块 `block_*.dat` / 行业 `tdxhy.cfg` / 历史财报 `gpcw*.zip` **下载即解析**成 dataclass(纯 stdlib),无需自己拆二进制 | + +## 安装 + +```bash +pip install pytdxdata # 或 uv add pytdxdata +``` + +从源码开发: + +```bash +git clone +cd pytdxdata +uv sync # 创建虚拟环境 + 安装CLI +uv run python -c "import pytdxdata" +``` + +可选加速(uvloop): + +```bash +uv add --optional uvloop uvloop +``` + +> 注:数据来自通达信公开行情服务器(无需账号),服务器清单由 `server_probe.py` 导入时自动探测可用性与延时,存于 `servers.json`。 + +## CLI 命令行工具 + +```bash +pip install -e . # 或 uv sync,注册 tdx 命令 + +tdx quotes sz000001 # 行情报价(五档) / hk00700 / usAAPL +tdx kline sz000001 --period day --count 30 --adjust qfq +tdx kline cffex:IFL0 --count 30 # 期货K线(hk/us/zz/dl/shf/cffex/gz) +tdx transactions sz000001 --count 50 # 逐笔(含成交笔数) +tdx minute sz000001 # 分时 +tdx index 000001 --count 30 # 指数K线 +tdx board list / members 881001 / ranking / change --days 5 +tdx info sz000001 # 个股快照 +tdx list --market sz --count 100 # 证券列表 +tdx auction sz000001 / unusual / xdxr / finance / flow +tdx block block_gn.dat --limit 10 # 板块文件下载+解析(成分股) +tdx industry 600000 --search 银行 # 行业分类下载+解析(tdxhy.cfg) +tdx financial-list # 历史专业财报文件索引(gpcw.txt) +tdx financial gpcw20260331.zip # 历史专业财报记录下载+解析 +tdx server-info / market-stat +``` + +## 快速开始 + +```python +import asyncio +from pytdxdata import TdxData +from pytdxdata.models import KlinePeriod, Market, Adjust + +async def main(): + async with TdxData() as td: # 自动建标准池(52台)+MAC池(3台)+缓存 + # 日K线(count>800 自动3连接并发分页) + bars = await td.get_kline(1, "600000", KlinePeriod.DAY, count=2400) + # 分钟K线(复权走MAC通道) + m1 = await td.get_kline(0, "300308", KlinePeriod.MIN_1, count=240) + # 五档报价(TTL 5s 缓存,重复请求0网络) + q = await td.get_quotes([(1, "600000"), (0, "000001")]) + # 逐笔成交(MAC通道,含 trade_count=成交笔数,支持历史回溯) + t = await td.get_transactions(0, "000001", date=20260811) + total_trades = sum(r.trade_count for r in t) # 当天总成交笔数 + # 证券列表/数量 + n = await td.get_security_count(0) + lst = await td.get_security_list(1, start=0, count=1000) + # 分时/除权除息/财务 + minute = await td.get_minute(1, "600000") + xdxr = await td.get_xdxr(1, "600000") + fin = await td.get_finance(1, "600000") + +asyncio.run(main()) +``` + +## API 参考 + +统一入口 `TdxData`(`async with TdxData() as td:` 自动 start/close)。 + +> 完整接口清单(每个方法的参数 / 返回值 / 通道)见 [`docs/API_REFERENCE.md`](docs/API_REFERENCE.md)。 + +### 连接管理 + +| 方法 | 说明 | +|------|------| +| `start()` / `close()` | 启动(服务器探测+预建连接+缓存)/ 关闭 | +| `__aenter__` / `__aexit__` | 异步上下文管理器 | + +### 行情接口 + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_kline(market, code, period, *, start, count, adjust)` | `list[SecurityBar]` | K线(count>800自动并发分页;adjust≠NONE走MAC复权;88/000/399指数走指数命令) | +| `get_index_kline(market, code, period, *, start, count)` | `list[SecurityBar]` | 指数K线(多涨跌家数字段已处理) | +| `get_quotes(stocks)` | `list[SecurityQuote]` | 五档报价(>80只自动切批,缓存键=规范化股票集) | +| `get_transactions(market, code, *, date, start, count)` | `list[TransactionRecord]` | **逐笔成交(MAC,含 trade_count 成交笔数)**,支持历史回溯 | +| `get_minute(market, code, *, date)` | `list[MinuteBar]` | 分时(今日/历史,MAC通道) | +| `get_kline_batch(stocks, period, *, start, count, adjust, concurrency)` | `dict[str, list[SecurityBar]]` | **批量K线**(多股票自动并发,连接池分配不同服务器) | +| `get_minute_batch(stocks, *, date, concurrency)` | `dict[str, list[MinuteBar]]` | **批量分时**(多股票自动并发) | +| `get_security_count(market)` | `int` | 市场证券数量 | +| `get_security_list(market, *, start, count)` | `list[SecurityInfo]` | 证券列表(count=None=全部,先查总数分页) | +| `get_security_list_all(market)` | `list[SecurityInfo]` | 全市场列表(缓存1天) | +| `get_xdxr(market, code)` | `list[XdxrRecord]` | 除权除息历史(标准) | +| `get_finance(market, code)` | `FinanceRecord \| None` | 财务快照(标准) | + +### 板块/竞价/异动(MAC) + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_board_list(board_type, count)` | `list[BoardInfo]` | 板块列表(0=全部 1=行业 2=概念) | +| `get_board_members(board_symbol, count, sort_type)` | `list[MemberQuote]` | 板块成分股报价(自定义字段) | +| `get_board_summary(board_symbol)` | `dict` | 板块汇总(成分数/成交额/主力净流入/涨跌家数) | +| `get_board_ranking(board_type, top_n)` | `list[dict]` | 板块涨跌幅排行 | +| `get_board_change_ranking(board_type, days, top_n)` | `list[dict]` | 板块N日涨跌幅排行(板块指数K线) | +| `get_belong_board(market, code)` | `list[BelongBoard]` | 个股所属板块 | +| `get_auction(market, code)` | `list[AuctionItem]` | 集合竞价 | +| `get_unusual(market, start, count)` | `list[UnusualItem]` | 市场异动 | +| `get_capital_flow(market, code)` | `CapitalFlow` | 资金流向 | + +### 快照/服务器/扩展(MAC) + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_symbol_info(market, code)` | `SymbolSnapshot` | 个股特征快照 | +| `get_stock_quotes(stocks, bits)` | `list[MemberQuote]` | MAC自定义字段报价 | +| `get_stock_quotes_list(category, count)` | `list[MemberQuote]` | 市场分类报价列表(0沪A/6全A/8科创/14创业) | +| `get_stock_kline_with_indicators(market, code, indicators, ...)` | `dict` | K线+指标(MACD/KDJ/RSI/BOLL/MA/EMA) | +| `get_goods_list(market, start, count)` | `list[GoodsItem]` | 扩展市场商品(期货/期权) | +| `get_server_info()` | `ServerInfo` | 服务器交易时段 | +| `get_kline_offset(offset, count)` | `tuple[int,int]` | K线偏移 | +| `get_chart_sampling(market, code)` | `list[float]` | 分时采样(扩展市场命令,A股不适用) | + +### 文件/信息(标准+MAC) + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_company_info_category/content(...)` | `list` / `str` | F10公司信息(标准) | +| `get_block_info(filename)` | `bytes` | 板块文件下载(标准) | +| `get_report_file(filename)` | `bytes` | 报告/财务文件下载(标准) | +| `get_financial_file_list()` | `list[str]` | 财务文件列表(tdxfin/gpcw.txt,原始行) | +| `get_financial_records(filename)` | `bytes` | 财务文件内容(gpcw*.zip,原始字节) | +| **`get_block_parsed(filename)`** | `list[TdxBlock]` | 板块文件**下载+解析**:板块名/类别/成分股代码 | +| **`get_industry_map()`** | `dict[str, IndustryInfo]` | tdxhy.cfg **下载+解析**:通达信行业+申万行业 | +| **`get_financial_file_infos()`** | `list[FinancialFileInfo]` | gpcw.txt **下载+解析**:文件名/MD5/大小 | +| **`get_financial_records_parsed(filename)`** | `list[FinancialRecord]` | gpcw*.zip **解压+解析**:每股原始字段 | +| `get_file_meta(filename)` / `download_file(filename)` | `FileMeta` / `bytes` | 远程文件(MAC) | +| `get_price_limits(market, code, name, pre_close, listed_days)` | `tuple` | 涨跌停价(本地计算) | +| `get_market_stat()` | `MarketStat` | 全市场统计(涨跌家数/总额/市值/涨跌停数) | + +### 数据模型(dataclass) + +| 模型 | 关键字段 | +|------|---------| +| `SecurityBar` | market/code/open/high/low/close/vol/amount/year/month/day/hour/minute + `datetime`属性 | +| `SecurityQuote` | price/pre_close/OHLC/vol/amount/s_vol/b_vol/`bid[5]`/`ask[5]`/server_time/trading_status | +| `TransactionRecord` | time/price/vol/`trade_count`/bs_flag | +| `SecurityInfo` | code/name(GBK)/volunit/decimal_point/pre_close | +| `XdxrRecord` | category/分红送转/缩股/行权等(除权除息) | +| `MinuteBar` | price/vol | +| `FinanceRecord` | 股本/资产/营收/利润(万元/万股,最新快照) | +| `TdxBlock` | name/category/count/`codes`(板块成分,block_*.dat) | +| `IndustryInfo` | code/tdx_industry/sw_industry(tdxhy.cfg) | +| `FinancialFileInfo` | filename/hash/filesize(gpcw.txt) | +| `FinancialRecord` | code/market/report_date/`fields`(历史专业财报) | + +### 枚举 + +- `Market`:SZ=0 / SH=1 / **BJ=2**(北交所) +- `KlinePeriod`:MIN_5=0 / MIN_15=1 / MIN_30=2 / MIN_60=3 / **DAY=4** / WEEK=5 / MONTH=6 / MIN_1=7 / MIN_3=8 / YEAR=9 / SEASON=10(**与通达信协议字节一致**) +- `Adjust`:NONE / QFQ(前复权)/ HFQ(后复权) + +## 通道路由 + +| 命令 | 默认通道 | 说明 | +|------|---------|------| +| K线(复权) | **MAC** | 支持QFQ/HFQ | +| K线(MIN_1不复权) | **MAC** | 0x122E,串行分页,避免多服务器数据空洞 | +| K线(其他周期不复权)/指数 | 标准 | 价格小数位规则已验证 | +| 报价 | 标准 | 股票2位/指数ETF转债3位 | +| **逐笔成交** | **MAC** | 含trade_count,支持历史日期 | +| **分时** | **MAC** | 0x122D,完整历史分时 | +| 列表/数量/除权/财务 | 标准 | 仅标准通道提供 | + +## 架构 + +``` +src/pytdxdata/ +├── api.py # 统一接口 TdxData(对外唯一入口) +├── router.py # 标准/MAC 通道选路 +├── config.py # 服务器清单加载(servers.json + 环境变量覆盖) +├── protocol/ # 独立协议层(帧/变长整数/差分还原/自定义浮点/命令) +│ └── commands/ # kline/quotes/list/transaction/minute/xdxr/finance/mac +├── pool/ # 动态连接池(connection/health/pool) +├── cache/ # 双层TTL缓存(key/ttl/memory/disk/manager) +├── scheduler/ # 并发分页调度器(Paginator,3连接并行) +├── models/ # dataclass 模型 + 枚举 +└── codec/ # 缓存载荷序列化(pickle+zlib)+ 服务器文件解析(板块/行业/财报) +``` + +## 扩展市场(EX通道:港股/美股/期货/期权) + +`get_kline` 的 market 参数接受 `ExMarket` 枚举(港股/美股/期货/期权等52个市场): + +```python +from pytdxdata.models import ExMarket, KlinePeriod + +# 港股(HK_MAIN_BOARD=31,代码5位) +bars = await td.get_kline(ExMarket.HK_MAIN_BOARD, '00700', KlinePeriod.DAY, count=700) +# 美股(US_STOCK=74) +bars = await td.get_kline(ExMarket.US_STOCK, 'AAPL', KlinePeriod.DAY, count=700) +# 期货(CFFEX=47 / 郑28 / 大29 / 上30,合约如 IFL0/rb2610) +bars = await td.get_kline(ExMarket.CFFEX_FUTURES, 'IFL0', KlinePeriod.DAY, count=700) +# 股票期权(沪8/深9,合约代码8位) +bars = await td.get_kline(ExMarket.SH_STOCK_OPTION, '10010971', KlinePeriod.DAY, count=700) +``` + +- 端口 7727,统一走 **pytdx EX 协议**(0x23FF K线 / 0x23FA 报价 / 0x23F4 市场列表) +- 4台EX服务器:MAC EX 2台(116.205.135.205 / 121.37.232.167,0x2454 login)+ pytdx EX 2台(通达信扩展市场 116.205.143.214 / 国泰君安 103.221.142.82,0x2454 setup) +- EX 辅助接口:get_goods_list / get_goods_count / get_goods_quotes / get_goods_tick_chart / get_goods_transaction 等 +- 实测:腾讯00700=466.6 / AAPL=332.8 / IFL0=4683.2 / 沪期权10010971=0.192 +- 美股/港股期权:通达信协议无数据(美股期权可用 CBOE 公开接口,见 quantdata/scripts/import/cboe_options.py) + +## 服务器文件:板块 / 行业 / 财报 + +通达信服务器上除了行情,还有一批**静态文件**(板块成分、行业分类、历史专业财报)。 +pytdxdata 提供「**下载 + 解析**」一步到位:既保留原始 `bytes` 通道,也提供解析成 dataclass 的方法。 + +```python +async with TdxData() as td: + # ① 板块文件(block_zs.dat 行业 / block_gn.dat 概念 / block_fg.dat 风格) + blocks = await td.get_block_parsed("block_gn.dat") + for b in blocks: + print(b.name, b.category, b.count, b.codes[:5]) + + # ② 行业分类(tdxhy.cfg → 通达信行业 + 申万行业) + hy = await td.get_industry_map() # {6位代码: IndustryInfo} + print(hy["600000"].tdx_industry, hy["600000"].sw_industry) + + # ③ 历史专业财报(gpcw.txt 索引 + gpcw*.zip 解压解析) + infos = await td.get_financial_file_infos() # 文件名 / MD5 / 大小 + records = await td.get_financial_records_parsed(infos[-1].filename) + print(records[0].code, records[0].report_date, records[0].fields[:4]) +``` + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_block_parsed(filename)` | `list[TdxBlock]` | 板块成分:板块名 / 类别 / 成分股代码 | +| `get_industry_map()` | `dict[str, IndustryInfo]` | 通达信行业 + 申万行业(GBK 解析) | +| `get_financial_file_infos()` | `list[FinancialFileInfo]` | gpcw 文件索引(文件名 / MD5 / 大小) | +| `get_financial_records_parsed(filename)` | `list[FinancialRecord]` | 解压 zip + 解析每股原始字段 | + +不解析的原始字节通道仍保留(向后兼容):`get_block_info()` / `get_report_file()` / +`get_financial_file_list()` / `get_financial_records()`。 + +> 解析器集中在 `codec/`(`block.py` / `industry.py` / `financial.py`),纯标准库(struct / zipfile), +> 可脱网单测;也可直接 `from pytdxdata.codec import parse_block_dat` 单独复用。 + +## 与 easy-tdx 对比 + +| 维度 | easy-tdx | pytdxdata | +|------|----------|----------| +| 并发分页 | ❌ 串行 while 循环 | ✅ 3连接并行 | +| 连接池 | ❌ 单连接单锁 | ✅ 动态池+故障转移 | +| 缓存 | 仅证券列表(1天) | ✅ 双层TTL+single-flight | +| 数据正确性 | 基准 | ✅ K线6字段/报价7字段逐字段一致(实测) | +| 2400根日K耗时 | 0.31s | **0.16s(1.97x)** | +| 缓存命中重取 | — | 0.016s(0网络) | +| 接口覆盖 | 基准 | **36/36 全覆盖** | +| 依赖 | pandas等 | 零运行时依赖 | + +> 实测环境:Windows 11 / Python 3.13 / 同一批通达信服务器(2026-08 实测)。 + +## 已知限制 + +- 标准通道K线 `count<100` 部分服务器返回空 → 分页器已强制 `min_page=100` 规避 +- 标准当日逐笔的笔数字段协议中有但 easy-tdx 模型丢弃(pytdxdata 已显式解析);历史逐笔无笔数字段 +- 报价价格小数位按品种推断(股票2位/指数ETF转债3位),同代码不同市场含义不同(SH 000001=指数 vs SZ 000001=股票) +- 行情数据来自通达信公开服务器,**15分钟延时**;需实时数据请使用券商行情 +- 港股/美股/期货/期权数据走通达信 EX 通道(7727),部分市场(美股/港股期权)协议内无数据 +- 服务器可用性随时间变化,可用 `python -m pytdxdata.server_probe` 重新探测 + +## 免责声明 + +本包仅用于**技术研究与教育目的**。行情数据来自通达信公开接口,不保证实时性与准确性;作者不对任何投资决策负责。使用前请遵守通达信服务条款及相关法律法规。 + +## 测试 + +```bash +uv run pytest tests/ # 85个测试:协议/池/缓存/分页/API/文件解析/CLI +``` + +## 目录 + +- `docs/API_REFERENCE.md` — 完整对外接口清单(Python API / 数据模型 / 枚举 / CLI) +- `examples/quickstart.py` — 快速上手(报价 / K线 / 分时 / 板块) +- `examples/server_files.py` — 服务器文件下载 + 解析(板块 / 行业 / 财报) + +## Changelog + +### v0.4.0 (2026-09-15) + +- **新增「服务器文件下载 + 解析」能力**(对齐并超越 easy-tdx 的文件解析): + - `get_block_parsed(filename)` — 板块 `block_zs/gn/fg.dat` → `TdxBlock` 列表(板块名 / 类别 / 成分股代码) + - `get_industry_map()` — `tdxhy.cfg` → `{代码: IndustryInfo}`(通达信行业 + 申万行业) + - `get_financial_file_infos()` — `tdxfin/gpcw.txt` → `FinancialFileInfo` 列表(文件名 / MD5 / 大小) + - `get_financial_records_parsed(filename)` — `gpcw*.zip` 解压 + 解析 → `FinancialRecord` 列表(每股原始字段) +- **新增解析器** `codec/block.py`、`codec/industry.py`、`codec/financial.py`(纯标准库,脱网可单测) +- **新增模型** `TdxBlock` / `IndustryInfo` / `FinancialFileInfo` / `FinancialRecord` +- 修复财报索引市场字段解析(`<6s1c1L` → `<6sBL`,市场 0=深 / 1=沪 正确,此前恒判深市) +- 公开异步方法 47 → **51**,新增接口全部向后兼容(原 `bytes` 方法保留) +- **CLI 新增 4 个命令**:`tdx block` / `tdx industry` / `tdx financial-list` / `tdx financial` +- 新增 `tests/test_cli_files.py`(11 项 CLI 离线测试,FakeTD 桩) +- README 与 `docs/API_REFERENCE.md` 同步更新 + +### v0.3.2 (2026-08-17) + +- **新增** `get_kline_batch(stocks, period, ...)` — 批量K线接口,多股票自动并发下载,连接池分配不同服务器 +- **新增** `get_minute_batch(stocks, ...)` — 批量分时接口,多股票自动并发下载 + +### v0.3.1 (2026-08-17) + +- **`get_kline(MIN_1)` 切换为 MAC 通道** (0x122E),解决标准通道 1 分钟K线深度不足问题 +- **修复 MAC K线并发分页数据空洞** — 并发分页器将不同页面分配到不同服务器,当服务器数据深度不一致时 `stop_when_short` 会导致非确定性数据丢失;改用**单连接串行分页**(`_fetch_mac_kline_serial`,start 动态偏移,与 easy-tdx 一致) +- 路由更新:`router.py` 中 `KlinePeriod.MIN_1` 强制路由到 MAC 通道 +- 标准通道其他周期(MIN_5/15/30/60/DAY/WEEK/MONTH)不受影响 + +### v0.3.0 (2026-08-17) + +- **`get_minute` 切换为 MAC 通道** (0x122D `MacTickChartCmd`),历史分时数据稳定返回完整 240 条(09:30~14:59) +- 新增 `protocol/commands/mac_tick.py` — MAC 单日分时图命令 +- 移除不再使用的 `GetHistoryMinuteTimeDataCmd` 导入 + +### v0.2.2 及之前 + +- 初始版本:动态连接池 + 双层TTL缓存 + 3连接并发分页 + 双通道(标准/MAC)+ 扩展市场(港股/美股/期货/期权) diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md new file mode 100644 index 0000000..888933d --- /dev/null +++ b/docs/API_REFERENCE.md @@ -0,0 +1,238 @@ +# pytdxdata 对外接口清单 + +> 以 `src/pytdxdata` 源码为准整理(v0.3.2)。分四类:Python API / 数据模型 / 辅助模块 / CLI。 + +## 一、包级导出 + +```python +from pytdxdata import ( + TdxData, # 唯一对外入口类 + Market, ExMarket, KlinePeriod, Adjust, # 枚举 + SecurityBar, SecurityQuote, TransactionRecord, # 数据模型 + MinuteBar, XdxrRecord, FinanceRecord, + __version__, +) +``` + +## 二、入口类 `TdxData` + +构造参数(全部 keyword-only): + +| 参数 | 默认 | 说明 | +|------|------|------| +| `standard_servers` / `mac_servers` / `ex_servers` | None | 自定义服务器清单,None 用内置探测结果 | +| `port` | 7709 | 标准/MAC 端口(EX 固定 7727) | +| `timeout` | 3.0 | 连接超时(秒) | +| `cache_dir` | None | 磁盘缓存目录;None=仅内存缓存 | +| `default_adjust` | `Adjust.NONE` | 默认复权方式 | +| `max_inflight` | 64 | single-flight 并发上限 | +| `pool_min` / `pool_max` | 3 / 12 | 动态池上下限 | +| `mac_pool_cap` | 4 | 单台 MAC 服务器连接上限 | + +生命周期: + +| 方法 | 说明 | +|------|------| +| `async start()` | 启动(服务器探测 + 预建连接 + 缓存) | +| `async close()` | 关闭全部池与缓存 | +| `async with TdxData() as td:` | 上下文管理器,自动 start/close | + +### 2.1 K线(共 6 个) + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_kline(market, code, period, *, start=0, count=800, adjust=None, start_date=None, end_date=None)` | `list[SecurityBar]` | 主接口;count>页自动并发分页;复权走 MAC;支持日期区间(二分定位) | +| `get_kline_since(market, code, period, since, adjust=None)` | `list[SecurityBar]` | 增量便捷方法(≥since,含当天) | +| `get_kline_batch(stocks, period, *, start=0, count=800, adjust=None, concurrency=6)` | `dict[str, list[SecurityBar]]` | 多股票并发,返回 {code: bars} | +| `get_index_kline(market, code, period, *, start=0, count=800)` | `list[SecurityBar]` | 指数K线(含涨跌家数字段) | +| `get_stock_kline_with_indicators(market, code, indicators, *, period=DAY, count=30, adjust=None, params=None)` | `dict` | K线+技术指标(MACD/KDJ/RSI/BOLL/MA/EMA) | +| `get_kline_offset(offset=0, count=1)` | `tuple[int, int]` | K线偏移信息(MAC) | + +### 2.2 报价(共 4 个) + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_quotes(stocks)` | `list[SecurityQuote]` | 五档报价;>80 只自动切批;TTL 5s | +| `get_stock_quotes(stocks, bits=None)` | `list[MemberQuote]` | MAC 自定义字段报价(≤80只/次) | +| `get_stock_quotes_list(category, *, start=0, count=80, sort_type=0)` | `list[MemberQuote]` | 市场分类报价(0沪A/2深A/6全A/8科创/14创业) | +| `get_market_stat()` | `MarketStat` | 全市场统计(涨跌家数/总额/市值/涨跌停数) | + +### 2.3 逐笔成交 / 分时(共 4 个) + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_transactions(market, code, *, date=None, start=0, count=2000)` | `list[TransactionRecord]` | 逐笔成交(MAC,含 trade_count),支持历史日期 | +| `get_minute(market, code, *, date=None)` | `list[MinuteBar]` | 分时(MAC 0x122D,可历史) | +| `get_minute_batch(stocks, *, date=None, concurrency=6)` | `dict[str, list[MinuteBar]]` | 批量分时 | +| `get_chart_sampling(market, code)` | `list[float]` | 分时缩略采样(A股,MAC 0x254D) | + +### 2.4 证券列表(共 3 个) + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_security_count(market)` | `int` | 市场证券数量 | +| `get_security_list(market, *, start=0, count=None)` | `list[SecurityInfo]` | count=None=全部(先查总数防死循环) | +| `get_security_list_all(market)` | `list[SecurityInfo]` | 全市场列表(高并发一次拉完,缓存 1 天) | + +### 2.5 除权 / 财务 / F10 / 文件(共 15 个) + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_xdxr(market, code)` | `list[XdxrRecord]` | 除权除息历史(标准) | +| `get_finance(market, code)` | `FinanceRecord \| None` | 财务快照(标准) | +| `get_price_limits(market, code, name, pre_close, listed_days=9999)` | `tuple[float\|None, float\|None]` | 涨跌停价(本地计算) | +| `get_company_info_category(market, code)` | `list` | F10 目录(标准) | +| `get_company_info_content(market, code, filename, offset=0, length=65535)` | `str` | F10 内容(分块) | +| `get_block_info(filename)` | `bytes` | 板块文件**原始字节**(先查元数据再分块下载) | +| `get_report_file(filename)` | `bytes` | 报告/财务文件**原始字节**(标准 0x06B9) | +| `get_financial_file_list()` | `list[str]` | 财务文件列表**原始行**(tdxfin/gpcw.txt) | +| `get_financial_records(filename)` | `bytes` | 财务文件内容(gpcw*.zip 原始字节) | +| `get_file_meta(filename)` | `FileMeta` | 远程文件元信息(MAC 0x1215) | +| `download_file(filename, filesize=0)` | `bytes` | 下载完整远程文件(MAC 0x1217,分块) | +| **`get_block_parsed(filename)`** | `list[TdxBlock]` | 板块文件**下载+解析**为板块成分(category 按文件名推断:0行业/2概念/3风格) | +| **`get_industry_map()`** | `dict[str, IndustryInfo]` | tdxhy.cfg **下载+解析**:`{代码: 通达信行业+申万行业}` | +| **`get_financial_file_infos()`** | `list[FinancialFileInfo]` | gpcw.txt **下载+解析**:文件名/MD5/大小 | +| **`get_financial_records_parsed(filename)`** | `list[FinancialRecord]` | gpcw*.zip **解压+解析**:每股原始 float 字段(报告期取自文件名,缺失回退头部) | + +> 解析层独立于网络层:解析器位于 `pytdxdata.codec`(纯标准库 `struct`/`zipfile`),可脱网单测; +> 原 `get_*` 返回 `bytes`/`list[str]` 的方法保持**向后兼容**,解析版为**新增**方法。 + +### 2.6 板块(共 6 个) + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_board_list(board_type=0, count=10000)` | `list[BoardInfo]` | 板块列表(0全部/1行业/2概念) | +| `get_board_members(board_symbol, count=100000, sort_type=0)` | `list[MemberQuote]` | 板块成分股报价(如 881001) | +| `get_belong_board(market, code)` | `list[BelongBoard]` | 个股所属板块 | +| `get_board_summary(board_symbol)` | `dict` | 汇总:成分数/成交额/主力净流入/涨跌家数 | +| `get_board_ranking(board_type=1, top_n=20)` | `list[dict]` | 板块涨跌幅排行 | +| `get_board_change_ranking(board_type=1, days=20, top_n=20)` | `list[dict]` | 板块 N 日涨跌幅排行(板块指数K线) | + +### 2.7 MAC 扩展:竞价 / 异动 / 快照 / 资金流 / 服务器(共 5 个) + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_auction(market, code)` | `list[AuctionItem]` | 集合竞价 | +| `get_unusual(market, start=0, count=600)` | `list[UnusualItem]` | 市场异动 | +| `get_symbol_info(market, code)` | `SymbolSnapshot` | 个股特征快照 | +| `get_capital_flow(market, code)` | `CapitalFlow` | 个股资金流向 | +| `get_server_info()` | `ServerInfo` | 服务器交易时段信息 | + +### 2.8 EX 扩展市场(港股/美股/期货/期权,共 8 个) + +| 方法 | 返回 | 说明 | +|------|------|------| +| `get_goods_list(market, start=0, count=100)` | `list[dict]` | 扩展市场商品列表(0x23F5,二分定位市场) | +| `get_goods_count(market=None)` | `int` | 商品数量(None=全市场) | +| `get_goods_quotes(stocks, bits=None)` | `list[MemberQuote]` | 扩展市场报价(0x122B) | +| `get_goods_quotes_list(market, count=80)` | `list[MemberQuote]` | 商品列表+报价组合 | +| `get_goods_chart_sampling(market, code)` | `list[float]` | 扩展市场分时采样(0x254D) | +| `get_goods_tick_chart(market, code, ymd=0)` | `list[dict]` | 扩展市场分时 tick(0x122D) | +| `get_goods_transaction(market, code, *, ymd=0, start=0, count=1800)` | `list[dict]` | 扩展市场逐笔(港股 0x23FC/0x2406) | +| `get_goods_transaction_all(market, code, ymd=0)` | `list[dict]` | 港股全量逐笔(自动翻页,≤50页/9万条) | + +**合计:`TdxData` 公开异步方法 51 个**(另有 `_execute_standard/_execute_mac/_execute_ex` 等内部执行方法)。 + +## 三、数据模型(`pytdxdata.models`) + +**枚举** + +| 名称 | 取值 | +|------|------| +| `Market` | SZ=0 / SH=1 / BJ=2 | +| `KlinePeriod` | MIN_5=0 / MIN_15=1 / MIN_30=2 / MIN_60=3 / DAY=4 / WEEK=5 / MONTH=6 / MIN_1=7 / MIN_3=8 / YEAR=9 / SEASON=10 / YEAR_ALT=11(`is_minute` 属性) | +| `Adjust` | NONE=0 / QFQ=1 / HFQ=2 | +| `ExMarket` | 52 个扩展市场:HK_MAIN_BOARD=31 / US_STOCK=74 / CFFEX_FUTURES=47 / ZZ_FUTURES=28 / DL_FUTURES=29 / SH_FUTURES=30 / GZ_FUTURES=66 / SH_STOCK_OPTION=8 / SZ_STOCK_OPTION=9 等 | + +**dataclass(均为 `@dataclass(slots=True)`)** + +| 模型 | 关键字段 | +|------|---------| +| `SecurityBar` | market/code/open/high/low/close/vol/amount/year/month/day/hour/minute + `datetime` 属性 | +| `IndexBar` | 继承 `SecurityBar`(指数K线) | +| `SecurityQuote` | price/pre_close/OHLC/vol/amount/s_vol/b_vol/`bid[5]`/`ask[5]`/server_time/trading_status/cur_vol/rise_speed/limit_up/limit_down/decimal_point/open_amount | +| `SecurityInfo` | market/code/name/volunit/decimal_point/pre_close | +| `TransactionRecord` | market/code/time/price/vol/`trade_count`/bs_flag | +| `MinuteBar` | price/vol/avg_price/hour/minute | +| `XdxrRecord` | 分红送转/配股/缩股/行权 + 股本变动 + `date` 属性 | +| `FinanceRecord` | 股本/资产/负债/营收/利润等 36 个字段(万元/万股) | +| `BoardInfo` | 板块 + 领涨股信息 | +| `MemberQuote` | market/code/name/`fields: dict[str, float]` | +| `BelongBoard` | 个股所属板块 | +| `AuctionItem` | time/price/matched/unmatched | +| `UnusualItem` | 市场异动 | +| `SymbolSnapshot` | 个股特征快照 | +| `GoodsItem` | 扩展市场商品 | +| `ServerInfo` | date/last_trade_date/sessions | +| `CapitalFlow` | 主力/散户买卖 + 5档资金 | +| `TdxBlock` | name/category(0行业1地域2概念3风格)/count/`codes: list[str]`(板块成分,来自 block_*.dat) | +| `IndustryInfo` | code/tdx_industry/sw_industry(来自 tdxhy.cfg) | +| `FinancialFileInfo` | filename/hash/filesize(来自 gpcw.txt) | +| `FinancialRecord` | code/market/report_date/`fields: list[float]`(历史专业财报,注意与 `FinanceRecord` 区分) | + +> 注:`__init__` 仅导出 `Market/ExMarket/KlinePeriod/Adjust` 与 6 个核心模型;`BoardInfo/MemberQuote/AuctionItem` 等需 `from pytdxdata.models import ...`。 + +## 四、辅助模块 + +**`pytdxdata.indicator`**(技术指标,纯函数) + +| 函数 | 说明 | +|------|------| +| `ma(values, period)` | 移动平均 | +| `ema(values, period)` | 指数移动平均 | +| `macd(values, fast=12, slow=26, signal=9)` | MACD | +| `rsi(values, period=14)` | RSI | +| `kdj(high, low, close, ...)` | KDJ | +| `boll(values, period=20, mult=2.0)` | 布林带 | +| `compute_indicators(high, low, close, indicators, params=None, tail=None)` | 统一入口(被 `get_stock_kline_with_indicators` 调用) | + +**`pytdxdata.config`**(服务器清单) + +| 函数 | 说明 | +|------|------| +| `get_standard_hosts()` | 标准通道服务器列表 | +| `get_mac_hosts()` | MAC 通道服务器列表 | +| `get_ex_hosts()` | EX 扩展市场服务器列表 | +| `get_ex_handshakes()` | EX 握手方式映射 | + +**`pytdxdata.server_probe`**(服务器探测,可 `python -m pytdxdata.server_probe` 运行) +`load_servers()` / `save_servers(cfg)` / `SERVERS_FILE` + +## 五、CLI(`tdx` 命令,来自 `python -m pytdxdata.cli`) + +| 命令 | 说明 | +|------|------| +| `tdx quotes ` | 行情报价(五档) | +| `tdx kline [--period day] [--count 30] [--start 0] [--adjust none]` | K线 | +| `tdx index [--count 30]` | 指数K线 | +| `tdx transactions [--count 30] [--date YYYYMMDD]` | 逐笔成交 | +| `tdx minute [--date YYYYMMDD]` | 分时 | +| `tdx board list\|members\|ranking\|change [--type 1] [--count 30] [--top 10] [--days 20]` | 板块 | +| `tdx info ` | 个股特征快照 | +| `tdx list [--market sz] [--count 30]` | 证券列表 | +| `tdx auction ` | 集合竞价 | +| `tdx unusual [--market sh] [--count 30]` | 市场异动 | +| `tdx xdxr ` | 除权除息 | +| `tdx finance ` | 财务快照 | +| `tdx flow ` | 资金流向 | +| `tdx block [filename] [--limit 30] [--codes]` | 板块文件 下载+解析(block_*.dat → 成分股) | +| `tdx industry [codes...] [--limit 30] [--search 词]` | 行业分类 下载+解析(tdxhy.cfg) | +| `tdx financial-list [--limit 30]` | 历史专业财报文件索引(gpcw.txt) | +| `tdx financial [--code X] [--limit 5] [--fields 4]` | 历史专业财报记录 下载+解析(gpcw*.zip) | +| `tdx server-info` | 服务器交易时段 | +| `tdx market-stat` | 市场统计 | + +**标的写法**:`sz000001` / `sh600000` / `bj920992` / `hk00700` / `usAAPL` / `cffex:IFL0` / `000001`(6位数字=指数) + +## 六、通道路由(内部自动) + +| 命令 | 通道 | +|------|------| +| K线(复权 QFQ/HFQ) | MAC | +| K线(MIN_1 不复权) | MAC(串行分页) | +| K线(其他周期)/ 指数 | 标准 | +| 报价 | 标准 | +| 逐笔成交 / 分时 | MAC | +| 列表 / 数量 / 除权 / 财务 | 标准 | +| 板块 / 竞价 / 异动 / 快照 / 资金流 / 服务器 / 分类报价 | MAC | +| 港股 / 美股 / 期货 / 期权 | EX(端口 7727) | diff --git a/examples/quickstart.py b/examples/quickstart.py new file mode 100644 index 0000000..556a344 --- /dev/null +++ b/examples/quickstart.py @@ -0,0 +1,48 @@ +"""pytdxdata 快速上手:报价 / K线 / 分时 / 板块汇总。 + +运行: + python examples/quickstart.py + +说明:数据来自通达信公开行情服务器(无需账号,15 分钟延时)。 +""" + +import asyncio + +from pytdxdata import KlinePeriod, Market, TdxData + + +async def main() -> None: + # TdxData 为异步上下文管理器:进入时建连,退出时释放 + async with TdxData() as td: + # 1) 批量五档报价(可一次传多只,超过 80 只自动切批) + quotes = await td.get_quotes([(Market.SH, "600000"), (Market.SZ, "000001")]) + print("== 报价 ==") + for q in quotes: + print( + f"{q.code} 现价={q.price} 昨收={q.pre_close} " + f"买一={q.bid[0]} 卖一={q.ask[0]} 成交量={q.vol}" + ) + + # 2) 日K线(最近 10 根) + bars = await td.get_kline(Market.SH, "600000", KlinePeriod.DAY, count=10) + print("\n== 日K(最近 3 根)==") + for b in bars[-3:]: + print( + f"{b.datetime:%Y-%m-%d} " + f"开={b.open} 高={b.high} 低={b.low} 收={b.close} 量={b.vol}" + ) + + # 3) 当日分时 + minutes = await td.get_minute(Market.SH, "600000") + latest = minutes[-1].price if minutes else "-" + print(f"\n== 分时 ==\n共 {len(minutes)} 个点,最新价={latest}") + + # 4) 板块汇总(示例:通达信行业板块 881001) + summary = await td.get_board_summary("881001") + print("\n== 板块汇总 881001 ==") + for key, value in summary.items(): + print(f" {key}: {value}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/server_files.py b/examples/server_files.py new file mode 100644 index 0000000..6cf684f --- /dev/null +++ b/examples/server_files.py @@ -0,0 +1,50 @@ +"""服务器文件下载 + 解析:板块 / 行业分类 / 历史专业财报。 + +运行: + python examples/server_files.py + +这组接口把通达信服务器上的二进制文件(block_*.dat / tdxhy.cfg / gpcw*.zip) +直接下载并解析为结构化数据(解析器位于 pytdxdata.codec,纯标准库、可脱网单测)。 +""" + +import asyncio + +from pytdxdata import TdxData + + +async def main() -> None: + async with TdxData() as td: + # 1) 板块文件 -> 板块成分 + # block_zs.dat=行业 / block_gn.dat=概念 / block_fg.dat=风格 + blocks = await td.get_block_parsed("block_gn.dat") + print(f"== 概念板块(共 {len(blocks)} 个)==") + for blk in blocks[:5]: + print(f" [{blk.category}] {blk.name} 成分数={blk.count} 示例={blk.codes[:5]}") + + # 2) 行业分类(tdxhy.cfg)-> {6位代码: 通达信行业 + 申万行业} + industry = await td.get_industry_map() + print(f"\n== 行业分类(共 {len(industry)} 只)==") + for code in ("600000", "000001", "300750"): + info = industry.get(code) + if info is not None: + sw = info.sw_industry or "-" + print(f" {code} 通达信={info.tdx_industry} 申万={sw}") + + # 3) 历史专业财报文件索引(tdxfin/gpcw.txt) + infos = await td.get_financial_file_infos() + print(f"\n== 财报文件索引(共 {len(infos)} 个)==") + for fi in infos[-3:]: + print(f" {fi.filename} md5={fi.hash[:12]}... {fi.filesize} 字节") + + # 4) 下载并解析最新一期财报(gpcw*.zip -> 每股原始 float 字段) + if infos: + latest = infos[-1].filename + records = await td.get_financial_records_parsed(latest) + print(f"\n== {latest} 财报记录(共 {len(records)} 条)==") + for r in records[:5]: + market = "沪" if r.market == 1 else "深" + print(f" {r.code} 市场={market} 报告期={r.report_date} 字段数={len(r.fields)}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9228e11 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,56 @@ +[project] +name = "pytdxdata" +version = "0.4.0" +description = "通达信(TongDaXin)行情数据客户端:动态连接池 + 双层TTL缓存 + 3连接并发分页,纯asyncio零运行时依赖,支持A股/港股/美股/期货/期权" +readme = "README.md" +requires-python = ">=3.11" +license = "MIT" +license-files = ["LICENSE"] +keywords = ["tdx", "trading", "stock", "kline", "quote", "通达信", "A股", "港股", "美股", "期货", "asyncio", "quant"] +authors = [ + { name = "openbot-coder", email = "openbot-coder@users.noreply.github.com" } +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Financial and Insurance Industry", + "Topic :: Office/Business :: Financial :: Investment", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Operating System :: OS Independent", + "Framework :: AsyncIO", + "Typing :: Typed", +] +dependencies = [] + +[project.urls] +Homepage = "https://github.com/openbot-coder/pytdxdata" +Repository = "https://github.com/openbot-coder/pytdxdata" +Documentation = "https://github.com/openbot-coder/pytdxdata#readme" + +[project.optional-dependencies] +uvloop = ["uvloop>=0.21"] +dev = ["pytest>=8.0", "pytest-asyncio>=0.24", "ruff>=0.6"] + +[project.scripts] +tdx = "pytdxdata.cli:main" + +[build-system] +requires = ["uv_build>=0.11.1,<0.12.0"] +build-backend = "uv_build" + +[tool.ruff] +line-length = 100 +target-version = "py313" + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[dependency-groups] +dev = [ + "pytest>=9.1.1", + "pytest-asyncio>=1.4.0", +] + diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pytdxdata/__init__.py b/src/pytdxdata/__init__.py new file mode 100644 index 0000000..85a5ea6 --- /dev/null +++ b/src/pytdxdata/__init__.py @@ -0,0 +1,33 @@ +"""pytdxdata: 通达信行情数据客户端(动态连接池 + 双层TTL缓存 + 3连接并发分页,纯asyncio) + +支持 A股(沪/深/北) + 港股 + 美股 + 期货 + 期权,一套接口全市场。 +""" +from .api import TdxData +from .models import ( + Adjust, + ExMarket, + FinanceRecord, + KlinePeriod, + Market, + MinuteBar, + SecurityBar, + SecurityQuote, + TransactionRecord, + XdxrRecord, +) + +__version__ = "0.4.0" +__all__ = [ + "TdxData", + "Market", + "ExMarket", + "KlinePeriod", + "Adjust", + "SecurityBar", + "SecurityQuote", + "TransactionRecord", + "MinuteBar", + "XdxrRecord", + "FinanceRecord", + "__version__", +] diff --git a/src/pytdxdata/_binary.py b/src/pytdxdata/_binary.py new file mode 100644 index 0000000..35d4438 --- /dev/null +++ b/src/pytdxdata/_binary.py @@ -0,0 +1,20 @@ +"""二进制解包工具""" +from __future__ import annotations + +import struct +from typing import Any + + +def unpack_from(fmt: str, data: bytes | bytearray | memoryview, pos: int, what: str = "") -> tuple[Any, ...]: + """从 pos 处按 fmt 解包,越界抛清晰错误。""" + try: + return struct.unpack_from(fmt, data, pos) + except struct.error as e: + raise ValueError(f"{what} 解包失败 @{pos} fmt={fmt}: {e}") from e + + +def slice_bytes(data: bytes | bytearray | memoryview, pos: int, length: int, what: str = "") -> bytes: + end = pos + length + if end > len(data): + raise ValueError(f"{what} 截断 @{pos} need {length} have {len(data) - pos}") + return bytes(data[pos:end]) diff --git a/src/pytdxdata/api.py b/src/pytdxdata/api.py new file mode 100644 index 0000000..cdfb040 --- /dev/null +++ b/src/pytdxdata/api.py @@ -0,0 +1,1291 @@ +"""统一对外接口 TdxData:标准/MAC自动选路 + 自动缓存 + 自动并发分页""" +from __future__ import annotations + +import asyncio +import logging +from datetime import date as _date, datetime +from pathlib import Path +from typing import Sequence + +from .cache.disk import DiskCache +from .cache.key import CacheKey +from .cache.manager import CacheManager +from .cache.memory import MemoryCache +from .codec.payload import pack_payload, unpack_payload +from .config import get_ex_hosts, get_mac_hosts, get_standard_hosts +from .models import ( + Adjust, + ExMarket, + FinanceRecord, + FinancialFileInfo, + FinancialRecord, + IndustryInfo, + KlinePeriod, + Market, + MinuteBar, + SecurityBar, + SecurityInfo, + SecurityQuote, + TdxBlock, + TransactionRecord, + XdxrRecord, +) +from .pool.ex_pool import ExPool +from .pool.health import HealthEngine +from .pool.pool import DynamicPool +from .protocol.commands import ( + GetFinanceInfoCmd, + GetIndexBarsCmd, + GetSecurityBarsCmd, + GetSecurityCountCmd, + GetSecurityListCmd, + GetSecurityQuotesCmd, + GetTransactionDataCmd, + GetXdxrInfoCmd, + KLINE_PAGE, + LIST_PAGE, + MacKlineCmd, + MacTransactionCmd, + MAX_QUOTES, +) +from .router import route_for +from .scheduler.paginator import PageSpec, Paginator + +log = logging.getLogger(__name__) + +# 按K线周期预估最大条数上界(用于日期二分查找的初始上界) +# 日K:约30年 × 250天;分钟K:约3年 × 250天 × 对应倍数 +_KLINE_MAX_BARS: dict[KlinePeriod, int] = { + KlinePeriod.MIN_1: 200_000, + KlinePeriod.MIN_5: 60_000, + KlinePeriod.MIN_15: 25_000, + KlinePeriod.MIN_30: 15_000, + KlinePeriod.MIN_60: 10_000, + KlinePeriod.DAY: 10_000, + KlinePeriod.WEEK: 2_000, + KlinePeriod.MONTH: 500, +} + + +class TdxData: + """通达信行情数据统一入口。 + + 用法:: + + async with TdxData() as td: + bars = await td.get_kline(1, "600000", KlinePeriod.DAY, count=2400) + q = await td.get_quotes([(1, "600000"), (0, "000001")]) + t = await td.get_transactions(0, "000001", date=20260811) + """ + + def __init__( + self, + *, + standard_servers: Sequence[str] | None = None, + mac_servers: Sequence[str] | None = None, + ex_servers: Sequence[str] | None = None, + port: int = 7709, + timeout: float = 3.0, + cache_dir: str | Path | None = None, + default_adjust: Adjust = Adjust.NONE, + max_inflight: int = 64, + pool_min: int = 3, + pool_max: int = 12, + mac_pool_cap: int = 4, + ) -> None: + self.port = port + self.timeout = timeout + self.default_adjust = default_adjust + self.mac_pool_cap = mac_pool_cap + self._std_servers = list(standard_servers or get_standard_hosts()) + self._mac_servers = list(mac_servers or get_mac_hosts()) + self._ex_servers = list(ex_servers or get_ex_hosts()) + self._cache_dir = Path(cache_dir) if cache_dir else None + + # 双池 + 缓存 + self._std_pool = DynamicPool( + self._std_servers, port=port, timeout=timeout, + min_size=pool_min, max_size=pool_max, + health=HealthEngine(), + ) + from .protocol.commands.mac import MacKlineCmd + # 单连接约束下:MAC 并发上限 = MAC 服务器数(每服务器1连接), + # 故池上限取 pool_max 与服务器数的较大者,确保用满所有可用 MAC 节点。 + mac_cap = max(pool_max, len(self._mac_servers)) + self._mac_pool = DynamicPool( + self._mac_servers, port=port, timeout=timeout, + min_size=min(pool_min, len(self._mac_servers) or 1), + max_size=mac_cap, + per_server_cap=mac_pool_cap, + health=HealthEngine(), + probe_factory=lambda: MacKlineCmd(1, "600000", KlinePeriod.DAY, Adjust.NONE, 0, 100), + ) + # EX扩展市场池(港股/期货/美股,端口7727;MAC EX login + pytdx EX setup 混合) + from .config import EX_HANDSHAKES + self._ex_pool = ExPool(self._ex_servers, timeout=timeout, + min_size=min(pool_min, 2), max_size=min(pool_max, 6), + handshake_map=EX_HANDSHAKES) + disk = None + if self._cache_dir is not None: + disk = DiskCache(self._cache_dir / "tdx_cache.db") + self._cache = CacheManager( + MemoryCache(), disk=disk, max_inflight=max_inflight, + ) + self._paginator_std = Paginator(self._std_pool) + self._paginator_mac = Paginator(self._mac_pool) + self._started = False + + async def start(self) -> None: + await self._std_pool.start() + await self._mac_pool.start() + await self._ex_pool.start() + await self._cache.start() + self._started = True + + async def close(self) -> None: + await self._std_pool.close() + await self._mac_pool.close() + await self._ex_pool.close() + await self._cache.close() + self._started = False + + async def __aenter__(self) -> "TdxData": + await self.start() + return self + + async def __aexit__(self, *exc) -> None: + await self.close() + + # ------------------------------------------------------------------ # + # K线 + # ------------------------------------------------------------------ # + + async def get_kline( + self, market: Market, code: str, period: KlinePeriod, *, + start: int = 0, count: int = 800, adjust: Adjust | None = None, + start_date: datetime | None = None, end_date: datetime | None = None, + ) -> list[SecurityBar]: + """获取K线(count>页大小自动并发分页;adjust!=NONE 走MAC复权通道)。 + + 参数: + market: 市场 (Market.SH / Market.SZ) + code: 股票代码 + period: K线周期 + start: 起始偏移量(从最早K线开始的索引,默认0) + count: 获取条数(默认800) + adjust: 复权方式(默认使用实例 default_adjust) + start_date: 起始日期(与 start/count 互斥,优先使用日期过滤) + end_date: 截止日期(可选,配合 start_date 使用) + + 增量更新示例: + # 获取 2026-01-01 之后的所有日K线 + bars = await client.get_kline(Market.SH, "600000", KlinePeriod.DAY, + start_date=datetime(2026, 1, 1)) + + # 获取 2026-01-01 至 2026-06-30 之间的日K线 + bars = await client.get_kline(Market.SH, "600000", KlinePeriod.DAY, + start_date=datetime(2026, 1, 1), + end_date=datetime(2026, 6, 30)) + + # 便捷方法:获取指定日期之后的所有K线 + bars = await client.get_kline_since(Market.SH, "600000", + KlinePeriod.DAY, datetime(2026, 1, 1)) + """ + # 日期过滤模式:获取全量数据后按日期筛选 + if start_date is not None or end_date is not None: + return await self._get_kline_by_date( + market, code, period, start_date, end_date, adjust) + + adj = self.default_adjust if adjust is None else adjust + # EX扩展市场(港股/期货):market为ExMarket值,走ex池 + if isinstance(market, ExMarket) or int(market) not in (0, 1, 2): + # EX通道统一用pytdx EX协议(0x23FF): category=K线周期(与TDXParams一致, DAY=4) + from .protocol.commands.ex_proto import GetExInstrumentBarsCmd + cat = int(period) + fetch_count = max(700, start + count) + key = CacheKey("kline", int(market), code, period.name, start, fetch_count, + f"{adj.name}@{cat}") + async def fetch() -> bytes: + result = await self._fetch_paginated_ex( + lambda s2, c2: GetExInstrumentBarsCmd(int(market), code, cat, s2, c2), + PageSpec(page_size=700, total=fetch_count, min_page=100)) + bars = [] + for d in result: + bars.append(SecurityBar( + market=int(market), code=code, + year=d["year"], month=d["month"], day=d["day"], + hour=d.get("hour", 0), minute=d.get("minute", 0), + open=d["open"], high=d["high"], low=d["low"], close=d["close"], + vol=float(d.get("trade", 0) or 0), amount=0.0, + )) + return pack_payload(bars) + bars = unpack_payload(await self._cache.get_or_fetch(key, fetch)) + # EX 0x23FF 按时间升序返回(老→新):start=0 取最新 count 根 + if count < len(bars): + return bars[-count:] if start == 0 else bars[start:start + count] + return bars + + route = route_for("kline", adjust=adj, period=period) + key = CacheKey("kline", int(market), code, period.name, start, count, adj.name) + + def make_cmd(s: int, c: int): + if route == "mac": + return MacKlineCmd(int(market), code, period, adj, s, c) + # 指数类(88开头板块指数 / 沪市000指数 / 深市399指数)走指数K线命令(多4字节涨跌家数) + # 注意: 深市股票000001.SZ(market=0)不是指数, 必须加market判断 + mk = int(market) + is_index = code.startswith("88") or ( + code.startswith("000") and mk == int(Market.SH)) or ( + code.startswith("399") and mk == int(Market.SZ)) + if is_index: + return GetIndexBarsCmd(mk, code, period, s, c) + return GetSecurityBarsCmd(mk, code, period, s, c) + + if route == "mac": + payload = await self._cache.get_or_fetch( + key, lambda: self._fetch_mac_kline_serial( + make_cmd, start + count, start=start, page_size=700, + ) + ) + else: + payload = await self._cache.get_or_fetch( + key, lambda: self._fetch_paginated(route, make_cmd, PageSpec( + page_size=KLINE_PAGE, total=start + count, min_page=100, + start_offset=start, + )) + ) + bars = unpack_payload(payload) + # K线按时间正序(无论 start 是否 0 都排序,避免依赖协议层顺序保证) + bars.sort(key=lambda b: (b.year, b.month, b.day, b.hour, b.minute)) + return bars + + async def _get_kline_by_date( + self, market: Market, code: str, period: KlinePeriod, + start_date: datetime | None, end_date: datetime | None, + adjust: Adjust | None, + ) -> list[SecurityBar]: + """按日期范围获取K线(内部方法)。 + + 高效策略:TDX K线按时间正序返回,start 是偏移量。用二分查找在 + O(log N) 次单条探测内定位日期对应的偏移量,再只拉取目标区间。 + 优化点:按周期预估上界省掉指数探测;仅 start_date 时合并为单次 + 二分;需同时查 start/end 日期时用 asyncio.gather 并行。 + """ + # 归一化日期边界 + start_dt = (start_date if isinstance(start_date, datetime) + else datetime.combine(start_date, datetime.min.time())) \ + if start_date is not None else None + end_dt = (end_date if isinstance(end_date, datetime) + else datetime.combine(end_date, datetime.max.time())) \ + if end_date is not None else None + # end_date 传入 datetime 但时间为 00:00 时视为"当天结束" + if end_dt is not None and end_dt.time() == datetime.min.time().replace(tzinfo=None): + end_dt = end_dt.replace(hour=23, minute=59, second=59, microsecond=999999) + + # 按周期预估总条数上界(保守值,二分中越界自然修正) + est = _KLINE_MAX_BARS.get(period, 10_000) + + # 仅 start_date 的常见增量场景:单次二分定位起点,再取剩余全部 + if start_dt is not None and end_dt is None: + lo = await self._probe_kline_lower_bound( + market, code, period, adjust, est, start_dt) + # 命中上界说明实际条数可能超出预估,逐步扩大范围直到定位成功 + MAX_EXPAND = 5 # 最多扩大 2^5 = 32 倍 + for _ in range(MAX_EXPAND): + if lo < est: + break + if await self._probe_kline_bar( + market, code, period, adjust, est) is None: + break # est 处已无数据,说明 lo 就是总数上限 + est *= 2 + lo = await self._probe_kline_lower_bound( + market, code, period, adjust, est, start_dt) + if lo >= est: + return [] + return await self.get_kline( + market, code, period, start=lo, count=est - lo, adjust=adjust) + + # 需要 end_date 时,并行二分起点和终点 + tasks = [] + if start_dt is not None: + tasks.append(self._probe_kline_lower_bound( + market, code, period, adjust, est, start_dt)) + if end_dt is not None: + tasks.append(self._probe_kline_upper_bound( + market, code, period, adjust, est, end_dt)) + results = await asyncio.gather(*tasks) if tasks else [] + + lo = results[0] if start_dt is not None else 0 + hi = results[-1] if end_dt is not None else est + if start_dt is not None and end_dt is not None: + lo, hi = results[0], results[1] + + # 命中上界时逐步扩大范围重试(与上面相同的回退逻辑) + MAX_EXPAND = 5 # 最多扩大 2^5 = 32 倍 + for _ in range(MAX_EXPAND): + need_expand = False + if start_dt is not None and lo >= est: + if await self._probe_kline_bar( + market, code, period, adjust, est) is not None: + need_expand = True + if end_dt is not None and hi >= est: + if await self._probe_kline_bar( + market, code, period, adjust, est) is not None: + need_expand = True + if not need_expand: + break + est *= 2 + tasks2 = [] + if start_dt is not None: + tasks2.append(self._probe_kline_lower_bound( + market, code, period, adjust, est, start_dt)) + if end_dt is not None: + tasks2.append(self._probe_kline_upper_bound( + market, code, period, adjust, est, end_dt)) + results2 = await asyncio.gather(*tasks2) + lo = results2[0] if start_dt is not None else 0 + hi = results2[-1] if end_dt is not None else est + if start_dt is not None and end_dt is not None: + lo, hi = results2[0], results2[1] + + if lo >= hi: + return [] + bars = await self.get_kline( + market, code, period, start=lo, count=hi - lo, adjust=adjust) + # min_page 可能导致多拉,做精确日期裁剪 + if start_dt is not None: + bars = [b for b in bars if b.datetime >= start_dt] + if end_dt is not None: + bars = [b for b in bars if b.datetime <= end_dt] + return bars + + async def _probe_kline_lower_bound( + self, market: Market, code: str, period: KlinePeriod, + adjust: Adjust | None, total: int, target: datetime, + ) -> int: + """二分查找第一个 datetime >= target 的偏移量(在 [0, total] 内)。""" + lo, hi = 0, total + while lo < hi: + mid = (lo + hi) // 2 + bar = await self._probe_kline_bar(market, code, period, adjust, mid) + if bar is None: + hi = mid # 越界,数据边界在左侧 + elif bar.datetime < target: + lo = mid + 1 # 太早,向右 + else: + hi = mid # 命中,向左收缩 + return lo + + async def _probe_kline_upper_bound( + self, market: Market, code: str, period: KlinePeriod, + adjust: Adjust | None, total: int, target: datetime, + ) -> int: + """二分查找第一个 datetime > target 的偏移量(在 [0, total] 内)。""" + lo, hi = 0, total + while lo < hi: + mid = (lo + hi) // 2 + bar = await self._probe_kline_bar(market, code, period, adjust, mid) + if bar is None: + hi = mid # 越界,数据边界在左侧 + elif bar.datetime <= target: + lo = mid + 1 # 还在范围内,向右 + else: + hi = mid # 超出,向左收缩 + return lo + + async def _probe_kline_bar( + self, market: Market, code: str, period: KlinePeriod, + adjust: Adjust | None, offset: int, + ) -> SecurityBar | None: + """探测指定偏移量处的单条K线(越界返回 None)。""" + adj = self.default_adjust if adjust is None else adjust + route = route_for("kline", adjust=adj, period=period) + if route == "mac": + cmd = MacKlineCmd(int(market), code, period, adj, offset, 1) + try: + bars = await self._execute_mac(cmd) + except (ConnectionError, OSError, asyncio.TimeoutError): + return None + else: + mk = int(market) + is_index = code.startswith("88") or ( + code.startswith("000") and mk == int(Market.SH)) or ( + code.startswith("399") and mk == int(Market.SZ)) + cmd = (GetIndexBarsCmd(mk, code, period, offset, 1) + if is_index else GetSecurityBarsCmd(mk, code, period, offset, 1)) + try: + bars = await self._execute_standard(cmd) + except (ConnectionError, OSError, asyncio.TimeoutError): + return None + return bars[0] if bars else None + + async def get_kline_since( + self, market: Market, code: str, period: KlinePeriod, + since: datetime, adjust: Adjust | None = None, + ) -> list[SecurityBar]: + """获取指定日期之后的所有K线(增量更新便捷方法)。 + + 参数: + since: 起始日期(含),只返回 datetime >= since 的K线 + + 增量更新示例: + # 获取2026年1月1日之后的所有日K线 + bars = await client.get_kline_since( + Market.SH, "600000", KlinePeriod.DAY, datetime(2026, 1, 1)) + + # 增量更新:本地已有数据截止到 last_date,拉取新数据 + last_date = datetime(2026, 8, 19) + new_bars = await client.get_kline_since( + Market.SH, "600000", KlinePeriod.DAY, last_date) + # 注意:结果包含 last_date 当天的K线,追加时需去重 + """ + return await self._get_kline_by_date( + market, code, period, since, None, adjust) + + async def get_kline_batch( + self, stocks: list[tuple[Market, str]], period: KlinePeriod, *, + start: int = 0, count: int = 800, adjust: Adjust | None = None, + concurrency: int = 6, + ) -> dict[str, list[SecurityBar]]: + """批量获取K线,自动并发不同服务器。返回 {code: bars}。""" + sem = asyncio.Semaphore(concurrency) + + async def _fetch(mkt: Market, code: str): + async with sem: + return code, await self.get_kline(mkt, code, period, start=start, count=count, adjust=adjust) + + results = await asyncio.gather(*[_fetch(m, c) for m, c in stocks]) + return {code: bars for code, bars in results} + + async def get_index_kline( + self, market: Market, code: str, period: KlinePeriod, *, + start: int = 0, count: int = 800, + ) -> list[SecurityBar]: + key = CacheKey("index_kline", int(market), code, period.name, start, count, "") + + def make_cmd(s: int, c: int): + return GetIndexBarsCmd(int(market), code, period, s, c) + + payload = await self._cache.get_or_fetch( + key, lambda: self._fetch_paginated("standard", make_cmd, PageSpec( + page_size=KLINE_PAGE, total=start + count, min_page=100, + )) + ) + return unpack_payload(payload) + + # ------------------------------------------------------------------ # + # 报价 + # ------------------------------------------------------------------ # + + async def get_quotes(self, stocks: Sequence[tuple[Market, str]]) -> list[SecurityQuote]: + """批量报价(>80自动切批,缓存键=规范化股票集,TTL 5s)。""" + stocks = list(stocks) + if not stocks: + return [] + key = CacheKey.quote_key(stocks) + payload = await self._cache.get_or_fetch( + key, lambda: self._fetch_quotes(stocks) + ) + return unpack_payload(payload) + + async def _fetch_quotes(self, stocks: list[tuple[Market, str]]) -> bytes: + out: list[SecurityQuote] = [] + for i in range(0, len(stocks), MAX_QUOTES): + batch = stocks[i:i + MAX_QUOTES] + cmds = GetSecurityQuotesCmd([(int(m), c) for m, c in batch]) + result = await self._execute_standard(cmds) + out.extend(result) + return pack_payload(out) + + # ------------------------------------------------------------------ # + # 逐笔成交(MAC通道,含成交笔数) + # ------------------------------------------------------------------ # + + async def get_transactions( + self, market: Market, code: str, *, + date: int | None = None, start: int = 0, count: int = 2000, + ) -> list[TransactionRecord]: + """获取逐笔成交(MAC 0x122F,每条含 trade_count=成交笔数)。""" + key = CacheKey("transaction", int(market), code, "", start, count, + "" if date is None else str(date)) + + def make_cmd(s: int, c: int): + return MacTransactionCmd(int(market), code, date or 0, s, c) + + payload = await self._cache.get_or_fetch( + key, lambda: self._fetch_paginated("mac", make_cmd, PageSpec( + page_size=1000, total=start + count, min_page=100, + )) + ) + return unpack_payload(payload) + + # ------------------------------------------------------------------ # + # 证券列表 + # ------------------------------------------------------------------ # + + async def get_security_count(self, market: Market) -> int: + key = CacheKey("count", int(market)) + + async def fetch() -> bytes: + n = await self._execute_standard(GetSecurityCountCmd(int(market))) + return pack_payload(n) + + payload = await self._cache.get_or_fetch(key, fetch) + return unpack_payload(payload) + + async def get_security_list(self, market: Market, *, + start: int = 0, count: int | None = None) -> list[SecurityInfo]: + """分页获取证券列表(count=None=全部,先查总数避免死循环)。""" + if count is None: + count = await self.get_security_count(market) + total = count + key = CacheKey("list", int(market), "", "", start, total or 0, "") + + def make_cmd(s: int, c: int): + return GetSecurityListCmd(int(market), s) + + spec = PageSpec(page_size=LIST_PAGE, total=total, stop_when_short=True) + + async def fetch() -> bytes: + return await self._fetch_paginated("standard", make_cmd, spec) + + payload = await self._cache.get_or_fetch(key, fetch) + return unpack_payload(payload) + + # ------------------------------------------------------------------ # + # 分时 / 除权除息 / 财务 + # ------------------------------------------------------------------ # + + async def get_minute(self, market: Market, code: str, *, + date: int | None = None) -> list[MinuteBar]: + """获取分时(date=None=今天;已切换为 MAC 通道 0x122D,可获取完整历史分时)。""" + if date is None: + d = _date.today() + date = d.year * 10000 + d.month * 100 + d.day + key = CacheKey("minute", int(market), code, "", 0, 0, str(date)) + payload = await self._cache.get_or_fetch( + key, lambda: self._fetch_minute(market, code, date) + ) + return unpack_payload(payload) + + async def _fetch_minute(self, market, code: str, date: int) -> bytes: + from .protocol.commands.mac_tick import MacTickChartCmd + cmd = MacTickChartCmd(int(market), code, date) + result = await self._execute_mac(cmd) + return pack_payload(result) + + async def get_minute_batch( + self, stocks: list[tuple[Market, str]], *, + date: int | None = None, concurrency: int = 6, + ) -> dict[str, list[MinuteBar]]: + """批量获取分时,自动并发不同服务器。返回 {code: bars}。""" + sem = asyncio.Semaphore(concurrency) + + async def _fetch(mkt: Market, code: str): + async with sem: + return code, await self.get_minute(mkt, code, date=date) + + results = await asyncio.gather(*[_fetch(m, c) for m, c in stocks]) + return {code: bars for code, bars in results} + + async def get_xdxr(self, market: Market, code: str) -> list[XdxrRecord]: + key = CacheKey("xdxr", int(market), code) + + async def fetch() -> bytes: + recs = await self._execute_standard(GetXdxrInfoCmd(int(market), code)) + return pack_payload(recs) + + payload = await self._cache.get_or_fetch(key, fetch) + return unpack_payload(payload) + + async def get_finance(self, market: Market, code: str) -> FinanceRecord | None: + key = CacheKey("finance", int(market), code) + + async def fetch() -> bytes: + rec = await self._execute_standard(GetFinanceInfoCmd(int(market), code)) + return pack_payload(rec) + + payload = await self._cache.get_or_fetch(key, fetch) + return unpack_payload(payload) + + # ------------------------------------------------------------------ # + # 内部执行 + # ------------------------------------------------------------------ # + + async def _execute_standard(self, cmd) -> object: + """标准通道单命令执行(带连接池+失败换连接)。""" + pc = await self._std_pool.acquire() + try: + for attempt in range(3): + try: + return await pc.execute(cmd) + except Exception: + if attempt >= 2: + raise + await self._std_pool.release(pc, broken=True) + pc = await self._std_pool.acquire() + finally: + await self._std_pool.release(pc) + + async def _fetch_paginated(self, route: str, make_cmd, spec: PageSpec, + *, concurrency: int | None = None) -> bytes: + pag = self._paginator_mac if route == "mac" else self._paginator_std + result = await pag.fetch_all(make_cmd, spec, concurrency=concurrency) + return pack_payload(result) + + async def _fetch_paginated_list(self, route: str, make_cmd, spec: PageSpec, + *, concurrency: int | None = None) -> list: + """分页拉取,直接返回列表(避免 pack/unpack 序列化开销)。""" + pag = self._paginator_mac if route == "mac" else self._paginator_std + return await pag.fetch_all(make_cmd, spec, concurrency=concurrency) + + async def _fetch_mac_kline_serial(self, make_cmd, total: int, *, + start: int = 0, + page_size: int = 700) -> bytes: + """MAC K线串行分页(单连接),避免多服务器数据深度不一致导致空洞。""" + pc = await self._mac_pool.acquire() + try: + result: list = [] + while start < total: + cnt = min(page_size, total - start) + cmd = make_cmd(start, cnt) + for attempt in range(3): + try: + bars = await pc.execute(cmd) + break + except Exception: + if attempt >= 2: + raise + await self._mac_pool.release(pc, broken=True) + pc = await self._mac_pool.acquire() + bars = [] + if not bars: + break + result.extend(bars) + start += len(bars) + if len(bars) < cnt: + break + return pack_payload(result) + finally: + await self._mac_pool.release(pc) + + async def _execute_mac(self, cmd) -> object: + """MAC通道单命令执行(带连接池+失败换连接)。""" + pc = await self._mac_pool.acquire() + try: + for attempt in range(3): + try: + return await pc.execute(cmd) + except Exception: + if attempt >= 2: + raise + await self._mac_pool.release(pc, broken=True) + pc = await self._mac_pool.acquire() + finally: + await self._mac_pool.release(pc) + + async def _execute_ex(self, cmd) -> object: + """EX扩展市场单命令执行(连接池+失败/空结果重建)。""" + conn = await self._ex_pool.acquire() + try: + for attempt in range(3): + try: + result = await conn.execute(cmd) + # 空结果可能是login会话失效(服务器静默返回0条)→ 新连接重试 + if result in (None, []) and attempt < 2: + await self._ex_pool.release(conn, broken=True) + conn = await self._ex_pool.acquire() + continue + return result + except Exception: + if attempt >= 2: + raise + await self._ex_pool.release(conn, broken=True) + conn = await self._ex_pool.acquire() + finally: + await self._ex_pool.release(conn) + + async def _fetch_paginated_ex(self, make_cmd, spec: PageSpec) -> list: + """EX通道分页拉取(串行逐页,扩展市场页数少)。""" + result = [] + start = 0 + for page in range(spec.max_pages if spec.max_pages else 64): + if spec.total is not None and start >= spec.total: + break + cnt = spec.page_size + if spec.total is not None: + cnt = min(cnt, spec.total - start) + items = await self._execute_ex(make_cmd(start, cnt)) + if items is None: + break + result.extend(items) + start += len(items) + if not items or (spec.stop_when_short and len(items) < cnt): + break + return result + + # ------------------------------------------------------------------ # + # MAC 扩展接口:板块/竞价/异动/快照/资金流/服务器/扩展市场 + # ------------------------------------------------------------------ # + + async def get_board_list(self, board_type: int = 0, count: int = 10000) -> list: + """板块列表(board_type: 0=全部 1=行业 2=概念)。""" + from .protocol.commands.mac_ext import MacBoardListCmd + items: list = [] + start = 0 + while start < count: + batch = await self._execute_mac(MacBoardListCmd(board_type, start, 150)) + if not batch: + break + items.extend(batch) + start += len(batch) // 2 + if len(batch) < 150: + break + return items + + async def get_board_members(self, board_symbol: str, count: int = 100000, + sort_type: int = 0) -> list: + """板块成分股报价(board_symbol 如 881001)。""" + from .protocol.commands.mac_ext import MacBoardMembersQuotesCmd + board_code = self._convert_board_code(board_symbol) + items: list = [] + start = 0 + while start < count: + batch = await self._execute_mac( + MacBoardMembersQuotesCmd(board_code, sort_type=sort_type, start=start, page_size=80)) + if not batch: + break + items.extend(batch) + start += len(batch) + if len(batch) < 80: + break + return items + + async def get_belong_board(self, market: Market, code: str) -> list: + """个股所属板块。""" + from .protocol.commands.mac_ext import MacBelongBoardCmd + return await self._execute_mac(MacBelongBoardCmd(int(market), code)) + + async def get_auction(self, market: Market, code: str) -> list: + """集合竞价。""" + from .protocol.commands.mac_ext import MacAuctionCmd + return await self._execute_mac(MacAuctionCmd(int(market), code)) + + async def get_unusual(self, market: Market, start: int = 0, count: int = 600) -> list: + """市场异动。""" + from .protocol.commands.mac_ext import MacUnusualCmd + return await self._execute_mac(MacUnusualCmd(int(market), start, count)) + + async def get_symbol_info(self, market: Market, code: str): + """个股特征快照。""" + from .protocol.commands.mac_ext import MacSymbolInfoCmd + return await self._execute_mac(MacSymbolInfoCmd(int(market), code)) + + async def _ex_find_market_offset(self, market: int) -> int: + """二分定位指定市场在全局商品列表中的起始偏移。""" + from .protocol.commands.ex_ext import ( + GetExInstrumentCountCmd, GetExInstrumentInfoCmd) + total = await self._execute_ex(GetExInstrumentCountCmd()) + if total == 0: + return -1 + lo, hi = 0, total + while lo < hi: + mid = (lo + hi) // 2 + items = await self._execute_ex(GetExInstrumentInfoCmd(0, mid, 1)) + if not items: + hi = mid + continue + m = int(items[0]["market"]) + if m < market: + lo = mid + 1 + else: + hi = mid + return lo + + async def get_goods_list(self, market: int, start: int = 0, + count: int = 100) -> list[dict]: + """扩展市场商品列表(EX通道 0x23F5,二分定位市场边界+过滤)。""" + from .protocol.commands.ex_ext import ( + GetExInstrumentCountCmd, GetExInstrumentInfoCmd) + offset = await self._ex_find_market_offset(market) + if offset < 0: + return [] + items: list[dict] = [] + pos = offset + start + total = await self._execute_ex(GetExInstrumentCountCmd()) + while pos < total and len(items) < count: + page = await self._execute_ex(GetExInstrumentInfoCmd(0, pos, 1000)) + if not page: + break + for g in page: + if int(g["market"]) == market: + items.append(g) + if len(items) >= count: + break + elif int(g["market"]) > market: + return items + pos += len(page) + return items + + async def get_goods_count(self, market: int | None = None) -> int: + """扩展市场商品数量(EX通道 0x23F0)。 + market=None 返回全市场总数;指定市场=二分定位偏移+扫描计数。""" + from .protocol.commands.ex_ext import ( + GetExInstrumentCountCmd, GetExInstrumentInfoCmd) + total = await self._execute_ex(GetExInstrumentCountCmd()) + if market is None: + return int(total) + offset = await self._ex_find_market_offset(market) + if offset < 0: + return 0 + n = 0 + pos = offset + while pos < total: + page = await self._execute_ex(GetExInstrumentInfoCmd(0, pos, 1000)) + if not page: + break + for g in page: + m = int(g["market"]) + if m == market: + n += 1 + elif m > market: + return n + pos += len(page) + return n + + async def get_goods_quotes(self, stocks: list[tuple[int, str]], + bits: list[int] | None = None) -> list[MemberQuote]: + """扩展市场报价(EX通道 0x122B,stocks=[(ExMarket, code)])。""" + from .protocol.commands.mac_ext import MacSymbolQuotesCmd + return await self._execute_ex(MacSymbolQuotesCmd(stocks, bits)) + + async def get_goods_quotes_list(self, market: int, count: int = 80) -> list[MemberQuote]: + """扩展市场报价列表(goods_list + 报价组合)。""" + goods = await self.get_goods_list(market, 0, min(count, 80)) + if not goods: + return [] + stocks = [(int(g["market"] or market), g["code"]) for g in goods[:80]] + return await self.get_goods_quotes(stocks) + + async def get_goods_chart_sampling(self, market: int, code: str) -> list[float]: + """扩展市场分时采样(EX通道 0x254D)。""" + from .protocol.commands.mac_ext import MacChartSamplingCmd + return await self._execute_ex(MacChartSamplingCmd(market, code)) + + async def get_goods_tick_chart(self, market: int, code: str, + ymd: int = 0) -> list[dict]: + """扩展市场分时tick(EX通道 0x122D)。""" + from .protocol.commands.ex_ext import SymbolTickChartCmd + return await self._execute_ex(SymbolTickChartCmd(market, code, ymd)) + + async def get_goods_transaction(self, market: int, code: str, *, + ymd: int = 0, start: int = 0, + count: int = 1800) -> list[dict]: + """扩展市场逐笔(港股0x23FC/0x2406 16B/条;其他0x122F 18B/条)。""" + hk_markets = {27, 31, 48, 49, 71, 98} + if market in hk_markets: + from .protocol.commands.ex_ext import ( + GetExHistoryTransactionDataCmd, GetExTransactionDataCmd) + if ymd: + return await self._execute_ex( + GetExHistoryTransactionDataCmd(ymd, market, code, start, count)) + return await self._execute_ex( + GetExTransactionDataCmd(market, code, start, count)) + from .protocol.commands.transaction import MacTransactionCmd + items = await self._execute_ex( + MacTransactionCmd(market, code, ymd, start, count)) + return [dict(t._asdict()) if hasattr(t, "_asdict") else t for t in items] + + async def get_goods_transaction_all(self, market: int, code: str, + ymd: int = 0) -> list[dict]: + """扩展市场(仅港股)全部逐笔,自动翻页(最多50页/9万条)。""" + hk_markets = {27, 31, 48, 49, 71, 98} + if market not in hk_markets: + raise ValueError("仅港股支持全量逐笔(get_goods_transaction_all)") + out: list[dict] = [] + start = 0 + for _ in range(50): + page = await self.get_goods_transaction(market, code, ymd=ymd, + start=start, count=1800) + if not page: + break + out.extend(page) + start += len(page) + if len(page) < 1800: + break + return out + + async def get_server_info(self): + """服务器交易时段信息。""" + from .protocol.commands.mac_ext import MacServerInfoCmd + return await self._execute_mac(MacServerInfoCmd()) + + async def get_kline_offset(self, offset: int = 0, count: int = 1) -> tuple[int, int]: + """K线偏移信息。""" + from .protocol.commands.mac_ext import MacKlineOffsetCmd + return await self._execute_mac(MacKlineOffsetCmd(offset, count)) + + async def get_capital_flow(self, market: Market, code: str): + """个股资金流向。""" + from .protocol.commands.mac_ext import MacCapitalFlowCmd + return await self._execute_mac(MacCapitalFlowCmd(int(market), code)) + + @staticmethod + def _convert_board_code(board_symbol: str) -> int: + """板块代码转换(881001→20686 等,参照 easy-tdx)。""" + s = board_symbol.strip() + if s.startswith("US"): + return 30000 + int(s[2:]) + if s.startswith("HK"): + return 20000 + int(s[2:]) + if len(s) == 6: + if s.startswith("88"): + return int(s) - 880000 + 20000 + if s.startswith("399"): + return int(s) - 399000 + 30000 + if s.startswith("899"): + return int(s) - 899000 + 32000 + if s.startswith("000"): + return 31000 + int(s) + return int(s) + + # ------------------------------------------------------------------ # + # 标准通道扩展:F10/板块文件/市场统计/涨跌停价 + # ------------------------------------------------------------------ # + + async def get_company_info_category(self, market: Market, code: str) -> list: + """F10目录。""" + from .protocol.commands.std_ext import GetCompanyInfoCategoryCmd + return await self._execute_standard(GetCompanyInfoCategoryCmd(market, code)) + + async def get_company_info_content(self, market: Market, code: str, + filename: str, offset: int = 0, + length: int = 65535) -> str: + """F10内容(分块读取)。""" + from .protocol.commands.std_ext import GetCompanyInfoContentCmd + return await self._execute_standard( + GetCompanyInfoContentCmd(market, code, filename, offset, length)) + + async def get_block_info(self, filename: str) -> bytes: + """板块文件(先查元数据再下载全部)。""" + from .protocol.commands.std_ext import GetBlockInfoCmd, GetBlockInfoMetaCmd + meta = await self._execute_standard(GetBlockInfoMetaCmd(filename)) + if meta is None: + return b"" + data = bytearray() + pos = 0 + chunk = 30000 + while pos < meta.size: + part = await self._execute_standard(GetBlockInfoCmd(filename, pos, chunk)) + if not part: + break + data.extend(part) + pos += len(part) + if len(part) < chunk: + break + return bytes(data) + + async def get_price_limits(self, market: Market, code: str, name: str, + pre_close: float, listed_days: int = 9999) -> tuple[float | None, float | None]: + """涨跌停价(本地计算)。""" + from .protocol.commands.std_ext import compute_price_limits + return compute_price_limits(market, code, name, pre_close, listed_days) + + async def get_market_stat(self): + """全市场统计(用报价查上证指数880005/880001/880006组合)。""" + from .protocol.commands.std_ext import MarketStat + # 上证指数代码 + q = await self.get_quotes([(Market.SH, "880005"), (Market.SH, "880001"), + (Market.SH, "880006")]) + stat = MarketStat() + qmap = {q2.code: q2 for q2 in q} + q5 = qmap.get("880005") + if q5: + stat.up_count = int(q5.price * 10) + stat.down_count = int(q5.open * 10) + stat.flat_count = int(q5.low * 10) + stat.total_count = int(q5.high * 10) + stat.total_amount = q5.amount + stat.total_vol = q5.vol + q1 = qmap.get("880001") + if q1: + stat.total_market_cap = q1.price * 1e10 + q6 = qmap.get("880006") + if q6: + stat.limit_up_count = int(q6.price * 10) + stat.limit_down_count = int(q6.open * 10) + return stat + + # ------------------------------------------------------------------ # + # MAC 自定义字段报价 / 分类列表 + # ------------------------------------------------------------------ # + + async def get_stock_quotes(self, stocks: list[tuple[Market, str]], + bits: list[int] | None = None) -> list: + """MAC自定义字段报价(≤80只/次,bits=FieldBit列表)。""" + from .protocol.commands.mac_ext import MacSymbolQuotesCmd + out: list = [] + for i in range(0, len(stocks), 80): + batch = [(int(m), c) for m, c in stocks[i:i + 80]] + out.extend(await self._execute_mac(MacSymbolQuotesCmd(batch, bits))) + return out + + async def get_stock_quotes_list(self, category: int, *, start: int = 0, + count: int = 80, sort_type: int = 0) -> list: + """市场分类报价列表(category: 0=沪A 2=深A 6=全A 8=科创 14=创业板等)。""" + from .protocol.commands.mac_ext import MacBoardMembersQuotesCmd + items: list = [] + offset = start + while len(items) < count: + batch = await self._execute_mac( + MacBoardMembersQuotesCmd(category, sort_type=sort_type, + start=offset, page_size=min(80, count - len(items)))) + if not batch: + break + items.extend(batch) + offset += len(batch) + if len(batch) < 80: + break + return items + + # ------------------------------------------------------------------ # + # 远程文件 + # ------------------------------------------------------------------ # + + async def get_file_meta(self, filename: str): + """远程文件元信息(MAC 0x1215)。""" + from .protocol.commands.mac_ext import MacFileListCmd + return await self._execute_mac(MacFileListCmd(filename)) + + async def download_file(self, filename: str, filesize: int = 0) -> bytes: + """下载完整远程文件(MAC 0x1217,分块)。""" + from .protocol.commands.mac_ext import MacFileDownloadCmd + if filesize <= 0: + meta = await self.get_file_meta(filename) + filesize = meta.size if meta else 0 + data = bytearray() + pos = 0 + idx = 1 + chunk = 30000 + while pos < filesize: + part = await self._execute_mac( + MacFileDownloadCmd(filename, idx, pos, chunk)) + if not part: + break + data.extend(part) + pos += len(part) + idx += 1 + if len(part) < chunk: + break + return bytes(data) + + async def get_report_file(self, filename: str) -> bytes: + """下载报告/财务文件(标准 0x06B9,分块)。""" + from .protocol.commands.std_ext import GetReportFileCmd + data = bytearray() + pos = 0 + chunk = 30000 + while True: + part = await self._execute_standard(GetReportFileCmd(filename, pos, chunk)) + if not part: + break + data.extend(part) + pos += len(part) + if len(part) < chunk: + break + return bytes(data) + + async def get_financial_file_list(self) -> list[str]: + """财务文件列表(tdxfin/gpcw.txt,返回 文件名,md5,size 行列表)。""" + raw = await self.get_report_file("tdxfin/gpcw.txt") + text = raw.decode("gbk", errors="replace") + return [l.strip() for l in text.splitlines() if l.strip()] + + async def get_financial_records(self, filename: str) -> bytes: + """财务文件内容(tdxfin/gpcw*.zip,返回原始字节)。""" + return await self.get_report_file(filename) + + # ------------------------------------------------------------------ # + # 服务器文件解析(板块 / 行业分类 / 历史专业财报) + # ------------------------------------------------------------------ # + + async def get_block_parsed(self, filename: str) -> list[TdxBlock]: + """下载并解析板块文件,返回 TdxBlock 列表。 + + 常用文件名:``block_zs.dat``(行业)/ ``block_gn.dat``(概念)/ ``block_fg.dat``(风格)。 + category 由文件名推断(0=行业 2=概念 3=风格)。 + """ + from .codec.block import parse_block_dat + return parse_block_dat(await self.get_block_info(filename), filename) + + async def get_industry_map(self) -> dict[str, IndustryInfo]: + """下载并解析 tdxhy.cfg,返回 ``{6位代码: IndustryInfo}``(通达信+申万行业)。""" + from .codec.industry import parse_tdxhy_cfg + return parse_tdxhy_cfg(await self.get_report_file("tdxhy.cfg")) + + async def get_financial_file_infos(self) -> list[FinancialFileInfo]: + """下载并解析 tdxfin/gpcw.txt,返回 FinancialFileInfo 列表(文件名/MD5/大小)。""" + from .codec.financial import parse_financial_file_list + return parse_financial_file_list(await self.get_report_file("tdxfin/gpcw.txt")) + + async def get_financial_records_parsed(self, filename: str) -> list[FinancialRecord]: + """下载财报 zip(``tdxfin/gpcw*.zip``)并解压解析为 FinancialRecord 列表。 + + 报告期优先取文件名中的 8 位日期,缺失时回退到 .dat 头部内嵌值。 + """ + import io + import re + import zipfile + + from .codec.financial import parse_financial_dat + + zip_data = await self.get_report_file(filename) + if not zip_data: + return [] + with zipfile.ZipFile(io.BytesIO(zip_data)) as zf: + dat_names = [n for n in zf.namelist() if n.endswith(".dat")] + if not dat_names: + return [] + dat_data = zf.read(dat_names[0]) + match = re.search(r"(\d{8})", filename) + report_date = int(match.group(1)) if match else 0 + return parse_financial_dat(dat_data, report_date) + + # ------------------------------------------------------------------ # + # 板块聚合(基于 board_list/members/K线 组合) + # ------------------------------------------------------------------ # + + async def get_board_summary(self, board_symbol: str) -> dict: + """板块汇总:成分数/总成交额/总成交量/主力净流入/涨跌家数。""" + # 需要金额(0x07)与主力净流入(0x38)字段 + bits = list(range(0x00, 0x08)) + [0x38] + members = await self.get_board_members(board_symbol, count=100000) + # 重新拉含主力净流入字段(get_board_members 默认不含) + from .protocol.commands.mac_ext import MacBoardMembersQuotesCmd + board_code = self._convert_board_code(board_symbol) + all_m = [] + start = 0 + while start < 100000: + batch = await self._execute_mac( + MacBoardMembersQuotesCmd(board_code, sort_type=0, start=start, + page_size=80, bits=bits)) + if not batch: + break + all_m.extend(batch) + start += len(batch) + if len(batch) < 80: + break + amount = sum(m.fields.get("0x7", 0) for m in all_m) + main_net = sum(m.fields.get("0x38", 0) for m in all_m) + up = down = 0 + for m in all_m: + c, pre = m.fields.get("0x4", 0), m.fields.get("0x0", 0) + if pre and c > pre: + up += 1 + elif pre and c < pre: + down += 1 + return { + "member_count": len(all_m), + "amount": amount, "main_net_amount": main_net, + "up_count": up, "down_count": down, + } + + async def get_board_ranking(self, board_type: int = 1, top_n: int = 20) -> list[dict]: + """板块涨跌幅排行(行业/概念,基于板块K线计算涨跌幅)。""" + boards = await self.get_board_list(board_type, count=200) + # 去重(board_list返回板块+领涨股混合,取板块段) + seen = {} + for b in boards: + if not b.code.startswith("88"): # 只保留板块段(881xxx), 过滤领涨股 + continue + if b.code not in seen or len(b.name) < len(seen[b.code].name): + seen[b.code] = b + result = [] + for code, b in list(seen.items())[:top_n * 3]: + if b.pre_close <= 0: + continue + chg = (b.price / b.pre_close - 1) * 100 + result.append({"code": code, "name": b.name, "price": b.price, + "change_pct": round(chg, 2)}) + result.sort(key=lambda x: x["change_pct"], reverse=True) + return result[:top_n] + + async def get_board_change_ranking(self, board_type: int = 1, days: int = 20, + top_n: int = 20) -> list[dict]: + """板块N日涨跌幅排行(用板块指数K线)。""" + from .models import Adjust, KlinePeriod + boards = await self.get_board_list(board_type, count=200) + seen = {} + for b in boards: + if not b.code.startswith("88"): # 只保留板块段, 过滤领涨股 + continue + if b.code not in seen or len(b.name) < len(seen[b.code].name): + seen[b.code] = b + result = [] + for code, b in list(seen.items())[:top_n * 3]: + try: + bars = await self.get_kline(1, code, KlinePeriod.DAY, count=days + 5, + adjust=Adjust.NONE) + if len(bars) < 2: + continue + first, last = bars[0].close, bars[-1].close + if first <= 1.0: # 过滤异常基数(板块指数首根过小) + continue + result.append({"code": code, "name": b.name, + "close_end": last, "close_start": first, + "change_pct": round((last / first - 1) * 100, 2)}) + except Exception: + continue + result.sort(key=lambda x: x["change_pct"], reverse=True) + return result[:top_n] + + # ------------------------------------------------------------------ # + # 剩余接口:分时采样 / 列表all / K线+指标(带缓存) + # ------------------------------------------------------------------ # + + async def get_chart_sampling(self, market: Market, code: str) -> list[float]: + """分时缩略采样价格点(MAC 0x254D)。""" + from .protocol.commands.mac_ext import MacChartSamplingCmd + key = CacheKey("minute_sampling", int(market), code) + async def fetch() -> bytes: + return pack_payload(await self._execute_mac(MacChartSamplingCmd(int(market), code))) + return unpack_payload(await self._cache.get_or_fetch(key, fetch)) + + async def get_security_list_all(self, market: Market) -> list[SecurityInfo]: + """全市场证券列表(高并发分页拉取,缓存1天)。 + + 优化:高并发一次并发拉完所有页。 + """ + key = CacheKey("list_all", int(market)) + async def fetch() -> bytes: + # 先获取总数(带缓存),精确规划页数 + total = await self.get_security_count(market) + def make_cmd(s: int, c: int): + return GetSecurityListCmd(int(market), s) + spec = PageSpec(page_size=LIST_PAGE, total=total, stop_when_short=True) + result = await self._fetch_paginated_list( + "standard", make_cmd, spec, + concurrency=min(self._std_pool.max_size, 8)) + return pack_payload(result) + return unpack_payload(await self._cache.get_or_fetch(key, fetch)) + + async def get_stock_kline_with_indicators( + self, market: Market, code: str, indicators: list[str], *, + period: KlinePeriod = KlinePeriod.DAY, count: int = 30, + adjust: Adjust | None = None, params: dict | None = None, + ) -> dict: + """K线+技术指标(MACD/KDJ/RSI/BOLL/MA/EMA,缓存=复权K线TTL)。""" + from .indicator import compute_indicators + adj = self.default_adjust if adjust is None else adjust + fetch_count = max(120 + count, 200) # 指标预热 + bars = await self.get_kline(market, code, period, count=fetch_count, adjust=adj) + if not bars: + return {"bars": [], "indicators": {}} + high = [b.high for b in bars] + low = [b.low for b in bars] + close = [b.close for b in bars] + ind = compute_indicators(high, low, close, indicators, params, tail=count) + return { + "bars": bars[-count:], + "indicators": {k: (v[-count:] if len(v) >= count else v) for k, v in ind.items()}, + } diff --git a/src/pytdxdata/cache/__init__.py b/src/pytdxdata/cache/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pytdxdata/cache/disk.py b/src/pytdxdata/cache/disk.py new file mode 100644 index 0000000..8b0c960 --- /dev/null +++ b/src/pytdxdata/cache/disk.py @@ -0,0 +1,148 @@ +"""磁盘持久缓存:SQLite WAL + asyncio.to_thread(事件循环零阻塞)""" +from __future__ import annotations + +import asyncio +import sqlite3 +import threading +import time +from pathlib import Path + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS cache ( + key TEXT PRIMARY KEY, + ttl_name TEXT NOT NULL, + expires_at REAL NOT NULL, + size INTEGER NOT NULL, + payload BLOB NOT NULL, + created_at REAL NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_cache_expires ON cache(expires_at); +""" + + +class DiskCache: + def __init__(self, path: Path, *, max_bytes: int = 512 * 1024 * 1024) -> None: + self.path = Path(path) + self.max_bytes = max_bytes + self._write_conn: sqlite3.Connection | None = None + self._write_lock = threading.Lock() + self._read_conn: sqlite3.Connection | None = None + + async def open(self) -> None: + def _init(): + self.path.parent.mkdir(parents=True, exist_ok=True) + self._write_conn = sqlite3.connect(str(self.path), check_same_thread=False) + self._write_conn.execute("PRAGMA journal_mode=WAL") + self._write_conn.execute("PRAGMA synchronous=NORMAL") + self._write_conn.executescript(_SCHEMA) + self._write_conn.commit() + # 读连接 + self._read_conn = sqlite3.connect(str(self.path), check_same_thread=False) + self._read_conn.execute("PRAGMA query_only=ON") + await asyncio.to_thread(_init) + + async def close(self) -> None: + def _close(): + for c in (self._write_conn, self._read_conn): + if c is not None: + c.close() + await asyncio.to_thread(_close) + + async def get(self, key: str) -> bytes | None: + return (await self.get_many([key]))[0] + + async def get_many(self, keys: list[str]) -> list[bytes | None]: + """批量读,减少往返。""" + def _read() -> list[bytes | None]: + conn = self._read_conn + if conn is None: + return [None] * len(keys) + now = time.time() + out: list[bytes | None] = [] + for k in keys: + row = conn.execute( + "SELECT payload, expires_at FROM cache WHERE key=?", (k,) + ).fetchone() + if row is None: + out.append(None) + elif row[1] > 0 and now > row[1]: + out.append(None) + else: + out.append(row[0]) + return out + return await asyncio.to_thread(_read) + + async def set(self, key: str, value: bytes, ttl: float, ttl_name: str = "") -> None: + def _write(): + with self._write_lock: + if self._write_conn is None: + return + now = time.time() + expires_at = 0.0 if ttl <= 0 else now + ttl + self._write_conn.execute( + "INSERT OR REPLACE INTO cache (key, ttl_name, expires_at, size, payload, created_at) " + "VALUES (?,?,?,?,?,?)", + (key, ttl_name, expires_at, len(value), value, now), + ) + self._write_conn.commit() + await asyncio.to_thread(_write) + + async def delete(self, key: str) -> None: + def _del(): + with self._write_lock: + if self._write_conn is not None: + self._write_conn.execute("DELETE FROM cache WHERE key=?", (key,)) + self._write_conn.commit() + await asyncio.to_thread(_del) + + async def delete_prefix(self, prefix: str) -> int: + """按前缀批量删除,返回删除条数。""" + def _del_prefix() -> int: + with self._write_lock: + if self._write_conn is None: + return 0 + cur = self._write_conn.execute( + "DELETE FROM cache WHERE key LIKE ?", (f"{prefix}%",) + ) + self._write_conn.commit() + return cur.rowcount + return await asyncio.to_thread(_del_prefix) + + async def sweep(self) -> int: + """清理过期行 + 超限淘汰,返回清理条数。""" + def _sweep() -> int: + with self._write_lock: + if self._write_conn is None: + return 0 + now = time.time() + cur = self._write_conn.execute( + "DELETE FROM cache WHERE expires_at > 0 AND expires_at < ?", (now,) + ) + n = cur.rowcount + # 超限淘汰最旧 + total = self._write_conn.execute("SELECT COALESCE(SUM(size),0) FROM cache").fetchone()[0] + if total > self.max_bytes: + overflow = total - self.max_bytes + rows = self._write_conn.execute( + "SELECT key, size FROM cache ORDER BY created_at ASC" + ).fetchall() + freed = 0 + for k, sz in rows: + if freed >= overflow: + break + self._write_conn.execute("DELETE FROM cache WHERE key=?", (k,)) + freed += sz + n += 1 + self._write_conn.commit() + return n + return await asyncio.to_thread(_sweep) + + async def stats(self) -> dict: + def _stats() -> dict: + if self._read_conn is None: + return {"rows": 0, "bytes": 0} + r = self._read_conn.execute( + "SELECT count(*), COALESCE(SUM(size),0) FROM cache" + ).fetchone() + return {"rows": r[0], "bytes": r[1]} + return await asyncio.to_thread(_stats) diff --git a/src/pytdxdata/cache/key.py b/src/pytdxdata/cache/key.py new file mode 100644 index 0000000..0fbb5d6 --- /dev/null +++ b/src/pytdxdata/cache/key.py @@ -0,0 +1,46 @@ +"""缓存键构造:{category}|{market}|{code}|{period}|{start}|{count}|{extra}""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Sequence + + +@dataclass(frozen=True, slots=True) +class CacheKey: + category: str # kline|quote|transaction|list|minute|xdxr|finance + market: int | None = None + code: str | None = None + period: str | None = None + start: int | None = None + count: int | None = None + extra: str = "" # adjust/date/股票集哈希 等规范化参数 + + def to_str(self) -> str: + return "|".join([ + self.category, + "" if self.market is None else str(self.market), + self.code or "", + self.period or "", + "" if self.start is None else str(self.start), + "" if self.count is None else str(self.count), + self.extra, + ]) + + @classmethod + def from_str(cls, s: str) -> "CacheKey": + parts = s.split("|") + while len(parts) < 7: + parts.append("") + def _i(v: str) -> int | None: + return int(v) if v else None + return cls( + category=parts[0], market=_i(parts[1]), code=parts[2] or None, + period=parts[3] or None, start=_i(parts[4]), count=_i(parts[5]), + extra=parts[6], + ) + + @classmethod + def quote_key(cls, stocks: Sequence[tuple[int, str]]) -> "CacheKey": + """报价键:对股票集规范化排序,顺序无关可命中。""" + norm = sorted(f"{m}:{c}" for m, c in stocks) + return cls(category="quote", extra=",".join(norm)) diff --git a/src/pytdxdata/cache/manager.py b/src/pytdxdata/cache/manager.py new file mode 100644 index 0000000..61228f2 --- /dev/null +++ b/src/pytdxdata/cache/manager.py @@ -0,0 +1,125 @@ +"""缓存管理器:内存LRU + 磁盘SQLite 双层 + single-flight 并发去重""" +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from typing import Any + +from .disk import DiskCache +from .key import CacheKey +from .memory import MemoryCache +from .ttl import ttl_for + +log = logging.getLogger(__name__) + + +class CacheManager: + """双层缓存:读 内存→磁盘→回源;写 内存立即+异步回写磁盘。 + + single-flight:同 key 并发请求只回源一次,其余 await 同一 future。 + """ + + def __init__(self, memory: MemoryCache, disk: DiskCache | None = None, + *, max_inflight: int = 64) -> None: + self.memory = memory + self.disk = disk + self._locks: dict[str, asyncio.Lock] = {} + self._semaphore = asyncio.Semaphore(max_inflight) + + async def start(self) -> None: + if self.disk is not None: + await self.disk.open() + await self.disk.sweep() + + async def close(self) -> None: + if self.disk is not None: + await self.disk.close() + + async def get(self, key: CacheKey) -> bytes | None: + ks = key.to_str() + # 1. 内存 + v = self.memory.get(ks) + if v is not None: + return v + # 2. 磁盘 + if self.disk is not None: + v = (await self.disk.get_many([ks]))[0] + if v is not None: + ttl = ttl_for(key.category, period=key.period, adjust=key.extra, date=_parse_date(key.extra)) + self.memory.set(ks, v, ttl) + return v + return None + + async def set(self, key: CacheKey, payload: bytes, *, ttl: float | None = None) -> None: + ks = key.to_str() + t = ttl if ttl is not None else ttl_for( + key.category, period=key.period, adjust=key.extra, date=_parse_date(key.extra) + ) + self.memory.set(ks, payload, t) + if self.disk is not None: + # 异步回写(best-effort,失败不阻塞) + try: + await self.disk.set(ks, payload, t, ttl_name=key.category) + except Exception: + log.debug("磁盘缓存写失败: %s", ks) + + async def get_or_fetch( + self, + key: CacheKey, + fetcher: Callable[[], Awaitable[bytes]], + ) -> bytes: + """命中缓存返回;否则 single-flight 回源并缓存。""" + ks = key.to_str() + cached = await self.get(key) + if cached is not None: + return cached + + lock = self._locks.setdefault(ks, asyncio.Lock()) + try: + async with lock: + # 双检:等待期间可能已被其他任务写入 + cached = await self.get(key) + if cached is not None: + return cached + async with self._semaphore: + payload = await fetcher() + await self.set(key, payload) + return payload + finally: + # 确保清理锁(即使 fetcher 异常也清理,避免无限增长) + self._locks.pop(ks, None) + + async def invalidate(self, category: str, *, market: int | None = None, + code: str | None = None) -> int: + """按前缀批量失效,返回失效条数。""" + n = 0 + # 内存:遍历匹配(使用公共 keys() 方法) + keys = self.memory.keys() + for k in keys: + try: + ck = CacheKey.from_str(k) + except Exception: + continue + if ck.category == category: + if market is not None and ck.market != market: + continue + if code is not None and ck.code != code: + continue + self.memory.delete(k) + n += 1 + # 磁盘:按前缀删除(best-effort) + if self.disk is not None: + prefix = f"{category}|{'' if market is None else market}|" + try: + await self.disk.delete_prefix(prefix) + except Exception: + pass + return n + + +def _parse_date(extra: str) -> int | None: + """从 extra 解析 date(transaction类)。extra 可能是 '20260811' 或空。""" + if extra and extra.isdigit() and len(extra) == 8: + return int(extra) + return None diff --git a/src/pytdxdata/cache/memory.py b/src/pytdxdata/cache/memory.py new file mode 100644 index 0000000..f21f44f --- /dev/null +++ b/src/pytdxdata/cache/memory.py @@ -0,0 +1,63 @@ +"""内存TTL感知LRU缓存(OrderedDict,get命中刷新热度,超容量逐出最旧)""" +from __future__ import annotations + +import time +from collections import OrderedDict + + +class MemoryCache: + def __init__(self, max_entries: int = 4096, max_bytes: int = 128 * 1024 * 1024) -> None: + self.max_entries = max_entries + self.max_bytes = max_bytes + self._data: OrderedDict[str, tuple[bytes, float]] = OrderedDict() # key -> (value, expires_at) + self._bytes = 0 + + def get(self, key: str) -> bytes | None: + item = self._data.get(key) + if item is None: + return None + value, expires_at = item + if expires_at > 0 and time.monotonic() > expires_at: + self._delete(key) + return None + self._data.move_to_end(key) # LRU刷新 + return value + + def set(self, key: str, value: bytes, ttl: float) -> None: + expires_at = 0.0 if ttl <= 0 else time.monotonic() + ttl + old = self._data.pop(key, None) + if old is not None: + self._bytes -= len(old[0]) + self._data[key] = (value, expires_at) + self._bytes += len(value) + # 超容量逐出最旧 + while (len(self._data) > self.max_entries or self._bytes > self.max_bytes) and self._data: + self._data.popitem(last=False) + # 简化:逐出后重新计算字节数 + self._recount() + + def delete(self, key: str) -> None: + self._delete(key) + + def keys(self) -> list[str]: + """返回当前所有缓存 key(供失效/清理使用)。""" + return list(self._data.keys()) + + def _delete(self, key: str) -> None: + item = self._data.pop(key, None) + if item is not None: + self._bytes -= len(item[0]) + + def _recount(self) -> None: + self._bytes = sum(len(v[0]) for v in self._data.values()) + + def clear(self) -> None: + self._data.clear() + self._bytes = 0 + + def __len__(self) -> int: + return len(self._data) + + @property + def bytes_used(self) -> int: + return self._bytes diff --git a/src/pytdxdata/cache/ttl.py b/src/pytdxdata/cache/ttl.py new file mode 100644 index 0000000..36a35ea --- /dev/null +++ b/src/pytdxdata/cache/ttl.py @@ -0,0 +1,50 @@ +"""TTL分级策略:按数据类型返回缓存有效期""" +from __future__ import annotations + +from datetime import datetime, time as dtime, timedelta, timezone + +# 上海时区 UTC+8 +SH_TZ = timezone(timedelta(hours=8)) + +# 分钟级周期(盘中数据变化快) +MINUTE_PERIODS = {"MIN_1", "MIN_5", "MIN_15", "MIN_30", "MIN_60"} + + +def ttl_for( + category: str, + *, + period: str | None = None, + adjust: str | None = None, + date: int | None = None, +) -> float: + """返回 TTL(秒)。0=不过期(永久),负值=立即失效。""" + if category == "quote": + return 5.0 + if category == "kline": + if period in MINUTE_PERIODS: + return 60.0 + if adjust in ("QFQ", "HFQ"): + return 86400.0 + return _until_next_midnight() # 日线当天(次日00:30失效) + if category == "transaction": + if date is None or date == _today_ymd(): + return 60.0 # 当日逐笔变化快 + return 0.0 # 历史逐笔永久 + if category == "minute": + return 60.0 + if category in ("list", "count", "xdxr", "finance"): + return 86400.0 + return 60.0 + + +def _until_next_midnight() -> float: + """距次日00:30(上海时区)的秒数,12h兜底。""" + now = datetime.now(SH_TZ) + nxt = datetime.combine(now.date() + timedelta(days=1), dtime(0, 30), tzinfo=SH_TZ) + ttl = (nxt - now).total_seconds() + return max(0.0, min(ttl, 12 * 3600)) + + +def _today_ymd() -> int: + now = datetime.now(SH_TZ) + return now.year * 10000 + now.month * 100 + now.day diff --git a/src/pytdxdata/cli.py b/src/pytdxdata/cli.py new file mode 100644 index 0000000..3ba89d5 --- /dev/null +++ b/src/pytdxdata/cli.py @@ -0,0 +1,541 @@ +"""pytdxdata CLI 工具 + +用法: + tdx quotes sz000001 # 行情报价(五档) + tdx kline sz000001 --period day --count 30 --adjust qfq + tdx kline hk00700 --count 30 # 港股K线 + tdx kline cffex:IFL0 --count 30 # 期货K线(郑商所:zz / 大商所:dl / 上期所:sh / 中金所:cffex / 广期所:gz) + tdx transactions sz000001 --count 50 # 逐笔成交 + tdx minute sz000001 # 分时(今日/历史--date) + tdx index 000001 --count 30 # 指数K线 + tdx board list [--type 1] # 板块列表(0全部/1行业/2概念) + tdx board members 881001 --count 20 # 板块成分 + tdx board ranking [--top 10] # 板块涨跌幅排行 + tdx board change --days 5 --top 10 # 板块N日涨幅排行 + tdx info sz000001 # 个股特征快照 + tdx list [--market sz] [--count 100] # 证券列表 + tdx auction sz000001 # 集合竞价 + tdx unusual [--market sh] [--count 30] # 市场异动 + tdx xdxr sz000001 # 除权除息 + tdx finance sz000001 # 财务快照 + tdx flow sz000001 # 资金流向 + tdx server-info # 服务器交易时段 + tdx market-stat # 市场统计 + tdx block [block_gn.dat] [--limit N] # 板块文件下载+解析(成分股) + tdx industry [codes...] [--search 词] # 行业分类下载+解析(tdxhy.cfg) + tdx financial-list # 历史专业财报文件索引(gpcw.txt) + tdx financial gpcw20260331.zip # 历史专业财报记录下载+解析(gpcw*.zip) +""" +from __future__ import annotations + +import argparse +import asyncio +import sys +from datetime import datetime + +from .api import TdxData +from dataclasses import asdict + +from .models import Adjust, ExMarket, KlinePeriod, Market + +PERIODS = { + "min1": KlinePeriod.MIN_1, "min5": KlinePeriod.MIN_5, "min15": KlinePeriod.MIN_15, + "min30": KlinePeriod.MIN_30, "min60": KlinePeriod.MIN_60, + "day": KlinePeriod.DAY, "week": KlinePeriod.WEEK, "month": KlinePeriod.MONTH, + "year": KlinePeriod.YEAR, "season": KlinePeriod.SEASON, +} +ADJUSTS = {"none": Adjust.NONE, "qfq": Adjust.QFQ, "hfq": Adjust.HFQ} +MARKET_NAMES = {"sz": Market.SZ, "sh": Market.SH, "bj": Market.BJ} +EX_MARKETS = { + "hk": ExMarket.HK_MAIN_BOARD, "us": ExMarket.US_STOCK, + "zz": ExMarket.ZZ_FUTURES, "dl": ExMarket.DL_FUTURES, + "shf": ExMarket.SH_FUTURES, "cffex": ExMarket.CFFEX_FUTURES, "gz": ExMarket.GZ_FUTURES, +} + + +class Symbol: + """解析后的标的: kind=cn(标准) / ex(扩展市场) / index(指数)""" + + def __init__(self, kind: str, market: int, code: str): + self.kind = kind + self.market = market + self.code = code + + def to_market(self) -> Market: + return Market(self.market) + + +def parse_symbol(sym: str) -> Symbol: + """sz000001 / sh600000 / bj920992 / hk00700 / usAAPL / cffex:IFL0 / 000001(指数)""" + sym = sym.strip() + if ":" in sym: + mkt, code = sym.split(":", 1) + mkt = mkt.lower() + if mkt in EX_MARKETS: + return Symbol("ex", int(EX_MARKETS[mkt]), code.upper()) + if mkt in MARKET_NAMES: + return Symbol("cn", int(MARKET_NAMES[mkt]), code.upper()) + raise ValueError("未知市场前缀: %s (cn: sz/sh/bj; ex: hk/us/zz/dl/shf/cffex/gz)" % mkt) + prefix, code = sym[:2].lower(), sym[2:] + if prefix in MARKET_NAMES and len(code) == 6: + return Symbol("cn", int(MARKET_NAMES[prefix]), code.upper()) + if prefix == "hk" and len(code) == 5: + return Symbol("ex", int(ExMarket.HK_MAIN_BOARD), code.upper()) + if prefix == "us": + return Symbol("ex", int(ExMarket.US_STOCK), code.upper()) + if len(sym) == 6 and sym.isdigit(): + return Symbol("index", int(Market.SH), sym.upper()) + raise ValueError("无法识别标的: %s" % sym) + + +# ------------------------------------------------------------------ # +# 打印辅助 +# ------------------------------------------------------------------ # + +def _fmt_num(v: float) -> str: + if v >= 1e8: + return "%.2f亿" % (v / 1e8) + if v >= 1e4: + return "%.2f万" % (v / 1e4) + return str(int(v)) + + +def _pct(price: float, pre: float) -> str: + if not pre: + return "" + return "%+.2f%%" % ((price / pre - 1) * 100) + + +# ------------------------------------------------------------------ # +# 各子命令 +# ------------------------------------------------------------------ # + +async def cmd_quotes(td: TdxData, symbols: list[str]) -> None: + cn, ex = [], [] + for sym in symbols: + s = parse_symbol(sym) + if s.kind == "cn": + cn.append((Market(s.market), s.code)) + else: + ex.append((s.market, s.code)) + for q in await td.get_quotes(cn): + _print_cn_quote(q) + if ex: + from .protocol.commands.ex_proto import GetExInstrumentQuoteCmd + for market, code in ex: + q = await td._execute_ex(GetExInstrumentQuoteCmd(market, code)) + _print_ex_quote(market, code, q) + + +def _print_cn_quote(q) -> None: + pct = (q.price / q.pre_close - 1) * 100 if q.pre_close else 0 + print("=" * 58) + print(" %s %s" % (q.code, q.server_time or datetime.now().strftime("%H:%M:%S"))) + print(" 最新 %-10.2f 涨跌 %+8.2f (%+.2f%%)" % (q.price, q.price - q.pre_close, pct)) + print(" 今开 %-10.2f 最高 %.2f 最低 %.2f" % (q.open, q.high, q.low)) + print(" 昨收 %-10.2f 成交量 %s 成交额 %s" % (q.pre_close, _fmt_num(q.vol), _fmt_num(q.amount))) + if q.bid and q.ask: + print(" " + "-" * 56) + for i in range(max(len(q.bid), len(q.ask))): + b = q.bid[i] if i < len(q.bid) else (0, 0) + a = q.ask[i] if i < len(q.ask) else (0, 0) + print(" 买%-1d %8.2f %10s | 卖%-1d %8.2f %10s" % ( + i + 1, b[0], _fmt_num(b[1]), i + 1, a[0], _fmt_num(a[1]))) + print() + + +def _print_ex_quote(market: int, code: str, q: dict | None) -> None: + if not q: + print(" %s: 无报价数据" % code) + return + price, pre = q.get("price") or 0, q.get("pre_close") or 0 + mkt = {31: "港股", 74: "美股"}.get(market, "扩展") + print("=" * 58) + print(" %s %s" % (mkt, code)) + print(" 最新 %-10.2f 涨跌 %+8.2f (%s)" % (price, price - pre, _pct(price, pre))) + print(" 今开 %-10.2f 最高 %.2f 最低 %.2f" % ( + q.get("open") or 0, q.get("high") or 0, q.get("low") or 0)) + print(" 昨收 %-10.2f 总量 %s" % (pre, _fmt_num(q.get("zongliang") or 0))) + print() + + +async def cmd_kline(td: TdxData, sym: str, period: KlinePeriod, count: int, + start: int, adjust: Adjust) -> None: + s = parse_symbol(sym) + if s.kind == "cn": + bars = await td.get_kline(Market(s.market), s.code, period, start=start, + count=count, adjust=adjust) + elif s.kind == "index": + bars = await td.get_index_kline(Market(s.market), s.code, period, start=start, count=count) + else: + bars = await td.get_kline(ExMarket(s.market), s.code, period, start=start, count=count) + print("%s K线 (%s, %d根):" % (sym, period.name, len(bars))) + print(" %-12s %8s %8s %8s %8s %12s" % ("日期", "开盘", "最高", "最低", "收盘", "成交量")) + for b in bars: + dt = "%04d-%02d-%02d" % (b.year, b.month, b.day) + if b.hour: + dt += " %02d:%02d" % (b.hour, b.minute) + print(" %-12s %8.2f %8.2f %8.2f %8.2f %12s" % ( + dt, b.open, b.high, b.low, b.close, _fmt_num(b.vol))) + + +async def cmd_transactions(td: TdxData, sym: str, count: int, date: int | None) -> None: + s = parse_symbol(sym) + tx = await td.get_transactions(Market(s.market), s.code, date=date, count=count) + print("%s 逐笔成交 (%d条):" % (sym, len(tx))) + print(" 时间 价格 成交量 方向") + for t in tx: + bs = "买" if getattr(t, "bs_flag", 0) == 1 else "卖" if getattr(t, "bs_flag", 0) == 2 else "-" + tc = getattr(t, "trade_count", 0) + print(" %-10s %8.2f %10s %s%s" % (t.time, t.price, _fmt_num(t.vol), bs, + " (%d笔)" % tc if tc else "")) + + +async def cmd_minute(td: TdxData, sym: str, date: int | None) -> None: + s = parse_symbol(sym) + bars = await td.get_minute(Market(s.market), s.code, date=date) + print("%s 分时 (%d条):" % (sym, len(bars))) + print(" 时间 价格 均价 成交量") + for i, b in enumerate(bars): + print(" %02d:%02d %8.2f %8.2f %10s" % ( + i // 60, i % 60, b.price, b.avg_price, _fmt_num(b.vol))) + + +async def cmd_board(td: TdxData, args) -> None: + if args.action == "list": + items = await td.get_board_list(args.type, args.count) + print("板块列表 (%d条, type=%d):" % (len(items), args.type)) + print(" %-10s %-16s %10s %10s" % ("代码", "名称", "涨跌幅%", "成交额")) + for b in items: + name = getattr(b, "name", "") + code = getattr(b, "code", "") + chg = getattr(b, "change_pct", None) + amt = getattr(b, "amount", None) + print(" %-10s %-16s %10s %10s" % ( + code, name, ("%.2f" % chg) if chg is not None else "-", + _fmt_num(amt) if amt else "-")) + elif args.action == "members": + items = await td.get_board_members(args.symbol, count=args.count) + print("%s 成分股 (%d条):" % (args.symbol, len(items))) + for m in items: + f = getattr(m, "fields", {}) + price = f.get("0x4") or f.get("price") or 0 + chg = f.get("0x2e") + print(" %-10s %-10s 价=%.2f%s" % ( + m.code, m.name[:6], price, " 涨%.2f%%" % chg if chg else "")) + elif args.action == "ranking": + rows = await td.get_board_ranking(args.type, args.top) + print("板块涨跌幅排行 (type=%d, top%d):" % (args.type, args.top)) + for r in rows: + print(" %-12s %+8.2f%%" % (r.get("name", ""), r.get("change_pct", 0))) + elif args.action == "change": + rows = await td.get_board_change_ranking(args.type, args.days, args.top) + print("板块%d日涨幅排行 (top%d):" % (args.days, args.top)) + for r in rows: + print(" %-12s %+8.2f%%" % (r.get("name", ""), r.get("change_pct", 0))) + + +async def cmd_info(td: TdxData, sym: str) -> None: + s = parse_symbol(sym) + info = await td.get_symbol_info(Market(s.market), s.code) + if isinstance(info, dict): + items = info.items() + else: + items = asdict(info).items() + for k, v in items: + print(" %-24s %s" % (k, v)) + + +async def cmd_list(td: TdxData, market: str, count: int) -> None: + m = MARKET_NAMES[market] + n = await td.get_security_count(m) + items = await td.get_security_list(m, count=count) + print("%s市场证券数: %d, 前%d条:" % (market.upper(), n, len(items))) + for it in items: + print(" %s %s" % (it.code, it.name.strip())) + + +async def cmd_auction(td: TdxData, sym: str) -> None: + s = parse_symbol(sym) + items = await td.get_auction(Market(s.market), s.code) + print("%s 集合竞价 (%d条):" % (sym, len(items))) + for a in items: + print(" %s 价格=%.3f 量=%d %s" % ( + getattr(a, "time", ""), getattr(a, "price", 0), + getattr(a, "volume", 0), getattr(a, "side", ""))) + + +async def cmd_unusual(td: TdxData, market: str, count: int) -> None: + items = await td.get_unusual(MARKET_NAMES[market], count=count) + print("市场异动 (%d条):" % len(items)) + for u in items: + print(" %s %s %s" % (getattr(u, "code", ""), getattr(u, "name", ""), + getattr(u, "describe", "") or getattr(u, "type", ""))) + + +async def cmd_xdxr(td: TdxData, sym: str) -> None: + s = parse_symbol(sym) + items = await td.get_xdxr(Market(s.market), s.code) + print("%s 除权除息 (%d条):" % (sym, len(items))) + for x in items[:20]: + print(" %s 送%g 配%g 派%g" % ( + getattr(x, "year", ""), getattr(x, "songzhuangu", 0) or 0, + getattr(x, "peigu", 0) or 0, getattr(x, "peixian", 0) or 0)) + + +async def cmd_finance(td: TdxData, sym: str) -> None: + s = parse_symbol(sym) + f = await td.get_finance(Market(s.market), s.code) + if not f: + print("无财务数据") + return + for k, v in asdict(f).items(): + print(" %-24s %s" % (k, v)) + + +async def cmd_flow(td: TdxData, sym: str) -> None: + s = parse_symbol(sym) + f = await td.get_capital_flow(Market(s.market), s.code) + if isinstance(f, dict): + items = f.items() + else: + items = asdict(f).items() + for k, v in items: + print(" %-20s %s" % (k, v)) + + +_BLOCK_CATEGORY = {0: "行业", 1: "地域", 2: "概念", 3: "风格"} + + +async def cmd_block(td: TdxData, filename: str, limit: int, show_codes: bool) -> None: + blocks = await td.get_block_parsed(filename) + print("%s 板块文件 (%d 个板块):" % (filename, len(blocks))) + if not blocks: + return + print(" %-4s %-14s %6s %s" % ("类别", "板块名", "成分数", "代码示例")) + shown = blocks if limit <= 0 else blocks[:limit] + for b in shown: + cat = _BLOCK_CATEGORY.get(getattr(b, "category", -1), "?") + codes = getattr(b, "codes", None) or [] + count = getattr(b, "count", len(codes)) + sample = " ".join(codes[:5]) + (" ..." if len(codes) > 5 else "") + print(" %-4s %-14s %6d %s" % (cat, getattr(b, "name", ""), count, sample)) + if show_codes and codes: + print(" " + " ".join(codes)) + if 0 < limit < len(blocks): + print(" ... 其余 %d 个板块省略(--limit 0 显示全部)" % (len(blocks) - limit)) + + +async def cmd_industry(td: TdxData, codes: list[str], limit: int, search: str | None) -> None: + hy = await td.get_industry_map() + if codes: + print("行业分类 (共 %d 只):" % len(hy)) + for c in codes: + info = hy.get(c) + if info is None: + print(" %-8s 未收录" % c) + else: + print(" %-8s 通达信=%-12s 申万=%s" % ( + c, info.tdx_industry or "-", info.sw_industry or "-")) + return + rows = list(hy.items()) + if search: + rows = [(c, i) for c, i in rows + if search in (i.tdx_industry or "") or search in (i.sw_industry or "")] + tip = ",匹配 '%s' %d 条" % (search, len(rows)) if search else "" + print("行业分类 tdxhy.cfg (共 %d 只%s):" % (len(hy), tip)) + print(" %-8s %-14s %s" % ("代码", "通达信行业", "申万行业")) + shown = rows if limit <= 0 else rows[:limit] + for c, i in shown: + print(" %-8s %-14s %s" % (c, i.tdx_industry or "-", i.sw_industry or "-")) + if 0 < limit < len(rows): + print(" ... 其余 %d 条省略(--limit 0 显示全部)" % (len(rows) - limit)) + + +async def cmd_financial_list(td: TdxData, limit: int) -> None: + infos = await td.get_financial_file_infos() + print("历史专业财报文件 (gpcw.txt, 共 %d 个):" % len(infos)) + if not infos: + return + print(" %-24s %-34s %12s" % ("文件名", "MD5", "大小")) + shown = infos if limit <= 0 else infos[:limit] + for i in shown: + print(" %-24s %-34s %12d" % (i.filename, i.hash, i.filesize)) + if 0 < limit < len(infos): + print(" ... 其余 %d 个省略(--limit 0 显示全部)" % (len(infos) - limit)) + print("\n 提示: 用 `tdx financial %s --limit 5` 查看最新一期财报" % infos[-1].filename) + + +async def cmd_financial(td: TdxData, filename: str, code: str | None, + limit: int, fields: int) -> None: + records = await td.get_financial_records_parsed(filename) + if code: + records = [r for r in records if r.code == code] + print("%s 财报记录 (%d 条):" % (filename, len(records))) + if not records: + return + shown = records if limit <= 0 else records[:limit] + for r in shown: + sample = " ".join("%g" % v for v in r.fields[:fields]) if fields > 0 else "" + print(" %-8s 市场=%s 报告期=%s 字段数=%d %s" % ( + r.code, "沪" if r.market == 1 else "深", + r.report_date or "-", len(r.fields), sample)) + if 0 < limit < len(records): + print(" ... 其余 %d 条省略(--limit 0 显示全部)" % (len(records) - limit)) + + +# ------------------------------------------------------------------ # +# 入口 +# ------------------------------------------------------------------ # + +def main() -> None: + ap = argparse.ArgumentParser(prog="tdx", description="通达信行情CLI (pytdxdata)") + sub = ap.add_subparsers(dest="cmd", required=True) + + def add(name, help): + return sub.add_parser(name, help=help) + + q = add("quotes", "行情报价") + q.add_argument("symbols", nargs="+") + q.set_defaults(func=lambda td, a: cmd_quotes(td, a.symbols)) + + k = add("kline", "K线") + k.add_argument("symbol") + k.add_argument("--period", choices=list(PERIODS), default="day") + k.add_argument("--count", type=int, default=30) + k.add_argument("--start", type=int, default=0) + k.add_argument("--adjust", choices=list(ADJUSTS), default="none") + k.set_defaults(func=lambda td, a: cmd_kline( + td, a.symbol, PERIODS[a.period], a.count, a.start, ADJUSTS[a.adjust])) + + t = add("transactions", "逐笔成交") + t.add_argument("symbol") + t.add_argument("--count", type=int, default=30) + t.add_argument("--date", type=str, default=None) + t.set_defaults(func=lambda td, a: cmd_transactions( + td, a.symbol, a.count, int(a.date) if a.date else None)) + + m = add("minute", "分时") + m.add_argument("symbol") + m.add_argument("--date", type=str, default=None) + m.set_defaults(func=lambda td, a: cmd_minute(td, a.symbol, int(a.date) if a.date else None)) + + ix = add("index", "指数K线") + ix.add_argument("code") + ix.add_argument("--count", type=int, default=30) + ix.set_defaults(func=lambda td, a: cmd_kline( + td, a.code, KlinePeriod.DAY, a.count, 0, Adjust.NONE)) + + b = add("board", "板块") + b.add_argument("action", choices=["list", "members", "ranking", "change"]) + b.add_argument("symbol", nargs="?", default="") + b.add_argument("--type", type=int, default=1) + b.add_argument("--count", type=int, default=30) + b.add_argument("--top", type=int, default=10) + b.add_argument("--days", type=int, default=20) + b.set_defaults(func=lambda td, a: cmd_board(td, a)) + + i = add("info", "个股特征快照") + i.add_argument("symbol") + i.set_defaults(func=lambda td, a: cmd_info(td, a.symbol)) + + ls = add("list", "证券列表") + ls.add_argument("--market", choices=list(MARKET_NAMES), default="sz") + ls.add_argument("--count", type=int, default=30) + ls.set_defaults(func=lambda td, a: cmd_list(td, a.market, a.count)) + + au = add("auction", "集合竞价") + au.add_argument("symbol") + au.set_defaults(func=lambda td, a: cmd_auction(td, a.symbol)) + + un = add("unusual", "市场异动") + un.add_argument("--market", choices=list(MARKET_NAMES), default="sh") + un.add_argument("--count", type=int, default=30) + un.set_defaults(func=lambda td, a: cmd_unusual(td, a.market, a.count)) + + xd = add("xdxr", "除权除息") + xd.add_argument("symbol") + xd.set_defaults(func=lambda td, a: cmd_xdxr(td, a.symbol)) + + fn = add("finance", "财务快照") + fn.add_argument("symbol") + fn.set_defaults(func=lambda td, a: cmd_finance(td, a.symbol)) + + fl = add("flow", "资金流向") + fl.add_argument("symbol") + fl.set_defaults(func=lambda td, a: cmd_flow(td, a.symbol)) + + bk = add("block", "板块文件 下载+解析 (block_*.dat)") + bk.add_argument("filename", nargs="?", default="block_gn.dat", + help="板块文件名(默认 block_gn.dat;常用 block_zs.dat 行业 / block_fg.dat 风格)") + bk.add_argument("--limit", type=int, default=30, help="最多显示板块数(0=全部)") + bk.add_argument("--codes", action="store_true", help="显示每个板块完整成分股代码") + bk.set_defaults(func=lambda td, a: cmd_block(td, a.filename, a.limit, a.codes)) + + ind = add("industry", "行业分类 下载+解析 (tdxhy.cfg)") + ind.add_argument("codes", nargs="*", help="查询指定 6 位代码(缺省列出全部)") + ind.add_argument("--limit", type=int, default=30, help="最多显示条数(0=全部)") + ind.add_argument("--search", default=None, help="按行业名关键词过滤") + ind.set_defaults(func=lambda td, a: cmd_industry(td, a.codes, a.limit, a.search)) + + ff = add("financial-list", "历史专业财报文件索引 下载+解析 (gpcw.txt)") + ff.add_argument("--limit", type=int, default=30, help="最多显示条数(0=全部)") + ff.set_defaults(func=lambda td, a: cmd_financial_list(td, a.limit)) + + fr = add("financial", "历史专业财报记录 下载+解析 (gpcw*.zip)") + fr.add_argument("filename", help="如 gpcw20260331.zip(用 financial-list 查看可用文件)") + fr.add_argument("--code", default=None, help="只显示指定 6 位股票代码") + fr.add_argument("--limit", type=int, default=5, help="最多显示条数(0=全部)") + fr.add_argument("--fields", type=int, default=4, help="每条显示前 N 个原始字段(0=不显示)") + fr.set_defaults(func=lambda td, a: cmd_financial(td, a.filename, a.code, a.limit, a.fields)) + + si = add("server-info", "服务器交易时段") + si.set_defaults(func=_server_info) + + ms = add("market-stat", "市场统计") + ms.set_defaults(func=_market_stat) + + args = ap.parse_args() + + async def run() -> None: + td = TdxData(cache_dir=None, pool_min=2, pool_max=6) + await td.start() + try: + await args.func(td, args) + finally: + await td.close() + + try: + asyncio.run(run()) + except ValueError as e: + print(e) + sys.exit(1) + + +async def _server_info(td: TdxData, args) -> None: + await _print_obj(await td.get_server_info()) + + +async def _market_stat(td: TdxData, args) -> None: + await _print_obj(await td.get_market_stat()) + + +async def _print_obj(obj) -> None: + if isinstance(obj, dict): + for k, v in obj.items(): + print(" %-24s %s" % (k, v)) + elif hasattr(obj, "_fields"): + for f in obj._fields: + print(" %-24s %s" % (f, getattr(obj, f))) + elif hasattr(obj, "__dataclass_fields__"): + for k, v in asdict(obj).items(): + print(" %-24s %s" % (k, v)) + elif hasattr(obj, "__slots__"): + for k in obj.__slots__: + print(" %-24s %s" % (k, getattr(obj, k))) + else: + print(obj) + + +if __name__ == "__main__": + main() diff --git a/src/pytdxdata/codec/__init__.py b/src/pytdxdata/codec/__init__.py new file mode 100644 index 0000000..225d865 --- /dev/null +++ b/src/pytdxdata/codec/__init__.py @@ -0,0 +1,20 @@ +"""服务器文件解析器:板块(.dat) / 行业(.cfg) / 历史专业财报(.zip+.dat)。 + +均为纯标准库实现(struct / zipfile),无第三方依赖,可独立单测。 +""" +from __future__ import annotations + +from .block import infer_block_category, parse_block_dat +from .financial import parse_financial_dat, parse_financial_file_list +from .industry import parse_tdxhy_cfg +from .payload import pack_payload, unpack_payload + +__all__ = [ + "infer_block_category", + "pack_payload", + "parse_block_dat", + "parse_financial_dat", + "parse_financial_file_list", + "parse_tdxhy_cfg", + "unpack_payload", +] diff --git a/src/pytdxdata/codec/block.py b/src/pytdxdata/codec/block.py new file mode 100644 index 0000000..1be59fd --- /dev/null +++ b/src/pytdxdata/codec/block.py @@ -0,0 +1,73 @@ +"""板块文件 (.dat) 解析:通达信 block_zs/gn/fg.dat -> TdxBlock 列表。 + +字节格式(与 easy-tdx / pytdx 一致):: + + Header : 384 字节(跳过,含版本/日期等未知字段) + Count : 2 字节 uint16 LE —— 板块条数 + Body : count × 2813 字节 + 每条 = 9s 板块名(GBK) + + H 成分股数量 + + H 类型 + + 2800s 代码区(每只股票占 7 字节 ASCII) +""" +from __future__ import annotations + +import struct + +from ..models import TdxBlock + +_HEADER_SIZE = 384 +_RECORD_SIZE = 2813 +_CODE_SLOT = 7 +_MAX_CODES = 2800 // _CODE_SLOT # 单条记录最多 400 只 + + +def infer_block_category(filename: str) -> int: + """按文件名推断板块分类:zs=行业(0) / gn=概念(2) / fg=风格(3)。""" + name = filename.lower() + if "gn" in name: + return 2 + if "fg" in name: + return 3 + return 0 + + +def parse_block_dat(data: bytes, filename: str = "") -> list[TdxBlock]: + """解析通达信 .dat 板块文件字节内容。 + + Args: + data: 文件完整字节。 + filename: 用于推断 category(如 ``block_gn.dat``)。 + + Returns: + TdxBlock 列表;数据不足或为空时返回空列表。 + """ + if len(data) < _HEADER_SIZE + 2: + return [] + + (count,) = struct.unpack_from(" len(data): + break + + name = data[pos : pos + 9].decode("gbk", errors="replace").strip("\x00").strip() + (stock_count,) = struct.unpack_from(" list[FinancialFileInfo]: + """解析 tdxfin/gpcw.txt 内容(每行 ``filename,md5hash,filesize``)。""" + if not data: + return [] + text = data.decode("utf-8", errors="replace").strip() + out: list[FinancialFileInfo] = [] + for line in text.splitlines(): + line = line.strip() + if not line: + continue + parts = line.split(",") + if len(parts) < 3: + continue + try: + filesize = int(parts[2]) + except ValueError: + continue + out.append(FinancialFileInfo(filename=parts[0], hash=parts[1], filesize=filesize)) + return out + + +def parse_financial_dat(data: bytes, report_date: int = 0) -> list[FinancialRecord]: + """解析 gpcw*.zip 内的 .dat 二进制。 + + Args: + data: .dat 文件完整字节。 + report_date: 报告期 YYYYMMDD;传 0 时使用 header 内嵌的报告期。 + + Returns: + FinancialRecord 列表;数据不足或字段非法时返回空列表。 + """ + header_size = struct.calcsize(_HEADER_FMT) + if len(data) < header_size: + return [] + + header = struct.unpack(_HEADER_FMT, data[:header_size]) + dat_report_date = header[1] + max_count = header[2] + report_size = header[4] + + if report_date == 0: + report_date = dat_report_date + + num_fields = report_size // 4 + if num_fields <= 0: + return [] + + index_size = struct.calcsize(_INDEX_FMT) + report_fmt = f"<{num_fields}f" + report_pack_size = struct.calcsize(report_fmt) + + out: list[FinancialRecord] = [] + for i in range(max_count): + idx_pos = header_size + i * index_size + if idx_pos + index_size > len(data): + break + + code_bytes, market, file_offset = struct.unpack( + _INDEX_FMT, data[idx_pos : idx_pos + index_size] + ) + code = code_bytes.decode("ascii", errors="replace").rstrip("\x00") + if not code or file_offset == 0: + continue + if file_offset + report_pack_size > len(data): + continue + + fields = list( + struct.unpack(report_fmt, data[file_offset : file_offset + report_pack_size]) + ) + out.append( + FinancialRecord( + code=code, market=market, report_date=report_date, fields=fields + ) + ) + return out diff --git a/src/pytdxdata/codec/industry.py b/src/pytdxdata/codec/industry.py new file mode 100644 index 0000000..36c4760 --- /dev/null +++ b/src/pytdxdata/codec/industry.py @@ -0,0 +1,40 @@ +"""通达信行业分类配置 (tdxhy.cfg) 解析。 + +文件为 GBK 文本,每行形如:: + + 市场|代码|通达信行业|||申万行业|... + +只保留 6 位 A 股代码;通达信行业取第 3 列,申万行业取第 6 列(可能缺失)。 +""" +from __future__ import annotations + +from ..models import IndustryInfo + + +def parse_tdxhy_cfg(content: bytes) -> dict[str, IndustryInfo]: + """解析 tdxhy.cfg 字节内容。 + + Args: + content: 文件完整字节(GBK 编码)。 + + Returns: + ``{6位代码: IndustryInfo}``;空输入或非法行会被跳过。 + """ + results: dict[str, IndustryInfo] = {} + if not content: + return results + + text = content.decode("gbk", errors="replace") + for line in text.splitlines(): + parts = line.strip().split("|") + if len(parts) < 3: + continue + code = parts[1].strip() + if len(code) != 6 or not code.isdigit(): + continue + results[code] = IndustryInfo( + code=code, + tdx_industry=parts[2].strip(), + sw_industry=parts[5].strip() if len(parts) >= 6 else "", + ) + return results diff --git a/src/pytdxdata/codec/payload.py b/src/pytdxdata/codec/payload.py new file mode 100644 index 0000000..df18eed --- /dev/null +++ b/src/pytdxdata/codec/payload.py @@ -0,0 +1,16 @@ +"""缓存载荷序列化:records <-> bytes(pickle + zlib level1,性能优先)""" +from __future__ import annotations + +import pickle +import zlib +from typing import Any + + +def pack_payload(obj: Any) -> bytes: + """序列化缓存载荷(压缩)。""" + return zlib.compress(pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL), level=1) + + +def unpack_payload(data: bytes) -> Any: + """反序列化缓存载荷。""" + return pickle.loads(zlib.decompress(data)) diff --git a/src/pytdxdata/config.py b/src/pytdxdata/config.py new file mode 100644 index 0000000..02415d5 --- /dev/null +++ b/src/pytdxdata/config.py @@ -0,0 +1,127 @@ +"""服务器清单与默认配置 + +服务器清单存于包内 servers.json,由 server_probe.py 导入时自动探测可用性与延时。 +config.py 加载时只采用 status="ok" 的服务器;环境变量可覆盖;代码内嵌清单为兜底。 +""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +DEFAULT_PORT = 7709 +DEFAULT_TIMEOUT = 10.0 + +_SERVERS_FILE = Path(__file__).resolve().parent / "servers.json" # 包内清单(随包分发) + + +def _load_ok_hosts(protocol: str, *, fast_only: bool = False) -> list[str]: + """从 servers.json 加载 status=ok 且支持指定协议的服务器。 + + fast_only=True 时只返回当日测速入池(fast_pool)的服务器(排名前N或延迟<阈值), + 用于动态连接池优先使用优质节点。 + """ + try: + cfg = json.loads(_SERVERS_FILE.read_text(encoding="utf-8")) + except (OSError, ValueError): + return [] + out = [] + for s in cfg.get("servers", []): + if s.get("status") != "ok": + continue + if protocol not in s.get("protocols", []): + continue + if fast_only and not s.get("fast_pool", False): + continue + out.append(s["ip"]) + return out + + +def _load_ex_handshakes() -> dict[str, str]: + try: + cfg = json.loads(_SERVERS_FILE.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + return {s["ip"]: s.get("handshake", "setup") for s in cfg.get("servers", []) + if "ex" in s.get("protocols", []) and s.get("status") == "ok"} + +# 标准行情服务器 —— 真实K线探测验证可用(2026-08-12, GetSecurityBarsCmd 600000 日K) +# 27台可用 = pytdxwrapper券商服务器(国泰君安/中信短线宝/华林/安信等) + 通达信官方主站 +# 来源①: pytdxwrapper config.py TDX_HOSTS(券商+官方, 51台探测27可用) +# 来源②: easy-tdx known_hosts + connect.cfg官方主站(54台探测仅5可用, 已被券商清单覆盖) +# 另2台端口80可用(池统一7709暂不收录): 180.153.18.172 / 202.108.253.139 +# 不可用特征: K线空响应(握手正常无数据, 仅ping测不出) —— 探测脚本: examples/probe_pytdxwrapper.py / probe_all_hosts.py +STANDARD_HOSTS: list[str] = [ + "139.9.52.158", "103.251.85.94", "58.67.221.146", "59.36.5.11", + "103.221.142.82", "218.106.92.182", "60.12.136.251", "218.106.92.183", + "60.12.136.250", "115.238.90.170", "220.178.55.86", "220.178.55.71", + "180.153.18.170", "115.238.90.165", "60.191.117.167", "218.75.126.9", + "115.238.56.198", "117.34.114.16", "159.75.55.232", "117.34.114.17", + "117.34.114.14", "117.34.114.15", "117.34.114.13", "117.34.114.27", + "117.34.114.18", "117.34.114.20", "117.34.114.30", +] + +# MAC 服务器(独立主机池, 2026-08-12 全量复测: 24台→23台可用, 剔除117.34.114.17空响应) +# MAC专用(3): 121.37.207.165 / 123.60.47.136 / 121.36.248.138 (对标准命令返回空, 池探测用MacKlineCmd) +# 标准服务器MAC支持20台: 券商(国泰君安/中信/安信)与官方主站, 与STANDARD_HOSTS同端口7709 +# 标准服务器不支持MAC的6台: 218.106.92.182/183 220.178.55.71/86(华林) 159.75.55.232 117.34.114.30 +# 探测脚本: examples/probe_mac_all.py / probe_mac_support.py +MAC_HOSTS: list[str] = [ + "121.37.207.165", "139.9.52.158", "58.67.221.146", "59.36.5.11", + "60.12.136.250", "60.12.136.251", "121.36.248.138", "60.191.117.167", + "103.221.142.82", "218.75.126.9", "123.60.47.136", "180.153.18.170", + "115.238.56.198", "115.238.90.170", "115.238.90.165", "103.251.85.94", + "117.34.114.15", "117.34.114.18", "117.34.114.14", "117.34.114.20", + "117.34.114.13", "117.34.114.27", "117.34.114.16", +] + +# EX扩展行情服务器(港股/期货/美股,端口7727) +# MAC EX专用2台(0x2454 login 80B): 116.205.135.205 / 121.37.232.167 +# pytdx EX协议2台(setup 0x2454 92B): 通达信扩展市场116.205.143.214(52市场) + 国泰君安103.221.142.82(60市场) +# 发现方式(2026-08-12): 券商服务器同机多协议——103.221.142.82同时是MAC服务器(7709)与EX服务器(7727) +EX_HOSTS: list[str] = [ + "116.205.135.205", "121.37.232.167", "116.205.143.214", "103.221.142.82", +] +# 各服务器握手模式: login(0x2454 login) / setup(0x2454 setup) +EX_HANDSHAKES: dict[str, str] = { + "116.205.135.205": "login", + "121.37.232.167": "login", + "116.205.143.214": "setup", + "103.221.142.82": "setup", +} + + +def get_standard_hosts() -> list[str]: + env = os.environ.get("PYPYTDX_DATA_STANDARD_HOSTS") + if env: + return [h.strip() for h in env.split(",") if h.strip()] + fast = _load_ok_hosts("standard", fast_only=True) + if fast: + return fast + from_json = _load_ok_hosts("standard") + return from_json or list(STANDARD_HOSTS) + + +def get_mac_hosts() -> list[str]: + env = os.environ.get("PYPYTDX_DATA_MAC_HOSTS") + if env: + return [h.strip() for h in env.split(",") if h.strip()] + fast = _load_ok_hosts("mac", fast_only=True) + if fast: + return fast + from_json = _load_ok_hosts("mac") + return from_json or list(MAC_HOSTS) + + +def get_ex_hosts() -> list[str]: + env = os.environ.get("PYPYTDX_DATA_EX_HOSTS") + if env: + return [h.strip() for h in env.split(",") if h.strip()] + from_json = _load_ok_hosts("ex") + return from_json or list(EX_HOSTS) + + +def get_ex_handshakes() -> dict[str, str]: + """EX服务器握手模式(json优先,代码兜底)。""" + from_json = _load_ex_handshakes() + return from_json or dict(EX_HANDSHAKES) diff --git a/src/pytdxdata/indicator.py b/src/pytdxdata/indicator.py new file mode 100644 index 0000000..d5179d4 --- /dev/null +++ b/src/pytdxdata/indicator.py @@ -0,0 +1,136 @@ +"""轻量技术指标(纯Python,零依赖):MACD/KDJ/RSI/BOLL/MA/EMA""" +from __future__ import annotations + +from typing import Callable + + +def _ema(values: list[float], period: int) -> list[float]: + """EMA。""" + if not values: + return [] + k = 2.0 / (period + 1) + out = [values[0]] + for v in values[1:]: + out.append(v * k + out[-1] * (1 - k)) + return out + + +def _sma(values: list[float], period: int) -> list[float]: + """简单移动平均(尾部对齐,前period-1个为NaN用None)。""" + out: list[float | None] = [] + s = 0.0 + for i, v in enumerate(values): + s += v + if i >= period: + s -= values[i - period] + out.append(s / period if i >= period - 1 else None) + return out + + +def ma(values: list[float], period: int) -> list[float | None]: + return _sma(values, period) + + +def ema(values: list[float], period: int) -> list[float]: + return _ema(values, period) + + +def macd(values: list[float], fast: int = 12, slow: int = 26, signal: int = 9) -> dict: + """MACD: DIF/DEA/MACD柱。返回与values等长。""" + ema_fast = _ema(values, fast) + ema_slow = _ema(values, slow) + dif = [f - s for f, s in zip(ema_fast, ema_slow)] + dea = _ema(dif, signal) + hist = [(d - e) * 2 for d, e in zip(dif, dea)] + return {"DIF": dif, "DEA": dea, "MACD": hist} + + +def rsi(values: list[float], period: int = 14) -> list[float | None]: + """RSI。""" + out: list[float | None] = [None] * len(values) + if len(values) <= period: + return out + gains = losses = 0.0 + for i in range(1, period + 1): + chg = values[i] - values[i - 1] + if chg > 0: + gains += chg + else: + losses -= chg + for i in range(period, len(values)): + if i > period: + chg = values[i] - values[i - 1] + gains = (gains * (period - 1) + max(chg, 0)) / period + losses = (losses * (period - 1) + max(-chg, 0)) / period + out[i] = 100.0 if losses == 0 else 100.0 - 100.0 / (1 + gains / losses) if losses else 100.0 + if losses == 0 and gains == 0: + out[i] = 50.0 + return out + + +def kdj(values_high: list[float], values_low: list[float], + values_close: list[float], period: int = 9) -> dict: + """KDJ(简化:9日随机指标)。""" + n = len(values_close) + k = [50.0] * n + d = [50.0] * n + j = [50.0] * n + for i in range(n): + lo = min(values_low[max(0, i - period + 1): i + 1]) + hi = max(values_high[max(0, i - period + 1): i + 1]) + rsv = 50.0 if hi == lo else (values_close[i] - lo) / (hi - lo) * 100 + k[i] = k[i - 1] * 2 / 3 + rsv / 3 + d[i] = d[i - 1] * 2 / 3 + k[i] / 3 + j[i] = 3 * k[i] - 2 * d[i] + return {"K": k, "D": d, "J": j} + + +def boll(values: list[float], period: int = 20, mult: float = 2.0) -> dict: + """BOLL布林带。""" + import statistics + n = len(values) + mid = [None] * n + upper = [None] * n + lower = [None] * n + for i in range(period - 1, n): + window = values[i - period + 1: i + 1] + m = sum(window) / period + sd = statistics.pstdev(window) + mid[i] = m + upper[i] = m + mult * sd + lower[i] = m - mult * sd + return {"MID": mid, "UPPER": upper, "LOWER": lower} + + +_INDICATORS: dict[str, Callable] = { + "MA": lambda c, **kw: {"MA": ma(c, kw.get("period", 5))}, + "EMA": lambda c, **kw: {"EMA": ema(c, kw.get("period", 12))}, + "MACD": macd, + "RSI": lambda c, **kw: {"RSI": rsi(c, kw.get("period", 14))}, + "KDJ": lambda h, l, c, **kw: kdj(h, l, c, kw.get("period", 9)), + "BOLL": lambda c, **kw: boll(c, kw.get("period", 20), kw.get("mult", 2.0)), +} + + +def compute_indicators(high: list[float], low: list[float], close: list[float], + names: list[str], params: dict | None = None, + tail: int | None = None) -> dict: + """计算指标,返回 {指标名: 序列}。""" + params = params or {} + result: dict = {} + for name in names: + name_u = name.upper() + fn = _INDICATORS.get(name_u) + if fn is None: + continue + kw = params.get(name, {}) or {} + try: + if name_u == "KDJ": + r = fn(high, low, close, **kw) + else: + r = fn(close, **kw) + except Exception: + continue + for k, v in r.items(): + result[k] = v[-tail:] if tail else v + return result diff --git a/src/pytdxdata/models/__init__.py b/src/pytdxdata/models/__init__.py new file mode 100644 index 0000000..03c0e66 --- /dev/null +++ b/src/pytdxdata/models/__init__.py @@ -0,0 +1,407 @@ +"""数据模型:K线/报价/列表/逐笔/分时/财务(全部dataclass,性能优先不依赖pandas)""" +from __future__ import annotations + +import struct +from dataclasses import dataclass, field +from datetime import datetime +from enum import IntEnum +from typing import Any + + + + +class ExMarket(IntEnum): + """扩展市场代码(港股/期货/期权,通达信EX通道)""" + TEMP_STOCK = 1 + ZZ_FUTURES_OPTION = 4 + DL_FUTURES_OPTION = 5 + SH_FUTURES_OPTION = 6 + CFFEX_OPTION = 7 + SH_STOCK_OPTION = 8 + SZ_STOCK_OPTION = 9 + BASIC_FX = 10 + CROSS_FX = 11 + INTL_INDEX = 12 + COMEX_FUTURES = 16 + NYMEX_FUTURES = 17 + CBOT_FUTURES = 18 + HK_FINANCIAL_FUTURES = 23 + HK_FINANCIAL_OPTIONS = 24 + HK_STOCK_FUTURES = 25 + HK_STOCK_OPTIONS = 26 + HK_INDEX = 27 + ZZ_FUTURES = 28 + DL_FUTURES = 29 + SH_FUTURES = 30 + HK_MAIN_BOARD = 31 + OPEN_END_FUND = 33 + MONETARY_FUND = 34 + MACRO_INDICATOR = 38 + FUTURES_INDEX = 42 + B_TO_H = 43 + NEEQ = 44 + SH_GOLD = 46 + CFFEX_FUTURES = 47 + HK_GEM = 48 + HK_FUND = 49 + SUNSHINE_PRIVATE_FUND = 56 + MAIN_FUTURES_CONTRACT = 60 + CSI_INDEX = 62 + GZ_ARBITRAGE_FUTURES = 65 + GZ_FUTURES = 66 + GZ_OPTIONS = 67 + EXTENDED_SECTOR_INDEX = 70 + HK_STOCK_GGT = 71 + GE_STOCK = 73 + US_STOCK = 74 + SG_STOCK = 78 + HK_DARK_POOL = 98 + CODE_MIRROR = 100 + SZSE_INDEX = 102 +class Market(IntEnum): + SZ = 0 + SH = 1 + BJ = 2 + + +class KlinePeriod(IntEnum): + """通达信K线周期编码(KlineCategory,与 easy-tdx 一致)""" + MIN_5 = 0 + MIN_15 = 1 + MIN_30 = 2 + MIN_60 = 3 + DAY = 4 + WEEK = 5 + MONTH = 6 + MIN_1 = 7 + MIN_3 = 8 + YEAR = 9 + SEASON = 10 + YEAR_ALT = 11 + + @property + def is_minute(self) -> bool: + return self in (KlinePeriod.MIN_1, KlinePeriod.MIN_5, KlinePeriod.MIN_15, + KlinePeriod.MIN_30, KlinePeriod.MIN_60) + + +class Adjust(IntEnum): + """复权方式(MAC通道)""" + NONE = 0 + QFQ = 1 # 前复权 + HFQ = 2 # 后复权 + + +@dataclass(slots=True) +class SecurityBar: + """K线记录(价格单位:元)""" + market: int + code: str + open: float + high: float + low: float + close: float + vol: float # 股票=股 / 转债=手(与vipdoc一致) + amount: float # 元 + year: int + month: int + day: int + hour: int = 0 + minute: int = 0 + + @property + def datetime(self) -> datetime: + return datetime(self.year, self.month, self.day, self.hour, self.minute) + + +@dataclass(slots=True) +class IndexBar(SecurityBar): + """指数K线(多涨跌家数字段,结构同K线仅解析时跳过4字节)""" + pass + + +@dataclass(slots=True) +class SecurityQuote: + """实时五档报价""" + market: int + code: str + price: float + pre_close: float + open: float + high: float + low: float + vol: float + amount: float + s_vol: float # 主动卖 + b_vol: float # 主动买 + bid: list[tuple[float, float]] = field(default_factory=list) # [(价,量)x5] + ask: list[tuple[float, float]] = field(default_factory=list) # [(价,量)x5] + server_time: str = "" + trading_status: int = 0 + cur_vol: float = 0.0 # 当前成交量(手) + rise_speed: float = 0.0 # 涨速(%) + limit_up: float = 0.0 # 涨停价(本地计算, 未填为0) + limit_down: float = 0.0 # 跌停价 + decimal_point: int = 2 # 价格小数位 + open_amount: float = 0.0 # 开盘金额(个股) + + +@dataclass(slots=True) +class SecurityInfo: + """证券列表条目""" + market: int + code: str + name: str + volunit: int + decimal_point: int + pre_close: float + + +@dataclass(slots=True) +class TransactionRecord: + """逐笔成交(MAC通道含 trade_count=成交笔数)""" + market: int + code: str + time: str # HH:MM:SS + price: float + vol: float # 手(MAC逐笔) + trade_count: int # 成交笔数(MAC=逐笔记录字段;标准当日逐笔协议也有但需单独解析) + bs_flag: int # 1=主动买 0=主动卖 + + +@dataclass(slots=True) +class MinuteBar: + """分时数据""" + price: float + vol: float + avg_price: float = 0.0 # 均价(协议第2字段) + hour: int = 0 + minute: int = 0 + + +@dataclass(slots=True) +class XdxrRecord: + """除权除息记录""" + market: int + code: str + year: int + month: int + day: int + category: int + name: str + fenhong: float = 0.0 # 每股分红 + peigujia: float = 0.0 + songzhuangu: float = 0.0 # 每股送转 + peigu: float = 0.0 + suogu: float = 0.0 + xingquanjia: float = 0.0 + fenshu: float = 0.0 + # 股本变动类(category 2~10): 前后流通/总股本(万股) + panqian_liutong: float = 0.0 + qian_zongguben: float = 0.0 + panhou_liutong: float = 0.0 + hou_zongguben: float = 0.0 + + @property + def date(self) -> str: + """日期字符串 YYYY-MM-DD。""" + return "%04d-%02d-%02d" % (self.year, self.month, self.day) + + +@dataclass(slots=True) +class FinanceRecord: + """财务快照(股本/金额单位:万股/万元)""" + market: int + code: str + liutong_guben: float = 0.0 + zong_guben: float = 0.0 + guojia_gu: float = 0.0 + faqiren_faren_gu: float = 0.0 + faren_gu: float = 0.0 + b_gu: float = 0.0 + h_gu: float = 0.0 + zhigong_gu: float = 0.0 + province: int = 0 + industry: int = 0 + updated_date: int = 0 + ipo_date: int = 0 + gudong_renshu: float = 0.0 + zong_zichan: float = 0.0 + liudong_zichan: float = 0.0 + guding_zichan: float = 0.0 + wuxing_zichan: float = 0.0 + liudong_fuzhai: float = 0.0 + changqi_fuzhai: float = 0.0 + ziben_gongjijin: float = 0.0 + jing_zichan: float = 0.0 + zhuying_shouru: float = 0.0 + zhuying_lirun: float = 0.0 + yingshou_zhangkuan: float = 0.0 + yingye_lirun: float = 0.0 + touzi_shouyu: float = 0.0 + jingying_xianjinliu: float = 0.0 + zong_xianjinliu: float = 0.0 + cunhuo: float = 0.0 + lirun_zonghe: float = 0.0 + shuihou_lirun: float = 0.0 + jing_lirun: float = 0.0 + weifen_lirun: float = 0.0 + meigujing_zichan: float = 0.0 + reserve2: float = 0.0 + + +# ===== MAC 扩展命令模型 ===== + +@dataclass(slots=True) +class BoardInfo: + """板块列表条目(含领涨股信息)""" + market: int + code: str + name: str + price: float + rise_speed: float + pre_close: float + # 领涨股(160B记录的后80B) + symbol_market: int = 0 + symbol_code: str = "" + symbol_name: str = "" + symbol_price: float = 0.0 + symbol_rise_speed: float = 0.0 + symbol_pre_close: float = 0.0 + + +@dataclass(slots=True) +class MemberQuote: + """板块成分股报价(MAC自定义字段)""" + market: int + code: str + name: str + fields: dict[str, float] = field(default_factory=dict) + + +@dataclass(slots=True) +class BelongBoard: + """个股所属板块""" + market: int + board_code: str + board_name: str + close: float + pre_close: float + + +@dataclass(slots=True) +class AuctionItem: + """集合竞价""" + time: str + price: float + matched: int + unmatched: int + + +@dataclass(slots=True) +class UnusualItem: + """市场异动""" + market: int + code: str + name: str + type: int + desc: str + value: float + time: str + + +@dataclass(slots=True) +class SymbolSnapshot: + """个股特征快照""" + market: int + code: str + name: str + date: int + time: int + activity: int + pre_close: float + open: float + high: float + low: float + close: float + vol: float + amount: float + turnover: float + avg_price: float + + +@dataclass(slots=True) +class GoodsItem: + """扩展市场商品(期货/期权)""" + category: int + name: str + code: float + switch: int + index: int + + +@dataclass(slots=True) +class ServerInfo: + """服务器交易时段信息""" + date: int + last_trade_date: int + sessions: list[tuple[int, int]] = field(default_factory=list) + + +@dataclass(slots=True) +class CapitalFlow: + """资金流向""" + market: int + code: str + main_buy: float + main_sell: float + retail_buy: float + retail_sell: float + buy5: float + sell5: float + big5: float + mid5: float + + +# ===== 服务器文件解析模型(板块/行业/历史专业财报) ===== + +@dataclass(slots=True) +class TdxBlock: + """通达信板块条目(解析自 block_zs/gn/fg.dat)。 + + category: 0=行业 1=地域 2=概念 3=风格(按文件名推断) + count: 文件内声明的成分股数量(可能大于 codes 实际解析数) + """ + name: str + category: int + count: int + codes: list[str] = field(default_factory=list) + + +@dataclass(slots=True) +class IndustryInfo: + """个股行业分类(解析自 tdxhy.cfg)""" + code: str + tdx_industry: str # 通达信行业代码 + sw_industry: str = "" # 申万行业代码(部分文件缺失) + + +@dataclass(slots=True) +class FinancialFileInfo: + """历史专业财报文件索引条目(解析自 tdxfin/gpcw.txt)""" + filename: str # 如 gpcw20260331.zip + hash: str # MD5 + filesize: int # 字节 + + +@dataclass(slots=True) +class FinancialRecord: + """历史专业财报记录(解析自 gpcw*.zip 内 .dat)。 + + 注意与 FinanceRecord(最新财务快照)区分:本类保留协议原始 float 字段, + 字段含义随 report_size 变化,需结合 gpcw.txt 字段字典解释。 + """ + code: str + market: int # 0=深 1=沪 + report_date: int # 报告期 YYYYMMDD + fields: list[float] = field(default_factory=list) diff --git a/src/pytdxdata/pool/__init__.py b/src/pytdxdata/pool/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pytdxdata/pool/connection.py b/src/pytdxdata/pool/connection.py new file mode 100644 index 0000000..7fa9dd4 --- /dev/null +++ b/src/pytdxdata/pool/connection.py @@ -0,0 +1,137 @@ +"""单条 asyncio TCP 连接:连接 + 3条握手 + 帧收发 + 命令执行""" +from __future__ import annotations + +import asyncio +import time +from typing import Generic, TypeVar + +from ..protocol.commands.base import BaseCommand +from ..protocol.commands.setup import SETUP_COMMANDS +from ..protocol.frame import HEADER_SIZE, decompress_body, parse_header + +T = TypeVar("T") + + +class TdxConnection(Generic[T]): + """一条通达信 TCP 连接(连接时自动握手,execute 发送命令并解析响应)。""" + + def __init__(self, host: str, port: int = 7709, timeout: float = 10.0) -> None: + self.host = host + self.port = port + self.timeout = timeout + self._reader: asyncio.StreamReader | None = None + self._writer: asyncio.StreamWriter | None = None + self._last_result: T | None = None + + @property + def alive(self) -> bool: + return self._writer is not None and not self._writer.is_closing() + + async def connect(self) -> None: + self._reader, self._writer = await asyncio.wait_for( + asyncio.open_connection(self.host, self.port), timeout=self.timeout + ) + await self._send_setup() + + async def close(self) -> None: + if self._writer is not None: + try: + self._writer.close() + await self._writer.wait_closed() + except (ConnectionError, OSError): + pass + self._writer = None + self._reader = None + + def _force_close(self) -> None: + """强制中断底层连接(abort transport),用于超时/错误后让挂起的 read 立即失效。""" + writer = self._writer + self._writer = None + self._reader = None + if writer is not None: + try: + transport = writer.transport + transport.abort() # 立即 RST,解除任何挂起的 read(),避免协程无法被取消 + except (ConnectionError, OSError, AttributeError): + try: + writer.close() + except (ConnectionError, OSError): + pass + + async def ping(self) -> float | None: + """连接+握手+发一条setup命令测延迟(秒)。失败返回 None。""" + t0 = time.monotonic() + try: + await self.connect() + assert self._writer is not None + self._writer.write(SETUP_COMMANDS[0]) + await self._writer.drain() + hdr_buf = await self._recv_exact(HEADER_SIZE) + hdr = parse_header(hdr_buf) + if hdr.zipsize > 0: + await self._recv_exact(hdr.zipsize) + return time.monotonic() - t0 + except (ConnectionError, OSError, asyncio.TimeoutError, ValueError): + return None + finally: + await self.close() + + async def execute(self, cmd: BaseCommand[T]) -> T: + """发送请求 → 读16B帧头 → 读body → 解压 → 命令解析。""" + if not self.alive: + raise ConnectionError(f"连接未建立或已关闭: {self.host}") + assert self._reader is not None and self._writer is not None + try: + try: + # 用手动 wait_for 包裹 IO,超时后强制关闭底层连接, + # 避免 asyncio.timeout 与外层 wait_for 取消产生竞态导致协程无法中断。 + await asyncio.wait_for(self._io(cmd), self.timeout) + except asyncio.TimeoutError: + # 半开连接/服务器无响应:关闭连接使其真正失效,便于重试时换连接 + self._force_close() + raise ConnectionError(f"通信超时 {self.host} ({self.timeout}s)") from None + except (ConnectionError, OSError) as e: + self._force_close() + raise ConnectionError(f"通信错误 {self.host}: {e}") from e + except (ConnectionError, OSError) as e: + raise e + return self._last_result # type: ignore[return-value] + + async def _io(self, cmd: BaseCommand[T]) -> None: + assert self._reader is not None and self._writer is not None + self._writer.write(cmd.build_request()) + await self._writer.drain() + header_buf = await self._recv_exact(HEADER_SIZE) + header = parse_header(header_buf) + raw_body = await self._recv_exact(header.zipsize) + body = decompress_body(header, raw_body) + self._last_result: T = cmd.parse_response(body) + + async def _send_setup(self) -> None: + """按序发送3条握手命令并丢弃响应。""" + assert self._reader is not None and self._writer is not None + for cmd_bytes in SETUP_COMMANDS: + try: + async with asyncio.timeout(self.timeout): + self._writer.write(cmd_bytes) + await self._writer.drain() + hdr_buf = await self._recv_exact(HEADER_SIZE) + hdr = parse_header(hdr_buf) + if hdr.zipsize > 0: + await self._recv_exact(hdr.zipsize) + except (ConnectionError, OSError, asyncio.TimeoutError): + # 部分服务器握手无响应,忽略 + pass + + async def _recv_exact(self, n: int) -> bytes: + """循环读取直到 n 字节。""" + assert self._reader is not None + buf = bytearray() + while len(buf) < n: + chunk = await asyncio.wait_for( + self._reader.read(n - len(buf)), timeout=self.timeout + ) + if not chunk: + raise ConnectionError(f"连接被服务器关闭: {self.host}") + buf.extend(chunk) + return bytes(buf) diff --git a/src/pytdxdata/pool/ex_connection.py b/src/pytdxdata/pool/ex_connection.py new file mode 100644 index 0000000..cbc9417 --- /dev/null +++ b/src/pytdxdata/pool/ex_connection.py @@ -0,0 +1,103 @@ +"""MAC EX 扩展行情通道:login(0x2454) + 端口7727 + 帧首字节改写""" +from __future__ import annotations + +import asyncio +import struct +import time + +from ..protocol.commands.base import BaseCommand +from ..protocol.frame import HEADER_SIZE, decompress_body, parse_header + +# 80字节 login body(来自 opentdx,easy-tdx 已验证) +_LOGIN_BODY = bytes( + bytearray.fromhex( + "e5bb1c2fafe52594" "1f32c6e5d53dfb41" "5b734cc9cdbf0ac9" + "2021bfdd1eb06d22" "d008884c1611cb13" "78f6abd824d899d2" + "1f32c6e5d53dfb41" "1f32c6e5d53dfb41" "a9325ac935dc0837" + "335a16e4ce17c1bb" + ) +) + + +class MacExLoginCmd(BaseCommand[bool]): + """EX扩展行情登录(0x2454,head_flag=0x01)。""" + + def build_request(self) -> bytes: + inner = struct.pack(" bool: + return len(body) >= 2 # 响应body非空即成功 + + +class ExTdxConnection: + """EX扩展行情连接:TCP(7727) + login + 帧首字节0x1C→0x01改写。 + + 扩展行情服务器要求先login,且请求帧head_flag为0x01(MAC为0x1C)。 + """ + + def __init__(self, host: str, port: int = 7727, timeout: float = 10.0, + handshake: str = "login") -> None: + self.host = host + self.port = port + self.timeout = timeout + self.handshake = handshake # "login"=0x2454 login 80B | "setup"=0x2454 setup 70B + self._reader: asyncio.StreamReader | None = None + self._writer: asyncio.StreamWriter | None = None + self.last_used: float = 0.0 + + @property + def alive(self) -> bool: + return self._writer is not None and not self._writer.is_closing() + + async def connect(self) -> None: + self._reader, self._writer = await asyncio.wait_for( + asyncio.open_connection(self.host, self.port), timeout=self.timeout + ) + if self.handshake == "setup": + from ..protocol.commands.ex_proto import ExSetupCmd + await self.execute(ExSetupCmd()) + else: + await self.execute(MacExLoginCmd()) + + async def close(self) -> None: + if self._writer is not None: + try: + self._writer.close() + await self._writer.wait_closed() + except (ConnectionError, OSError): + pass + self._writer = None + self._reader = None + + async def execute(self, cmd: BaseCommand) -> object: + if not self.alive: + raise ConnectionError(f"EX连接未建立: {self.host}") + assert self._reader is not None and self._writer is not None + try: + request = cmd.build_request() + if request and request[0] == 0x1C: + request = b"\x01" + request[1:] # EX模式首字节改写 + self._writer.write(request) + await self._writer.drain() + self.last_used = time.monotonic() + header_buf = await self._recv_exact(HEADER_SIZE) + header = parse_header(header_buf) + raw_body = await self._recv_exact(header.zipsize) + except (ConnectionError, OSError, asyncio.TimeoutError) as e: + raise ConnectionError(f"EX通信错误 {self.host}: {e}") from e + body = decompress_body(header, raw_body) + return cmd.parse_response(body) + + async def _recv_exact(self, n: int) -> bytes: + assert self._reader is not None + buf = bytearray() + while len(buf) < n: + chunk = await asyncio.wait_for( + self._reader.read(n - len(buf)), timeout=self.timeout + ) + if not chunk: + raise ConnectionError(f"EX连接被关闭: {self.host}") + buf.extend(chunk) + return bytes(buf) diff --git a/src/pytdxdata/pool/ex_pool.py b/src/pytdxdata/pool/ex_pool.py new file mode 100644 index 0000000..8d81049 --- /dev/null +++ b/src/pytdxdata/pool/ex_pool.py @@ -0,0 +1,119 @@ +"""EX扩展市场连接池(简化版:固定连接数,借还+坏连接重建+login自动)""" +from __future__ import annotations + +import asyncio +import logging +import time + +from .ex_connection import ExTdxConnection + +log = logging.getLogger(__name__) + + +class ExPool: + """EX通道连接池(港股/期货,端口7727,连接时自动login)。""" + + def __init__(self, servers: list[str], *, timeout: float = 10.0, + min_size: int = 2, max_size: int = 6, + per_server_cap: int = 1, + handshake: str = "login", + handshake_map: dict | None = None) -> None: + self.servers = servers + self.timeout = timeout + self.handshake = handshake + self.handshake_map = handshake_map or {} + self.min_size = min_size + self.max_size = max_size + self.per_server_cap = max(1, per_server_cap) # 一台服务器最多几个连接(默认1=单连接) + self._idle: list[tuple[str, ExTdxConnection]] = [] + self._server_count: dict[str, int] = {} + self._total = 0 + self._closed = False + self._lock = asyncio.Lock() + + async def start(self) -> None: + async with self._lock: + for host in self.servers[: self.min_size]: + if self._server_count.get(host, 0) >= self.per_server_cap: + continue + try: + conn = ExTdxConnection(host, 7727, self.timeout, self.handshake_map.get(host, self.handshake)) + await conn.connect() + self._idle.append((host, conn)) + self._total += 1 + self._server_count[host] = self._server_count.get(host, 0) + 1 + except Exception as e: + log.warning("EX连接失败 %s: %s", host, str(e)[:60]) + + async def close(self) -> None: + self._closed = True + async with self._lock: + for _, conn in self._idle: + try: + await conn.close() + except Exception: + pass + self._idle.clear() + self._total = 0 + self._server_count.clear() + + async def acquire(self) -> ExTdxConnection: + for _ in range(3): + stale_conn = None + new_conn = None + new_host = None + async with self._lock: + if self._idle: + _, conn = self._idle.pop() + fresh = time.monotonic() - conn.last_used < 30.0 + if conn.alive and fresh: + return conn + # 连接空闲过久(login会话可能失效)→ 丢弃重连 + self._total = max(0, self._total - 1) + self._server_count[conn.host] = max(0, self._server_count.get(conn.host, 0) - 1) + stale_conn = conn + # 扩容:在锁内预留槽位,锁外执行网络IO + elif self._total < self.max_size and self.servers: + host = self._next_available_server() + if host: + self._total += 1 + self._server_count[host] = self._server_count.get(host, 0) + 1 + new_conn = ExTdxConnection(host, 7727, self.timeout, self.handshake_map.get(host, self.handshake)) + new_host = host + # 锁外:关闭过期连接、建立新连接 + if stale_conn is not None: + try: + await stale_conn.close() + except Exception: + pass + if new_conn is not None: + try: + await new_conn.connect() + return new_conn + except Exception as e: + log.warning("EX扩容失败 %s: %s", new_host, str(e)[:60]) + # 连接失败,回滚预留槽位 + async with self._lock: + self._total = max(0, self._total - 1) + self._server_count[new_host] = max(0, self._server_count.get(new_host, 0) - 1) + await asyncio.sleep(0.05) + raise ConnectionError("EX连接池无可用连接") + + def _next_available_server(self) -> str | None: + for h in self.servers: + if self._server_count.get(h, 0) < self.per_server_cap: + return h + return None + + async def release(self, conn: ExTdxConnection, *, broken: bool = False) -> None: + if broken or not conn.alive: + async with self._lock: + self._total = max(0, self._total - 1) + self._server_count[conn.host] = max(0, self._server_count.get(conn.host, 0) - 1) + try: + await conn.close() + except Exception: + pass + return + async with self._lock: + self._idle.append((conn.host, conn)) diff --git a/src/pytdxdata/pool/health.py b/src/pytdxdata/pool/health.py new file mode 100644 index 0000000..8d8e158 --- /dev/null +++ b/src/pytdxdata/pool/health.py @@ -0,0 +1,61 @@ +"""健康分引擎:失败乘性×0.5 / 成功加性+0.2 / 连续失败冷却 / EWMA延迟""" +from __future__ import annotations + +import time +from collections import defaultdict + + +class HealthEngine: + """服务器健康状态管理(语义参照 easy-tdx 已验证实现)。""" + + COOLDOWN_SECONDS = 120.0 + FAIL_PENALTY = 0.5 + SUCCESS_BONUS = 0.2 + FAIL_STREAK_COOLDOWN = 1 # 任意一次通信失败即冷却,避免坏服务器被反复选中 + EWMA_ALPHA = 0.3 + + def __init__(self) -> None: + self._score: dict[str, float] = defaultdict(lambda: 1.0) + self._fail_streak: dict[str, int] = defaultdict(int) + self._cooldown_until: dict[str, float] = {} + self._latency: dict[str, float] = {} # EWMA延迟(秒) + + def record_success(self, server: str, latency: float | None = None) -> None: + self._score[server] = min(1.0, self._score[server] + self.SUCCESS_BONUS) + self._fail_streak[server] = 0 + self._cooldown_until.pop(server, None) + if latency is not None: + self.observe_latency(server, latency) + + def record_failure(self, server: str) -> None: + self._score[server] = max(0.05, self._score[server] * self.FAIL_PENALTY) + self._fail_streak[server] += 1 + if self._fail_streak[server] >= self.FAIL_STREAK_COOLDOWN: + self._cooldown_until[server] = time.monotonic() + self.COOLDOWN_SECONDS + + def observe_latency(self, server: str, latency: float) -> None: + old = self._latency.get(server) + self._latency[server] = ( + latency if old is None else self.EWMA_ALPHA * latency + (1 - self.EWMA_ALPHA) * old + ) + + def score(self, server: str) -> float: + return self._score.get(server, 1.0) + + def latency(self, server: str) -> float: + return self._latency.get(server, float("inf")) + + def in_cooldown(self, server: str) -> bool: + return time.monotonic() < self._cooldown_until.get(server, 0.0) + + def effective_latency(self, server: str) -> float: + """排序键 = 延迟/健康分(越高越差)。""" + return self.latency(server) / self.score(server) + + def rank(self, candidates: list[str]) -> list[str]: + """按 effective_latency 升序排序,冷却中的服务器排最后。""" + ok = [c for c in candidates if not self.in_cooldown(c)] + bad = [c for c in candidates if self.in_cooldown(c)] + ok.sort(key=self.effective_latency) + bad.sort(key=self.effective_latency) + return ok + bad diff --git a/src/pytdxdata/pool/pool.py b/src/pytdxdata/pool/pool.py new file mode 100644 index 0000000..60a2f3c --- /dev/null +++ b/src/pytdxdata/pool/pool.py @@ -0,0 +1,328 @@ +"""异步动态连接池(标准 / MAC 通道 7709)。 + +设计目标: +- 高并发:池内连接数 = 服务器数 × per_server_cap,多服务器并行请求。 +- 自愈:坏连接自动丢弃重建;服务器故障时自动切换到下一个可用服务器。 +- 动态:支持运行时 add() 新服务器;支持后台全量测速入库(当日首次启动)。 +- 单连接约束(核心需求):per_server_cap 默认 1 —— 一台服务器只允许建立 1 个连接, + 不允许多开,避免对单一券商服务器造成并发压力被限流/封禁。 +- 后台测速(核心需求):当日首次启动时,在后台对全量候选服务器测速,按 + 「排名前 FAST_POOL_TOP_N 或 延迟 < FAST_POOL_MAX_LATENCY」筛选入库,结果写回 servers.json。 +- 弹性:空闲连接超时后由 reaper 缩容回 min_size;支持 stats 统计。 +""" +from __future__ import annotations + +import asyncio +import logging +import random +import time + +from .connection import TdxConnection +from .health import HealthEngine +from .probe_cache import get_probe_cache + +log = logging.getLogger(__name__) + +DEFAULT_PORT = 7709 + + +class PoolTimeoutError(Exception): + """连接池在限定重试内无法获取可用连接。""" + + +class _Stats: + def __init__(self) -> None: + self.borrowed = 0 + self.total = 0 + self.created = 0 + self.dropped = 0 + self.reaped = 0 + + +class PooledConnection: + """池化连接包装:记录所属 host,转发 execute / close。 + + server 是 host 的别名(兼容历史调用)。 + """ + + __slots__ = ("conn", "host", "server", "last_used", "alive") + + def __init__(self, conn, host: str) -> None: + self.conn = conn + self.host = host + self.server = host # 别名 + self.last_used = time.monotonic() + self.alive = True + + async def execute(self, cmd): + self.last_used = time.monotonic() + return await self.conn.execute(cmd) + + async def close(self) -> None: + self.alive = False + try: + await self.conn.close() + except Exception: + pass + + +class DynamicPool: + """动态连接池:按服务器维度统计连接数,每台服务器最多 per_server_cap 个连接。""" + + def __init__(self, servers: list[str], *, port: int = DEFAULT_PORT, + timeout: float = 10.0, min_size: int = 4, max_size: int = 12, + per_server_cap: int = 1, + health: HealthEngine | None = None, + probe_factory=None, + idle_ttl: float = 30.0, + scale_down_grace: float = 5.0, + reaper_interval: float = 5.0, + auto_probe: bool = True) -> None: + self.servers = list(dict.fromkeys(servers)) # 去重保序 + self.port = port + self.timeout = timeout + self.min_size = max(1, min_size) + self.max_size = max(self.min_size, max_size) + self.per_server_cap = max(1, per_server_cap) # 一台服务器最多几个连接(默认1=单连接) + self.health = health or HealthEngine() + self.probe_factory = probe_factory # MAC通道用真实命令探测可用性 + self.idle_ttl = idle_ttl + self.scale_down_grace = scale_down_grace + self.reaper_interval = reaper_interval + self.auto_probe = auto_probe + + self.stats = _Stats() + self._idle: list[PooledConnection] = [] + self._conns: set[PooledConnection] = set() + self._server_count: dict[str, int] = {} # 每服务器已建立连接数 + self._total = 0 + self._closed = False + self._started = False + self._lock = asyncio.Lock() + self._cond = asyncio.Condition(self._lock) + self._reaper_task: asyncio.Task | None = None + + # ---------- 生命周期 ---------- + async def start(self) -> None: + if self._started: + return + self._started = True + + # 需求①:当日首次启动 → 后台全量测速入库(不阻塞主流程) + if self.auto_probe: + hosts = list(self.servers) + asyncio.create_task( + get_probe_cache().ensure_probe(hosts), + name="pytdxdata-daily-probe", + ) + + targets = self._pick_servers(self.min_size) + for host in targets: + if self._total >= self.max_size: + break + try: + pc = await self._new_conn(host) + self._idle.append(pc) + self._total += 1 + except Exception as e: + log.warning("初始连接失败 %s: %s", host, str(e)[:60]) + + if self._total == 0: + log.warning("连接池初始连接全部失败,将在使用时按需重试") + + # 启动 reaper:空闲连接超时后缩容回 min_size + try: + self._reaper_task = asyncio.create_task(self._reaper_loop()) + except RuntimeError: + pass # 无事件循环时跳过 + + def _pick_servers(self, n: int) -> list[str]: + ranked = self.health.rank(self.servers) + out: list[str] = [] + counts = dict(self._server_count) + for h in ranked: + if len(out) >= n: + break + if counts.get(h, 0) >= self.per_server_cap: + continue + out.append(h) + counts[h] = counts.get(h, 0) + 1 + if len(out) < n: # 兜底:服务器不足时允许突破 cap + for h in ranked: + if len(out) >= n: + break + out.append(h) + return out + + async def _new_conn(self, host: str) -> PooledConnection: + conn = TdxConnection(host, self.port, self.timeout) + await conn.connect() + if self.probe_factory is not None: + try: + await conn.execute(self.probe_factory()) + except Exception: + await conn.close() + raise + pc = PooledConnection(conn, host) + self._conns.add(pc) + self._server_count[host] = self._server_count.get(host, 0) + 1 + self.stats.created += 1 + self.stats.total = self._total + 1 + return pc + + async def close(self) -> None: + self._closed = True + if self._reaper_task is not None: + self._reaper_task.cancel() + self._reaper_task = None + async with self._lock: + for pc in list(self._conns): + try: + await pc.close() + except Exception: + pass + self._idle.clear() + self._conns.clear() + self._total = 0 + self._server_count.clear() + self.stats.total = 0 + + # ---------- 借还 ---------- + async def acquire(self, timeout: float | None = 30.0) -> PooledConnection: + """借出一条连接。 + + 单连接约束下,池大小 = 服务器数 × per_server_cap。当并发请求超过池大小时, + 超额协程在此**排队等待**(基于 Condition),待其他协程 release 后被唤醒, + 而非立即超时放弃。默认 timeout=30s 防止在坏服务器场景下永久挂起。 + """ + deadline = None if timeout is None else (time.monotonic() + timeout) + async with self._cond: + while True: + now = time.monotonic() + # 1. 优先借用空闲连接(跳过冷却中的服务器,避免反复选中坏节点) + while self._idle: + pc = self._idle.pop(0) + fresh = now - pc.last_used < self.idle_ttl + if pc.alive and fresh and not self.health.in_cooldown(pc.host): + self.stats.borrowed += 1 + return pc + self._drop_locked(pc) + + # 2. 可扩容(未达上限且有可用服务器)→ 新建连接 + if self._total < self.max_size: + host = self._next_available_server() + if host: + try: + pc = await self._new_conn(host) + self._total += 1 + self.stats.borrowed += 1 + return pc + except Exception as e: + log.warning("扩容连接失败 %s: %s", host, str(e)[:60]) + self.health.record_failure(host) + # 循环重试:可能换下一台服务器 + continue + + # 3. 池满:排队等待 release 唤醒(或超时) + if deadline is not None and now >= deadline: + raise PoolTimeoutError("连接池获取超时(连接数已达上限且无空闲)") + try: + if deadline is None: + await self._cond.wait() + else: + await asyncio.wait_for(self._cond.wait(), timeout=deadline - now) + except asyncio.TimeoutError: + raise PoolTimeoutError("连接池获取超时(连接数已达上限且无空闲)") + + async def acquire_many(self, k: int) -> list[PooledConnection]: + """批量借出 k 个连接(分页并发用),超额请求排队等待不丢。""" + out: list[PooledConnection] = [] + for _ in range(k): + try: + out.append(await self.acquire()) + except PoolTimeoutError: + break + return out + + def _next_available_server(self) -> str | None: + ranked = self.health.rank(self.servers) + # 1. 优先选未冷却且未达并发上限的服务器 + for h in ranked: + if self.health.in_cooldown(h): + continue + if self._server_count.get(h, 0) < self.per_server_cap: + return h + # 2. 兜底:仅从「未冷却」的已占用服务器里随机复用(避免反复打坏服务器) + alive = [ + h for h in ranked + if not self.health.in_cooldown(h) and self._server_count.get(h, 0) > 0 + ] + return random.choice(alive) if alive else None + + async def release(self, pc: PooledConnection, *, broken: bool = False) -> None: + if self.stats.borrowed > 0: + self.stats.borrowed -= 1 + # self._cond 的底层锁即 self._lock,统一用 _cond 加锁避免重复加锁死锁 + async with self._cond: + if broken or not pc.alive: + self.health.record_failure(pc.host) + self._drop_locked(pc) + else: + self.health.record_success(pc.host) + if not self._closed and pc.alive: + self._idle.append(pc) + self._cond.notify_all() + + def _drop_locked(self, pc: PooledConnection) -> None: + self._conns.discard(pc) + self._total = max(0, self._total - 1) + self.stats.total = self._total + if self._server_count.get(pc.host, 0) > 0: + self._server_count[pc.host] -= 1 + self._idle = [c for c in self._idle if c is not pc] + self.stats.dropped += 1 + asyncio.create_task(self._safe_close(pc)) + + @staticmethod + async def _safe_close(pc: PooledConnection) -> None: + try: + await pc.close() + except Exception: + pass + + # ---------- reaper(空闲缩容) ---------- + async def _reaper_loop(self) -> None: + try: + while not self._closed: + await asyncio.sleep(self.reaper_interval) + async with self._lock: + now = time.monotonic() + kept = [] + while self._idle: + pc = self._idle.pop(0) + if (now - pc.last_used > self.idle_ttl + and self._total > self.min_size): + self._drop_locked(pc) + self.stats.reaped += 1 + else: + kept.append(pc) + self._idle = kept + except asyncio.CancelledError: + pass + + # ---------- 动态管理 ---------- + def add(self, host: str) -> None: + if host not in self.servers: + self.servers.append(host) + + def remove(self, host: str) -> None: + if host in self.servers: + self.servers.remove(host) + + @property + def size(self) -> int: + return self._total + + @property + def idle_count(self) -> int: + return len(self._idle) diff --git a/src/pytdxdata/pool/probe_cache.py b/src/pytdxdata/pool/probe_cache.py new file mode 100644 index 0000000..5810790 --- /dev/null +++ b/src/pytdxdata/pool/probe_cache.py @@ -0,0 +1,162 @@ +"""当日首次启动的全量服务器测速与入库缓存。 + +设计要点: +- 仅在"当日首次启动"时触发一次后台全量测速(按本地日期判断是否已探测)。 +- 测速采用真实握手连通(TdxConnection.test),比单纯 ping 更能反映通达信可用性。 +- 入库规则:延迟排名前 FAST_POOL_TOP_N(10) **或** 延迟 < FAST_POOL_MAX_LATENCY(0.2s) 的服务器 + 才标记为 status="ok" 并写入 servers.json,其余降权(保留但不优先)。 +- 结果持久化到 servers.json,供 config / 连接池直接使用。 +""" +from __future__ import annotations + +import asyncio +import json +import logging +import time +from datetime import date +from pathlib import Path + +from ..config import _SERVERS_FILE + +log = logging.getLogger(__name__) + +# 入池阈值 +FAST_POOL_TOP_N = 10 # 延迟排名前 N +FAST_POOL_MAX_LATENCY = 0.2 # 延迟 < 200ms(秒)也可入池 +PROBE_PORT = 7709 +PROBE_TIMEOUT = 3.0 +PROBE_CONCURRENCY = 24 + + +class ProbeCache: + """管理当日测速缓存与 servers.json 入库。 + + 一台服务器不论协议(standard/mac/ex),测速结果共享: + 只要任一端口可达即视为该主机可用,并以最快端口延迟作为延迟基准。 + """ + + def __init__(self, servers_file: Path | None = None) -> None: + self._file = servers_file or _SERVERS_FILE + self._lock = asyncio.Lock() + self._last_probe_date: str | None = None + self._probed_today = False + + # ---- 公开 API ---- + def needs_probe_today(self) -> bool: + """是否有必要在今日执行一次全量测速(文件不存在 / 日期不符 / 无 ok 项)。""" + if not self._file.exists(): + return True + try: + cfg = json.loads(self._file.read_text(encoding="utf-8")) + except (OSError, ValueError): + return True + checked = str(cfg.get("checked_date", "")) + today = date.today().isoformat() + if checked != today: + return True + servers = cfg.get("servers", []) + if not any(s.get("status") == "ok" for s in servers): + return True + return False + + async def ensure_probe(self, hosts: list[str]) -> None: + """若今日尚未测速,则后台执行一次全量测速并入库(幂等)。""" + async with self._lock: + if self._probed_today: + return + if not self.needs_probe_today(): + self._probed_today = True + return + self._probed_today = True + + # 在锁外执行(耗时网络 IO),允许并发进入的调用方共享本次结果 + try: + await self._run_full_probe(hosts) + except Exception as e: # 测速失败不能阻塞主流程 + log.warning("后台全量测速失败,沿用既有清单: %s", str(e)[:80]) + + # ---- 内部实现 ---- + async def _run_full_probe(self, hosts: list[str]) -> None: + from .connection import TdxConnection + + log.info("开始当日全量服务器测速 (%d 台)...", len(hosts)) + sem = asyncio.Semaphore(PROBE_CONCURRENCY) + results: dict[str, float | None] = {} + + async def _probe_one(host: str) -> None: + async with sem: + try: + # 使用连接自带的 ping():建连+握手+发1条命令测往返延迟 + conn = TdxConnection(host, PROBE_PORT, PROBE_TIMEOUT) + dt = await asyncio.wait_for(conn.ping(), timeout=PROBE_TIMEOUT + 1.0) + results[host] = dt # 失败时为 None + except Exception: + results[host] = None + + await asyncio.gather(*(_probe_one(h) for h in hosts)) + + ok_hosts = {h: lt for h, lt in results.items() if lt is not None} + # 按延迟升序排名 + ranked = sorted(ok_hosts.items(), key=lambda kv: kv[1]) + fast = set() + for i, (h, lt) in enumerate(ranked): + if i < FAST_POOL_TOP_N or lt < FAST_POOL_MAX_LATENCY: + fast.add(h) + + log.info("测速完成: 可用 %d 台, 入池 %d 台 (前%d或<%dms)", + len(ok_hosts), len(fast), FAST_POOL_TOP_N, + int(FAST_POOL_MAX_LATENCY * 1000)) + self._write_results(results, fast) + + def _write_results(self, results: dict[str, float | None], fast: set[str]) -> None: + try: + cfg = json.loads(self._file.read_text(encoding="utf-8")) + except (OSError, ValueError): + cfg = {"servers": []} + servers = cfg.get("servers", []) + by_ip = {s.get("ip"): s for s in servers} + + today = date.today().isoformat() + now = time.strftime("%Y-%m-%d %H:%M:%S") + checked_any = False + for host, lt in results.items(): + entry = by_ip.get(host) + if entry is None: + entry = {"ip": host, "name": host, "protocols": ["standard"]} + servers.append(entry) + by_ip[host] = entry + if lt is not None: + checked_any = True + entry["status"] = "ok" if host in fast else "slow" + entry["latency"] = round(lt, 3) + entry["fast_pool"] = host in fast + else: + # 探测不可达:不降级已有 ok(避免误杀),仅标记本次不可达 + entry.setdefault("status", "unknown") + entry["latency"] = None + entry["fast_pool"] = False + entry["checked_date"] = today + entry["checked_at"] = now + + # 若有 ok 服务器写入可入池日期;否则保留今日以免反复重试 + cfg["checked_date"] = today + cfg["note"] = ( + f"当日全量测速入库({today}): 入池={len(fast)}台 " + f"(排名前{FAST_POOL_TOP_N}或延迟<{int(FAST_POOL_MAX_LATENCY*1000)}ms)" + ) + if not checked_any: + # 全部不可达时不写 checked_date,下次启动重试 + cfg.pop("checked_date", None) + + self._file.write_text( + json.dumps(cfg, ensure_ascii=False, indent=2), encoding="utf-8" + ) + log.info("测速结果已写入 %s", self._file) + + +# 进程级单例(跨池共享,确保当日仅测一次) +_probe_cache = ProbeCache() + + +def get_probe_cache() -> ProbeCache: + return _probe_cache diff --git a/src/pytdxdata/protocol/__init__.py b/src/pytdxdata/protocol/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pytdxdata/protocol/commands/__init__.py b/src/pytdxdata/protocol/commands/__init__.py new file mode 100644 index 0000000..950542f --- /dev/null +++ b/src/pytdxdata/protocol/commands/__init__.py @@ -0,0 +1,32 @@ +"""命令注册表:命令类型 → (命令类, 通道, 页大小) 元数据""" +from __future__ import annotations + +from .base import BaseCommand, TdxDecodeError, build_standard_request +from .finance import GetFinanceInfoCmd +from .kline import GetIndexBarsCmd, GetSecurityBarsCmd, PAGE_SIZE as KLINE_PAGE +from .mac import MacKlineCmd, MAC_KLINE_PAGE +from .minute import GetHistoryMinuteTimeDataCmd +from .quotes import GetSecurityQuotesCmd, MAX_QUOTES +from .security_list import GetSecurityCountCmd, GetSecurityListCmd, PAGE_SIZE as LIST_PAGE +from .transaction import ( + GetHistoryTransactionDataCmd, + GetTransactionDataCmd, + MacTransactionCmd, + MAC_PAGE_SIZE, + STD_PAGE_SIZE, +) + +__all__ = [ + "BaseCommand", "TdxDecodeError", "build_standard_request", + "GetSecurityBarsCmd", "GetIndexBarsCmd", "KLINE_PAGE", + "GetSecurityQuotesCmd", "MAX_QUOTES", + "GetSecurityCountCmd", "GetSecurityListCmd", "LIST_PAGE", + "GetTransactionDataCmd", "GetHistoryTransactionDataCmd", "MacTransactionCmd", + "STD_PAGE_SIZE", "MAC_PAGE_SIZE", + "GetHistoryMinuteTimeDataCmd", + "GetXdxrInfoCmd", "GetFinanceInfoCmd", + "MacKlineCmd", "MAC_KLINE_PAGE", +] + +# 循环依赖规避:xdxr 需要在此导入 +from .xdxr import GetXdxrInfoCmd # noqa: E402 diff --git a/src/pytdxdata/protocol/commands/base.py b/src/pytdxdata/protocol/commands/base.py new file mode 100644 index 0000000..210d3ba --- /dev/null +++ b/src/pytdxdata/protocol/commands/base.py @@ -0,0 +1,38 @@ +"""命令基类:请求构造 + 响应解析(不含IO)""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Generic, TypeVar + +T = TypeVar("T") + + +class BaseCommand(ABC, Generic[T]): + """所有行情命令的基类。子类实现 build_request/parse_response。""" + + @abstractmethod + def build_request(self) -> bytes: + """构造请求包(含完整帧头)。""" + ... + + @abstractmethod + def parse_response(self, body: bytes) -> T: + """解析解压后的响应 body,返回强类型结果。""" + ... + + +class TdxDecodeError(Exception): + """协议解析错误(服务器截断/ret_count撒谎等)。""" + + +def build_standard_request( + cmd_id: int, + method: int, + payload: bytes, + seq_id: int = 0x01016408, +) -> bytes: + """构造标准通道请求帧:12字节头 + payload。""" + import struct + total = 12 + len(payload) + header = struct.pack(" bytes: + """EX类命令帧:01 + 4B customize + 01 + zipsize + unzipsize + cmd(LE)。""" + inner_len = 2 + len(body) + return struct.pack(" None: + self.market = market + self.start = start + self.count = count + + def build_request(self) -> bytes: + return _ex_frame(0x23F5, struct.pack(" list[dict]: + if len(body) < 6: + return [] + n = len(body) - 6 + items: list[dict] = [] + off = 6 + for _ in range(n // 64): + if off + 64 > len(body): + break + rec = body[off:off + 64] + # 前40B: BB3s9s17s9s (category/market/3B/code9/name17/desc9) + _, mkt, _, code_b, name_b, desc_b = struct.unpack(" bytes: + return _ex_frame(0x23F0, b"") + + def parse_response(self, body: bytes) -> int: + if len(body) < 23: + return 0 + (count,) = unpack_from(" None: + self.market = market + self.code = code + self.start = start + self.count = count + + def build_request(self) -> bytes: + body = struct.pack(" list[dict]: + if len(body) < 16: + return [] + (_, _, _, num) = struct.unpack(" len(body): + break + minutes, price_int, vol, zengcang, direction = struct.unpack( + " None: + self.ymd = ymd + self.market = market + self.code = code + self.start = start + self.count = count + + def build_request(self) -> bytes: + body = struct.pack(" list[dict]: + if len(body) < 16: + return [] + (_, _, _, num) = struct.unpack(" len(body): + break + minutes, price_int, vol, zengcang, direction = struct.unpack( + " None: + self.market = market + self.code = code + self.ymd = ymd + + def build_request(self) -> bytes: + from .mac_ext import build_mac_request + raw = self.code.encode("gbk") + padded = (raw + b"\x00" * 22)[:22] + body = struct.pack(" list[dict]: + if len(body) < 35: + return [] + (_, _, _, _, ref_price, count) = struct.unpack(" len(body): + break + minutes, price, avg, vol, momentum = struct.unpack(" bytes: + """EX类命令帧:01 + 4B customize + 01 + zipsize + unzipsize + cmd(LE)。""" + inner_len = 2 + len(body) + return struct.pack(" bytes: + return EX_SETUP_CMD + + def parse_response(self, body: bytes) -> bool: + return len(body) >= 2 + + +class GetMarketsCmd(BaseCommand[list[dict]]): + """EX市场列表(0x23F4,请求无body)。""" + + def build_request(self) -> bytes: + return _ex_frame(0x23F4, b"") + + def parse_response(self, body: bytes) -> list[dict]: + if len(body) < 2: + return [] + (cnt,) = struct.unpack(" len(body): + break + category, name_b, market, short_b, _, _ = struct.unpack( + " None: + self.market = market + self.code = code + self.category = category + self.start = start + self.count = count + + def build_request(self) -> bytes: + body = struct.pack(" list[dict]: + if len(body) < 20: + return [] + (ret_count,) = unpack_from(" len(body): + break + # 时间4字节按category分格式:分钟级=2B zipday+2B tminutes;日线及以上=YYYYMMDD + year, month, day, hour, minute, _ = get_datetime(self.category, body, off) + open_, high, low, close, position, trade, _ = struct.unpack( + " None: + self.market = market + self.code = code + + def build_request(self) -> bytes: + return _ex_frame(0x23FA, struct.pack(" dict | None: + if len(body) < 150: + return None + # None: + self.market = market + self.code = code.encode("utf-8") + + def build_request(self) -> bytes: + header = bytes.fromhex("0c1f187600010b000b0010000100".replace(" ", "")) + return header + struct.pack(" FinanceRecord | None: + pos = 2 + market_b, code_b = unpack_from(" len(body): + return None + fields = struct.unpack(_FIN_FMT, slice_bytes(body, pos, _FIN_SIZE, "finance body")) + ( + liutong_guben, province, industry, updated_date, ipo_date, + zong_guben, guojia_gu, faqiren_faren_gu, faren_gu, b_gu, + h_gu, zhigong_gu, zong_zichan, liudong_zichan, guding_zichan, + wuxing_zichan, gudong_renshu, liudong_fuzhai, changqi_fuzhai, ziben_gongjijin, + jing_zichan, zhuying_shouru, zhuying_lirun, yingshou_zhangkuan, yingye_lirun, + touzi_shouyu, jingying_xianjinliu, zong_xianjinliu, cunhuo, lirun_zonghe, + shuihou_lirun, jing_lirun, weifen_lirun, meigujing_zichan, _reserve2, + ) = fields + return FinanceRecord( + market=market_b, code=code_b.decode("utf-8").rstrip("\x00"), + liutong_guben=liutong_guben * _SCALE, zong_guben=zong_guben * _SCALE, + guojia_gu=guojia_gu * _SCALE, faqiren_faren_gu=faqiren_faren_gu * _SCALE, + faren_gu=faren_gu * _SCALE, b_gu=b_gu * _SCALE, h_gu=h_gu * _SCALE, + zhigong_gu=zhigong_gu * _SCALE, + province=int(province), industry=int(industry), + updated_date=int(updated_date), ipo_date=int(ipo_date), + gudong_renshu=gudong_renshu, + zong_zichan=zong_zichan * _SCALE, liudong_zichan=liudong_zichan * _SCALE, + guding_zichan=guding_zichan * _SCALE, wuxing_zichan=wuxing_zichan * _SCALE, + liudong_fuzhai=liudong_fuzhai * _SCALE, changqi_fuzhai=changqi_fuzhai * _SCALE, + ziben_gongjijin=ziben_gongjijin * _SCALE, + jing_zichan=jing_zichan * _SCALE, zhuying_shouru=zhuying_shouru * _SCALE, + zhuying_lirun=zhuying_lirun * _SCALE, yingshou_zhangkuan=yingshou_zhangkuan * _SCALE, + yingye_lirun=yingye_lirun * _SCALE, touzi_shouyu=touzi_shouyu * _SCALE, + jingying_xianjinliu=jingying_xianjinliu * _SCALE, + zong_xianjinliu=zong_xianjinliu * _SCALE, + cunhuo=cunhuo * _SCALE, lirun_zonghe=lirun_zonghe * _SCALE, + shuihou_lirun=shuihou_lirun * _SCALE, jing_lirun=jing_lirun * _SCALE, + weifen_lirun=weifen_lirun * _SCALE, meigujing_zichan=meigujing_zichan, + reserve2=_reserve2, + ) diff --git a/src/pytdxdata/protocol/commands/kline.py b/src/pytdxdata/protocol/commands/kline.py new file mode 100644 index 0000000..a4a8b0a --- /dev/null +++ b/src/pytdxdata/protocol/commands/kline.py @@ -0,0 +1,115 @@ +"""K线命令(标准通道 0x010C,含差分还原解析;指数K线多4字节跳过)""" +from __future__ import annotations + +import struct +from typing import Any + +from ..._binary import unpack_from +from ...models import IndexBar, KlinePeriod, SecurityBar +from ..datetime_ import get_datetime +from ..price import get_price +from ..volume import get_volume +from .base import BaseCommand, TdxDecodeError + +PAGE_SIZE = 800 + + +class GetSecurityBarsCmd(BaseCommand[list[SecurityBar]]): + """获取指定股票K线(分页,最多800/次)。""" + + def __init__(self, market: int, code: str, category: KlinePeriod, + start: int, count: int = 800) -> None: + self.market = market + self.code = code.encode("utf-8") + self.category = int(category) + self.start = start + self.count = count + + def build_request(self) -> bytes: + return struct.pack( + " list[SecurityBar]: + (ret_count,) = unpack_from(" list[IndexBar]: + (ret_count,) = unpack_from(" None: + self.market = market + self.code = code + self.period = period + self.adjust = adjust + self.start = start + self.count = count + self.times = times + + def build_request(self) -> bytes: + # 与 easy-tdx SymbolBarCmd 一致: list[SecurityBar]: + # 头部: market(2)+code(22)+category(2)+flag(1)+count(2)+start(4)=33B; count@offset24 len(body): + break + ymd, time_num, open_, high, low, close, amount, vol, _fs = unpack_from( + " 20991231: + continue + year, month, day = ymd // 10000, (ymd % 10000) // 100, ymd % 100 + hour = minute = 15 + if is_intraday and time_num: + hour = time_num // 3600 + minute = (time_num % 3600) // 60 + bars.append(SecurityBar( + market=self.market, code=self.code, + open=open_, close=close, high=high, low=low, + vol=vol, amount=amount, + year=year, month=month, day=day, hour=hour, minute=minute, + )) + return bars diff --git a/src/pytdxdata/protocol/commands/mac_ext.py b/src/pytdxdata/protocol/commands/mac_ext.py new file mode 100644 index 0000000..7b12644 --- /dev/null +++ b/src/pytdxdata/protocol/commands/mac_ext.py @@ -0,0 +1,508 @@ +"""MAC 扩展命令:板块/集合竞价/异动/个股快照/扩展市场/服务器信息/资金流""" +from __future__ import annotations + +import struct + +from ..._binary import slice_bytes, unpack_from +from ...models import ( + AuctionItem, BelongBoard, BoardInfo, CapitalFlow, GoodsItem, MemberQuote, + ServerInfo, SymbolSnapshot, UnusualItem, +) +from ..mac_frame import build_mac_request +from .base import BaseCommand + +# 板块成分股常用字段位(FieldBit: 0x00昨收 0x01开 0x02高 0x03低 0x04收 0x05量 0x06量比 0x07额) +_MEMBER_BASIC_BITS = [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07] + + +def _build_bitmap(bits: list[int], size: int = 16) -> bytes: + bm = bytearray(size) + for b in bits: + bm[b // 8] |= 1 << (b % 8) + return bytes(bm) + + +def _field_fmt(bit: int) -> str: + """按位返回解析格式( None: + self.board_type = board_type # 0=全部 1=行业 2=概念 ... + self.start = start + self.page_size = page_size + + def build_request(self) -> bytes: + body = struct.pack(" list[BoardInfo]: + results: list[BoardInfo] = [] + pos = 4 # 跳过2B count_all + 2B total + n = (len(body) - pos) // 160 + for _ in range(n): + # 每160B记录=板块段(80B) + 领涨股段(80B),合并为一条 + b_seg = slice_bytes(body, pos, 80, "board segment") + market, code_b, _pad, name_b, price, rise, pre = struct.unpack(" None: + self.board_code = board_code + self.sort_type = sort_type + self.start = start + self.page_size = page_size + self.bits = bits if bits is not None else list(_MEMBER_BASIC_BITS) + + def build_request(self) -> bytes: + bitmap = _build_bitmap(self.bits) + body = (struct.pack(" list[MemberQuote]: + # 响应: 20B响应位图 + None: + self.market = market + self.code = code + + def build_request(self) -> bytes: + body = struct.pack(" list[BelongBoard]: + # 响应: 27B头 + GBK JSON 数组 + json_part = body[27:].decode("gbk", errors="replace") + results: list[BelongBoard] = [] + try: + import json + rows = json.loads(json_part) + for row in rows: + if len(row) < 6: + continue + results.append(BelongBoard( + market=int(row[1]), board_code=str(row[2]), + board_name=str(row[3]), + close=float(row[4] or 0), pre_close=float(row[5] or 0), + )) + except Exception: + pass + return results + + +# ------------------------------------------------------------------ # +# 集合竞价 0x123D +# ------------------------------------------------------------------ # + +class MacAuctionCmd(BaseCommand[list[AuctionItem]]): + def __init__(self, market: int, code: str, start: int = 0, count: int = 100) -> None: + self.market = market + self.code = code + self.start = start + self.count = count + + def build_request(self) -> bytes: + body = struct.pack(" list[AuctionItem]: + (count,) = unpack_from(" None: + self.market = market + self.start = start + self.count = min(count, 600) + + def build_request(self) -> bytes: + body = struct.pack(" list[UnusualItem]: + (count,) = unpack_from(" len(body): + break + # tuple[str, float]: + """异动类型描述。""" + if len(data) < 13: + return "", 0.0 + v1, v2, v3, v4 = struct.unpack(" None: + self.market = market + self.code = code + + def build_request(self) -> bytes: + body = struct.pack(" SymbolSnapshot: + _market, code_b, name_b = unpack_from(" None: + self.market = market + self.start = start + self.count = count + + def build_request(self) -> bytes: + body = struct.pack(" list[GoodsItem]: + (count,) = unpack_from(" bytes: + body = bytes.fromhex("04002d31" + "00" * 8 + "0027060e" + "00" * 52) + return build_mac_request(0x120F, body) + + def parse_response(self, body: bytes) -> ServerInfo: + if len(body) < 87: + return ServerInfo(date=0, last_trade_date=0) + pos = 0 + pos += 2 # count + pos += 8 # flags + pos += 3 # tag + pos += 9 # reserved + (date,) = unpack_from(" None: + self.offset = offset + self.count = min(count, 128000) + + def build_request(self) -> bytes: + body = struct.pack(" tuple[int, int]: + total = struct.unpack(">I", body[:4])[0] # 大端 + (returned,) = unpack_from(" None: + self.market = market + self.code = code + + def build_request(self) -> bytes: + body = struct.pack(" CapitalFlow | None: + json_part = body[27:].decode("gbk", errors="replace") + try: + import json + rows = json.loads(json_part) + if len(rows) < 2: + return None + r0 = rows[0] # [主买,主卖,散户买,散户卖] + r1 = rows[1] # [5日买,5日卖,超大,大,中,小] + return CapitalFlow( + market=self.market, code=self.code, + main_buy=float(r0[0]), main_sell=float(r0[1]), + retail_buy=float(r0[2]), retail_sell=float(r0[3]), + buy5=float(r1[0]), sell5=float(r1[1]), + big5=float(r1[3]), mid5=float(r1[4]), + ) + except Exception: + return None +class MacSymbolQuotesCmd(BaseCommand[list[MemberQuote]]): + """批量获取自定义字段报价(最多80只/次)。""" + + def __init__(self, stocks: list[tuple[int, str]], bits: list[int] | None = None) -> None: + self.stocks = stocks + self.bits = bits if bits is not None else list(range(0x00, 0x08)) + + @staticmethod + def _build_bitmap(bits: list[int]) -> bytes: + bm = bytearray(20) + for b in bits: + bm[b // 8] |= 1 << (b % 8) + return bytes(bm) + + @staticmethod + def _field_fmt(bit: int) -> str: + if bit in (0x05, 0x08, 0x09, 0x0A, 0x0B, 0x13, 0x14, 0x15, 0x18, 0x19, 0x1A): + return " bytes: + body = self._build_bitmap(self.bits) + struct.pack(" list[MemberQuote]: + resp_bitmap = body[:20] + pos = 20 # 20B响应位图 + _total, row_count = unpack_from(" None: + self.offset = offset + self.size = size + self.flag = flag + self.md5 = md5 + + +class MacFileListCmd(BaseCommand[FileMeta]): + """远程文件元信息 0x1215。""" + + def __init__(self, filename: str) -> None: + self.filename = filename + + def build_request(self) -> bytes: + fn = self.filename.encode("gbk", errors="replace")[:70].ljust(70, b"\x00") + body = struct.pack(" FileMeta: + offset, size, flag = unpack_from(" None: + self.filename = filename + self.index = index + self.offset = offset + self.size = size + + def build_request(self) -> bytes: + fn = self.filename.encode("gbk", errors="replace")[:70].ljust(70, b"\x00") + body = struct.pack(" bytes: + return body[8:] + +# ------------------------------------------------------------------ # +# 分时缩略采样 0x254D +# ------------------------------------------------------------------ # + +class MacChartSamplingCmd(BaseCommand[list[float]]): + """分时缩略采样价格点(约240个)。""" + + def __init__(self, market: int, code: str) -> None: + self.market = market + self.code = code + + def build_request(self) -> bytes: + raw = self.code.encode("gbk") + padded = (raw + b"\x00" * 22)[:22] + body = struct.pack(" list[float]: + if len(body) < 42: + return [] + (count,) = unpack_from(" len(body): + break + (p,) = unpack_from(" None: + self.market = market + self.code = code + self.date = date + + def build_request(self) -> bytes: + body = struct.pack( + " list[MinuteBar]: + # 头部: market(2) + code(22) + query_date(4) + reserved(1) + ref_price(4) + count(2) = 35 + if len(body) < 35: + return [] + (_market, _code_raw, _query_date, _reserved, _ref_price, count) = unpack_from( + " len(body): + break + (minutes, price, avg, vol, _momentum) = unpack_from( + " None: + self.market = market + self.code = code.encode("utf-8") + self.date = date + + def build_request(self) -> bytes: + header = bytes.fromhex("0c01300001010d000d00b40f".replace(" ", "")) + return header + struct.pack(" list[MinuteBar]: + return _parse_minute_body(body, skip=6) + + +def _parse_minute_body(body: bytes, skip: int) -> list[MinuteBar]: + (num,) = unpack_from(" int: + """推断报价有效小数位:股票2位(/100)、指数/ETF/基金/转债/国债3位(/1000)。""" + code = (code or "").strip().rstrip("\x00") + if market == 1: # SH + if code.startswith(("5", "000", "8")): + return 3 + return 2 + if market == 0: # SZ + if code.startswith("1"): + return 3 + return 2 + return 2 + + +def format_server_time(raw: int) -> str: + """服务器时间编码:小时 + 百万分之一小时。14999212 -> 14:59:57.163""" + hours, fractional_hour = divmod(raw, 1_000_000) + total_millis = fractional_hour * 3600 // 1000 + minutes, remainder = divmod(total_millis, 60_000) + seconds, millis = divmod(remainder, 1000) + return f"{hours:02d}:{minutes:02d}:{seconds:02d}.{millis:03d}" + + +class GetSecurityQuotesCmd(BaseCommand[list[SecurityQuote]]): + """批量获取实时五档行情(最多80只/次)。""" + + def __init__(self, stocks: list[tuple[int, str]]) -> None: + if not stocks: + raise ValueError("stocks 不能为空") + if len(stocks) > MAX_QUOTES: + raise ValueError(f"单次最多查询{MAX_QUOTES}只") + self.stocks = stocks + + def build_request(self) -> bytes: + n = len(self.stocks) + payload_len = n * 7 + 12 + header = struct.pack( + " list[SecurityQuote]: + pos = 2 # 跳过2字节魔数 + (num,) = unpack_from(" None: + self.market = market + + def build_request(self) -> bytes: + # 与 easy-tdx 一致: 0c0c186c 头 + market(H) + 4字节尾部 + header = bytes.fromhex("0c0c186c0001080008004e04".replace(" ", "")) + return header + struct.pack(" int: + (count,) = unpack_from(" None: + self.market = market + self.start = start + + def build_request(self) -> bytes: + header = bytes.fromhex("0c0118640101060006005004".replace(" ", "")) + return header + struct.pack(" list[SecurityInfo]: + (num,) = unpack_from(" bytes: + return bytes.fromhex(head_hex) + payload + + +# ------------------------------------------------------------------ # +# 公司信息 F10 0x109B / 0x109C +# ------------------------------------------------------------------ # + +class CompanyInfoCategory: + """F10目录条目""" + __slots__ = ("name", "filename", "start", "length") + + def __init__(self, name: str, filename: str, start: int, length: int) -> None: + self.name = name + self.filename = filename + self.start = start + self.length = length + + +class GetCompanyInfoCategoryCmd(BaseCommand[list[CompanyInfoCategory]]): + def __init__(self, market: Market, code: str) -> None: + self.market = market + self.code = code + + def build_request(self) -> bytes: + return _std_req("0c0f109b00010e000e00cf02", + struct.pack(" list[CompanyInfoCategory]: + (num,) = unpack_from(" None: + self.market = market + self.code = code + self.filename = filename + self.offset = offset + self.length = length + + def build_request(self) -> bytes: + fn = self.filename.encode("gbk")[:80].ljust(80, b"\x00") + return _std_req("0c07109c000168006800d002", + struct.pack(" str: + # 12B头(10s+H len) + len字节GBK文本 + (length,) = unpack_from(" None: + self.size = size + self.md5 = md5 + self.filename = filename + + +class GetBlockInfoMetaCmd(BaseCommand[BlockInfo]): + def __init__(self, filename: str) -> None: + self.filename = filename + + def build_request(self) -> bytes: + fn = self.filename.encode("ascii", errors="replace")[:40].ljust(40, b"\x00") + return _std_req("0c39186900012a002a00c502", fn) + + def parse_response(self, body: bytes) -> BlockInfo: + size, _v1, md5_b, _v2 = struct.unpack(" None: + self.filename = filename + self.start = start + self.length = length + + def build_request(self) -> bytes: + fn = self.filename.encode("ascii", errors="replace")[:100].ljust(100, b"\x00") + return _std_req("0c37186a00016e006e00b906", + struct.pack(" bytes: + return body[4:] # 跳过前4字节返回原始二进制 + + +# ------------------------------------------------------------------ # +# 涨跌停价(本地计算,无网络命令) +# ------------------------------------------------------------------ # + +def compute_price_limits(market: Market, code: str, name: str, + pre_close: float, listed_days: int = 9999) -> tuple[float | None, float | None]: + """计算涨跌停价(参照 easy-tdx codec/price_rules)。指数/基金无涨跌停。""" + code = (code or "").strip() + if pre_close is None or pre_close <= 0: + return None, None + # 指数类无涨跌停 + if market == Market.SH and (code.startswith(("000", "880", "999")) or "指数" in (name or "")): + return None, None + if market == Market.SZ and (code.startswith(("395", "399")) or "指数" in (name or "")): + return None, None + # 涨跌幅 + if market == Market.BJ: + pct = 0.30 + elif code.startswith(("688", "300", "301")): + pct = 0.20 + elif "ST" in (name or "").upper(): + pct = 0.05 + else: + pct = 0.10 + # 上市初期不设限(沪/深前5日, 北交首日) + if listed_days <= 5 and market != Market.BJ: + return None, None + if listed_days <= 0: + return None, None + up = round(pre_close * (1 + pct) + 0.00001, 2) + down = round(pre_close * (1 - pct) + 0.00001, 2) + return up, down + + +# ------------------------------------------------------------------ # +# 市场统计(用报价接口查上证指数880005/880001/880006组合,由api层实现) +# ------------------------------------------------------------------ # + +class MarketStat: + __slots__ = ("up_count", "down_count", "flat_count", "total_count", + "total_amount", "total_vol", "total_market_cap", + "limit_up_count", "limit_down_count") + + def __init__(self) -> None: + self.up_count = 0 + self.down_count = 0 + self.flat_count = 0 + self.total_count = 0 + self.total_amount = 0.0 + self.total_vol = 0.0 + self.total_market_cap = 0.0 + self.limit_up_count = 0 + self.limit_down_count = 0 + + +# ------------------------------------------------------------------ # +# 报告/财务文件 0x06B9(分块下载,get_report_file/get_financial_file 共用) +# ------------------------------------------------------------------ # + +class GetReportFileCmd(BaseCommand[bytes]): + """下载远程报告/财务文件分块(tdxfin/gpcw.txt、gpcw*.zip 等)。""" + + def __init__(self, filename: str, start: int = 0, length: int = 30000) -> None: + self.filename = filename + self.start = start + self.length = length + + def build_request(self) -> bytes: + fn = self.filename.encode("ascii", errors="replace")[:100].ljust(100, b"\x00") + return _std_req("0c37186a00016e006e00b906", + struct.pack(" bytes: + return body[4:] # 跳过前4字节返回原始文件块 diff --git a/src/pytdxdata/protocol/commands/transaction.py b/src/pytdxdata/protocol/commands/transaction.py new file mode 100644 index 0000000..155be90 --- /dev/null +++ b/src/pytdxdata/protocol/commands/transaction.py @@ -0,0 +1,126 @@ +"""逐笔成交命令:标准通道(当日0x170801含笔数 / 历史0x013001)+ MAC通道(0x122F含trade_count)""" +from __future__ import annotations + +import struct + +from ..._binary import unpack_from +from ...models import TransactionRecord +from ..datetime_ import get_time +from ..mac_frame import build_mac_request +from ..price import get_price +from .base import BaseCommand + +STD_PAGE_SIZE = 800 +MAC_PAGE_SIZE = 1000 + + +class GetTransactionDataCmd(BaseCommand[list[TransactionRecord]]): + """标准通道当日逐笔(0x170801)。协议含成交笔数(_num_orders),此处显式解析暴露。""" + + def __init__(self, market: int, code: str, start: int, count: int = 800) -> None: + self.market = market + self.code = code.encode("utf-8") + self.start = start + self.count = count + + def build_request(self) -> bytes: + header = bytes.fromhex("0c17080101010e000e00c50f".replace(" ", "")) + return header + struct.pack(" list[TransactionRecord]: + (num,) = unpack_from(" None: + self.market = market + self.code = code.encode("utf-8") + self.date = date + self.start = start + self.count = count + + def build_request(self) -> bytes: + header = bytes.fromhex("0c013001000112001200b50f".replace(" ", "")) + return header + struct.pack(" list[TransactionRecord]: + (num,) = unpack_from(" None: + self.market = market + self.code = code + self.ymd = ymd + self.start = start + self.count = count + + def build_request(self) -> bytes: + body = struct.pack( + " list[TransactionRecord]: + (count,) = unpack_from(" None: + self.market = market + self.code = code.encode("utf-8") + + def build_request(self) -> bytes: + header = bytes.fromhex("0c1f187600010b000b000f000100".replace(" ", "")) + return header + struct.pack(" list[XdxrRecord]: + if len(body) < 11: + raise ValueError("xdxr body 过短") + pos = 9 + (num,) = unpack_from(" tuple[int, int, int, int, int, int]: + """按K线周期解析4字节时间,返回 (year, month, day, hour, minute, new_pos)。 + + 分钟级(category<4 或 ==7/8):2字节压缩日期+2字节分钟数,year=(>>11)+2004 + 日线及以上:4字节 YYYYMMDD,hour=15, minute=0(收盘时间) + """ + if category < 4 or category in (7, 8): + zipday, tminutes = unpack_from("> 11) + 2004 + month = (zipday % 2048) // 100 + day = (zipday % 2048) % 100 + hour = tminutes // 60 + minute = tminutes % 60 + return year, month, day, hour, minute, pos + 4 + (zipday,) = unpack_from(" tuple[int, int, int]: + """解析2字节时间(分钟数),返回 (hour, minute, new_pos)。""" + (tminutes,) = unpack_from(" FrameHeader: + magic, seq_id, method, zipsize, unzipsize = unpack_from(_HEADER_FMT, buf, 0, "frame header") + return FrameHeader(magic, seq_id, method, zipsize, unzipsize) + + +def decompress_body(header: FrameHeader, raw_body: bytes) -> bytes: + """按需 zlib 解压 body(zipsize==unzipsize 直接返回)。""" + if len(raw_body) != header.zipsize: + raise ValueError( + f"frame body 长度不符: header={header.zipsize}, actual={len(raw_body)}" + ) + if header.zipsize == header.unzipsize: + return raw_body + try: + body = zlib.decompress(raw_body) + except zlib.error as e: + raise ValueError(f"zlib 解压失败: {e}") from e + if len(body) != header.unzipsize: + raise ValueError( + f"解压长度不符: header={header.unzipsize}, actual={len(body)}" + ) + return body diff --git a/src/pytdxdata/protocol/mac_frame.py b/src/pytdxdata/protocol/mac_frame.py new file mode 100644 index 0000000..4b3ba50 --- /dev/null +++ b/src/pytdxdata/protocol/mac_frame.py @@ -0,0 +1,22 @@ +"""MAC 协议请求帧构建(10字节头 + 2字节命令ID + body)""" +from __future__ import annotations + +import struct + +_MAC_HEADER_FMT = " bytes: + """构建 MAC 协议请求帧。""" + inner = struct.pack(" tuple[int, int]: + """解码一个变长有符号整数,返回 (value, new_pos)。 + + 协议规则:首字节 bit7=继续, bit6=符号(1负), bit5~0=低6位;后续字节 bit7=继续, bit6~0=7位。 + """ + start = pos + try: + b = data[pos] + except IndexError: + raise ValueError(f"price varint 截断 @{start}") + value = b & 0x3F + negative = bool(b & 0x40) + if b & 0x80: + bit_shift = 6 + while True: + pos += 1 + b = data[pos] + value |= (b & 0x7F) << bit_shift + bit_shift += 7 + if not (b & 0x80): + break + pos += 1 + return (-value if negative else value), pos + + +def put_price(value: int) -> bytes: + """将整数编码为变长格式(构造请求包用)。""" + negative = value < 0 + value = abs(value) + first = value & 0x3F + value >>= 6 + if negative: + first |= 0x40 + if value: + first |= 0x80 + result = bytearray([first]) + while value: + b = value & 0x7F + value >>= 7 + if value: + b |= 0x80 + result.append(b) + return bytes(result) + + +def decode_ohlc_diffs(data: bytes | bytearray | memoryview, pos: int) -> tuple[float, float, float, float, int]: + """解码 K线单条记录的 OHLC 差分(相对基准),返回 (open, close, high, low, new_pos)。 + + 差分还原(与 pytdx/easy-tdx 一致): + open_abs = open_diff + pre_diff_base + close_abs = open_abs + close_diff + high_abs = open_abs + high_diff + low_abs = open_abs + low_diff + pre_diff_base 更新为 open_abs + close_diff(由调用方维护) + 价格 /1000 还原为元。 + """ + open_diff, pos = get_price(data, pos) + close_diff, pos = get_price(data, pos) + high_diff, pos = get_price(data, pos) + low_diff, pos = get_price(data, pos) + return open_diff, close_diff, high_diff, low_diff, pos diff --git a/src/pytdxdata/protocol/volume.py b/src/pytdxdata/protocol/volume.py new file mode 100644 index 0000000..81c4105 --- /dev/null +++ b/src/pytdxdata/protocol/volume.py @@ -0,0 +1,46 @@ +"""通达信 4 字节自定义浮点解码(成交量/股本专用)""" +from __future__ import annotations + +from .._binary import unpack_from + + +def get_volume(data: bytes | bytearray | memoryview, pos: int) -> tuple[float, int]: + """从 data[pos:pos+4] 解码成交量,返回 (volume_float, new_pos)。""" + (ivol,) = unpack_from(" float: + """解码单个自定义浮点整数。""" + return _decode_volume(raw) + + +def _decode_volume(ivol: int) -> float: + if ivol == 0: + return 0.0 + logpoint = (ivol >> 24) & 0xFF + hleax = (ivol >> 16) & 0xFF + lheax = (ivol >> 8) & 0xFF + lleax = ivol & 0xFF + + exp = logpoint * 2 - 0x7F + base = _pow2(exp) + + exp_h = logpoint * 2 - 0x86 + if hleax > 0x80: + hi = _pow2(exp_h) * 128 + (hleax & 0x7F) * _pow2(exp_h + 1) + else: + hi = _pow2(exp_h) * hleax + + mid = _pow2(logpoint * 2 - 0x8E) * lheax + lo = _pow2(logpoint * 2 - 0x96) * lleax + if hleax & 0x80: + mid *= 2.0 + lo *= 2.0 + return base + hi + mid + lo + + +def _pow2(exp: int) -> float: + if exp >= 0: + return float(1 << exp) if exp < 63 else 2.0**exp + return 1.0 / (1 << (-exp)) if -exp < 63 else 2.0**exp diff --git a/src/pytdxdata/router.py b/src/pytdxdata/router.py new file mode 100644 index 0000000..b724b66 --- /dev/null +++ b/src/pytdxdata/router.py @@ -0,0 +1,20 @@ +"""通道选路:命令类型 → 标准/MAC""" +from __future__ import annotations + +from typing import Literal + +from .models import Adjust, KlinePeriod + +Route = Literal["standard", "mac"] + + +def route_for(cmd_type: str, *, adjust: Adjust = Adjust.NONE, + period: KlinePeriod | None = None) -> Route: + """返回命令应走的通道。""" + if cmd_type == "kline": + if period == KlinePeriod.MIN_1: + return "mac" # MIN_1 统一走 MAC (0x122E),支持深度分页无缺失 + return "mac" if adjust != Adjust.NONE else "standard" + if cmd_type == "transaction": + return "mac" # MAC 0x122F 含 trade_count 且支持历史日期 + return "standard" # quote/list/count/minute/xdxr/finance 均走标准 diff --git a/src/pytdxdata/scheduler/__init__.py b/src/pytdxdata/scheduler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pytdxdata/scheduler/paginator.py b/src/pytdxdata/scheduler/paginator.py new file mode 100644 index 0000000..d952163 --- /dev/null +++ b/src/pytdxdata/scheduler/paginator.py @@ -0,0 +1,155 @@ +"""并发分页调度器:单请求拆N页,用至多 concurrency 个连接并行拉取,按序拼接。 + +v5 设计(回归 v1 稳定模型 + 修复连接复用 bug): +- 预分配 k 个连接,每个 worker 复用同一连接(避免每页 acquire 开销)。 +- 单页失败时:释放坏连接 → 获取新连接 → 更新 worker 持有的连接引用, + 避免跨页使用已关闭连接导致的连锁失败。 +- 不额外加重试轮(避免引入更多 acquire 命中坏服务器的机会)。 +""" +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass +from typing import Any, Callable + +from ..pool.pool import DynamicPool, PooledConnection, PoolTimeoutError + +log = logging.getLogger(__name__) + + +@dataclass(slots=True) +class PageSpec: + page_size: int = 800 + total: int | None = None # 已知总量(None=用 stop_when_short) + stop_when_short: bool = True # 某页返回 < page_size 即停 + max_pages: int | None = None + ordered: bool = True # 按 start 升序拼接 + min_page: int = 100 # 规避标准K线 count<100 空响应坑 + start_offset: int = 0 # 起始偏移(从该偏移开始分页,避免拉取历史全量) + + +@dataclass(slots=True) +class PageResult: + start: int + count: int + items: list[Any] + + +class Paginator: + """用至多 concurrency 个连接并行拉取分页数据。""" + + def __init__(self, pool: DynamicPool, *, concurrency: int = 3, + retries: int = 2, strict: bool = True) -> None: + self.pool = pool + self.concurrency = max(1, concurrency) + self.retries = retries + self.strict = strict + + async def fetch_all( + self, + cmd_factory: Callable[[int, int], Any], # (start, count) -> BaseCommand + spec: PageSpec, + *, + concurrency: int | None = None, + ) -> list[Any]: + """按序拼接后的完整结果。""" + jobs = self._plan_starts(spec) + if not jobs: + return [] + eff_conc = min( + self.concurrency if concurrency is None else concurrency, + len(jobs), + ) + conns = await self.pool.acquire_many(eff_conc) + if not conns: + raise PoolTimeoutError("连接池获取超时(无可用连接)") + try: + stop = asyncio.Event() + out: list[PageResult] = [] + out_lock = asyncio.Lock() + k = len(conns) + workers = [ + asyncio.create_task( + self._worker(wid, conns[wid % k], jobs[wid::k], cmd_factory, + stop, out, out_lock, spec) + ) + for wid in range(k) + ] + await asyncio.gather(*workers) + finally: + for pc in conns: + await self.pool.release(pc) + return self._assemble(out, spec.ordered) + + def _plan_starts(self, spec: PageSpec) -> list[tuple[int, int]]: + """切页:返回 [(start, count), ...]。""" + jobs: list[tuple[int, int]] = [] + if spec.total is not None: + pos = spec.start_offset + pages = 0 + while pos < spec.total: + if spec.max_pages is not None and pages >= spec.max_pages: + break + cnt = min(spec.page_size, spec.total - pos) + # 已知 total 时直接使用精确 cnt,不强制 min_page(避免过量请求) + jobs.append((pos, cnt)) + pos += cnt + pages += 1 + else: + for page in range(spec.max_pages if spec.max_pages is not None else 1024): + jobs.append((spec.start_offset + page * spec.page_size, spec.page_size)) + return jobs + + async def _worker( + self, wid: int, conn: PooledConnection, jobs: list[tuple[int, int]], + cmd_factory: Callable[[int, int], Any], stop: asyncio.Event, + out: list[PageResult], out_lock: asyncio.Lock, spec: PageSpec, + ) -> None: + current = conn + for start, count in jobs: + if stop.is_set(): + return + items, current = await self._fetch_one( + current, cmd_factory(start, count), count) + if items is None: + continue # 该页失败(strict=False 时留空) + async with out_lock: + out.append(PageResult(start=start, count=count, items=items)) + if spec.stop_when_short and len(items) < count: + stop.set() + return + + async def _fetch_one(self, conn: PooledConnection, cmd, page_size: int): + """单页拉取,失败换连接重试。 + + 返回 (items, current_conn) — current_conn 始终指向当前可用连接, + worker 据此更新自己的连接引用,避免跨页使用已关闭连接。 + """ + current = conn + for attempt in range(self.retries + 1): + try: + result = await current.execute(cmd) + return (list(result) if result is not None else []), current + except Exception as e: + if attempt >= self.retries: + if self.strict: + raise + log.warning("分页拉取失败(strict=False留空): %s", e) + return None, current + # 换连接重试 + await self.pool.release(current, broken=True) + try: + current = await self.pool.acquire(timeout=3.0) + except PoolTimeoutError: + log.warning("分页重试获取连接失败,跳过该页: %s", e) + return None, current + return None, current + + def _assemble(self, results: list[PageResult], ordered: bool) -> list[Any]: + if ordered: + results.sort(key=lambda r: r.start) + out: list[Any] = [] + for r in results: + out.extend(r.items) + return out diff --git a/src/pytdxdata/server_probe.py b/src/pytdxdata/server_probe.py new file mode 100644 index 0000000..60cda63 --- /dev/null +++ b/src/pytdxdata/server_probe.py @@ -0,0 +1,148 @@ +"""服务器清单导入+自动探测工具 + +读 servers.json → 并发探测每台可用性与延时(标准/MAC用7709、EX用7727)→ 写回状态。 + +用法: + python -m pytdxdata.server_probe # 全量探测 + python -m pytdxdata.server_probe --group ex # 只测EX + python -m pytdxdata.server_probe --refresh # 重新探测(status=ok的) +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +import time +from pathlib import Path + +from .pool.connection import TdxConnection +from .pool.ex_connection import ExTdxConnection +from .protocol.commands.kline import GetSecurityBarsCmd +from .protocol.commands.mac import MacKlineCmd +from .protocol.commands.ex_proto import GetExInstrumentBarsCmd +from .models import KlinePeriod, Adjust + +SERVERS_FILE = Path(__file__).resolve().parent / "servers.json" + + +def load_servers() -> dict: + if not SERVERS_FILE.exists(): + raise FileNotFoundError(f"配置文件不存在: {SERVERS_FILE}") + return json.loads(SERVERS_FILE.read_text(encoding="utf-8")) + + +def save_servers(cfg: dict) -> None: + SERVERS_FILE.write_text(json.dumps(cfg, ensure_ascii=False, indent=2), encoding="utf-8") + + +async def probe_standard(ip: str, timeout: float = 5.0) -> tuple[bool, float]: + conn = TdxConnection(ip, 7709, timeout) + t0 = time.monotonic() + try: + await conn.connect() + bars = await conn.execute(GetSecurityBarsCmd(1, "600000", 5, 0, 100)) + return (bool(bars), time.monotonic() - t0) + except Exception: + return (False, time.monotonic() - t0) + finally: + await conn.close() + + +async def probe_mac(ip: str, timeout: float = 5.0) -> tuple[bool, float]: + conn = TdxConnection(ip, 7709, timeout) + t0 = time.monotonic() + try: + await conn.connect() + bars = await conn.execute(MacKlineCmd(1, "600000", KlinePeriod.DAY, Adjust.NONE, 0, 100)) + return (bool(bars), time.monotonic() - t0) + except Exception: + return (False, time.monotonic() - t0) + finally: + await conn.close() + + +async def probe_ex(ip: str, handshake: str, timeout: float = 5.0) -> tuple[bool, float]: + conn = ExTdxConnection(ip, 7727, timeout, handshake) + t0 = time.monotonic() + try: + await conn.connect() + bars = await conn.execute(GetExInstrumentBarsCmd(31, "00700", 0, 0, 5)) + return (bool(bars), time.monotonic() - t0) + except Exception: + return (False, time.monotonic() - t0) + finally: + await conn.close() + + +async def probe_one(server: dict, timeout: float) -> dict: + ip = server["ip"] + protocols = server.get("protocols", []) + best_lat = None + ok = False + # 逐协议探测(一台多协议: standard/mac同端口可复用连接——简化逐协议测) + for proto in protocols: + if proto == "ex": + r, lat = await probe_ex(ip, server.get("handshake", "setup"), timeout) + elif proto == "mac": + r, lat = await probe_mac(ip, timeout) + else: + r, lat = await probe_standard(ip, timeout) + if r: + ok = True + best_lat = lat if best_lat is None else min(best_lat, lat) + server["status"] = "ok" if ok else "fail" + server["latency"] = round(best_lat, 3) if best_lat is not None else None + server["checked_at"] = time.strftime("%Y-%m-%d %H:%M:%S") + return server + + +async def main() -> None: + ap = argparse.ArgumentParser(description="服务器清单导入+自动探测") + ap.add_argument("--group", choices=["standard", "mac", "ex"], default=None) + ap.add_argument("--timeout", type=float, default=5.0) + ap.add_argument("--refresh", action="store_true", help="只重测status=ok的") + args = ap.parse_args() + + cfg = load_servers() + servers = cfg["servers"] + if args.group: + servers = [s for s in servers if args.group in s.get("protocols", [])] + if args.refresh: + servers = [s for s in servers if s.get("status") == "ok"] + if not servers: + print("没有待探测的服务器") + return + + print("=" * 90) + print("探测 %d 台服务器 (标准/MAC:7709, EX:7727)..." % len(servers)) + print("=" * 90) + t0 = time.time() + results = await asyncio.gather(*[probe_one(s, args.timeout) for s in servers]) + elapsed = time.time() - t0 + + # 写回 + updated = {s["ip"]: s for s in results} + for s in cfg["servers"]: + if s["ip"] in updated: + s.update(updated[s["ip"]]) + save_servers(cfg) + + ok_list = [s for s in results if s["status"] == "ok"] + ok_list.sort(key=lambda s: s["latency"]) + fail_list = [s for s in results if s["status"] == "fail"] + print("\n可用: %d/%d (%.1fs)" % (len(ok_list), len(results), elapsed)) + print("-" * 90) + for s in ok_list: + print(" [OK] %-16s %-14s %-28s %.3fs %s" % ( + s["ip"], "/".join(s.get("protocols", [])), s["name"], + s["latency"], s.get("handshake") or "")) + if fail_list: + print("\n不可用: %d台" % len(fail_list)) + for s in fail_list: + print(" [X] %-16s %s" % (s["ip"], "/".join(s.get("protocols", [])))) + print("\n已写回 %s" % SERVERS_FILE) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/pytdxdata/servers.json b/src/pytdxdata/servers.json new file mode 100644 index 0000000..9027105 --- /dev/null +++ b/src/pytdxdata/servers.json @@ -0,0 +1,488 @@ +{ + "version": "2026-08-12", + "note": "当日全量测速入库(2026-09-08): 入池=9台 (排名前10或延迟<200ms)", + "servers": [ + { + "ip": "139.9.52.158", + "name": "国泰君安深圳", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "ok", + "latency": null, + "checked_at": "2026-08-21 11:23:14", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-21" + }, + { + "ip": "103.251.85.94", + "name": "国泰君安上海", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 0.481, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "58.67.221.146", + "name": "中信短线宝21", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "ok", + "latency": 0.254, + "checked_at": "2026-09-08 10:05:12", + "source": "probe-2026-08-12", + "fast_pool": true, + "checked_date": "2026-09-08" + }, + { + "ip": "59.36.5.11", + "name": "安信", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "ok", + "latency": 0.254, + "checked_at": "2026-09-08 10:05:12", + "source": "probe-2026-08-12", + "fast_pool": true, + "checked_date": "2026-09-08" + }, + { + "ip": "103.221.142.82", + "name": "国泰君安江苏", + "protocols": [ + "standard", + "mac", + "ex" + ], + "handshake": "setup", + "status": "slow", + "latency": 0.487, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "218.106.92.182", + "name": "华林", + "protocols": [ + "standard" + ], + "handshake": null, + "status": "ok", + "latency": 0.371, + "checked_at": "2026-09-08 10:05:12", + "source": "probe-2026-08-12", + "fast_pool": true, + "checked_date": "2026-09-08" + }, + { + "ip": "60.12.136.251", + "name": "中信短线宝22", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "ok", + "latency": 0.37, + "checked_at": "2026-09-08 10:05:12", + "source": "probe-2026-08-12", + "fast_pool": true, + "checked_date": "2026-09-08" + }, + { + "ip": "218.106.92.183", + "name": "华林", + "protocols": [ + "standard" + ], + "handshake": null, + "status": "ok", + "latency": 0.377, + "checked_at": "2026-09-08 10:05:12", + "source": "probe-2026-08-12", + "fast_pool": true, + "checked_date": "2026-09-08" + }, + { + "ip": "60.12.136.250", + "name": "中信短线宝26", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "ok", + "latency": 0.373, + "checked_at": "2026-09-08 10:05:12", + "source": "probe-2026-08-12", + "fast_pool": true, + "checked_date": "2026-09-08" + }, + { + "ip": "115.238.90.170", + "name": "中信短线宝23", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 0.482, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "220.178.55.86", + "name": "华林", + "protocols": [ + "standard" + ], + "handshake": null, + "status": "slow", + "latency": 0.479, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "220.178.55.71", + "name": "华林", + "protocols": [ + "standard" + ], + "handshake": null, + "status": "slow", + "latency": 0.47, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "180.153.18.170", + "name": "中信短线宝1", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 0.438, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "115.238.90.165", + "name": "杭州J4", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 0.476, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "60.191.117.167", + "name": "杭州J1", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "ok", + "latency": 0.362, + "checked_at": "2026-09-08 10:05:12", + "source": "probe-2026-08-12", + "fast_pool": true, + "checked_date": "2026-09-08" + }, + { + "ip": "218.75.126.9", + "name": "杭州J3", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 0.481, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "115.238.56.198", + "name": "杭州J2", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "ok", + "latency": 0.362, + "checked_at": "2026-09-08 10:05:12", + "source": "probe-2026-08-12", + "fast_pool": true, + "checked_date": "2026-09-08" + }, + { + "ip": "117.34.114.16", + "name": "国泰君安", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 1.575, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "159.75.55.232", + "name": "通达信深圳9", + "protocols": [ + "standard" + ], + "handshake": null, + "status": "ok", + "latency": 0.244, + "checked_at": "2026-09-08 10:05:12", + "source": "probe-2026-08-12", + "fast_pool": true, + "checked_date": "2026-09-08" + }, + { + "ip": "117.34.114.17", + "name": "国泰君安", + "protocols": [ + "standard" + ], + "handshake": null, + "status": "slow", + "latency": 1.573, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "117.34.114.14", + "name": "国泰君安", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 0.508, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "117.34.114.15", + "name": "国泰君安", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 0.636, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "117.34.114.13", + "name": "国泰君安", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 0.608, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "117.34.114.27", + "name": "国泰君安", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 1.138, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "117.34.114.18", + "name": "国泰君安", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 0.969, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "117.34.114.20", + "name": "国泰君安", + "protocols": [ + "standard", + "mac" + ], + "handshake": null, + "status": "slow", + "latency": 0.526, + "checked_at": "2026-08-19 15:37:35", + "source": "probe-2026-08-12", + "fast_pool": false, + "checked_date": "2026-08-19" + }, + { + "ip": "117.34.114.30", + "name": "国泰君安", + "protocols": [ + "standard" + ], + "handshake": null, + "status": "fail", + "latency": null, + "checked_at": "2026-08-12 18:36:21", + "source": "probe-2026-08-12" + }, + { + "ip": "121.37.207.165", + "name": "MAC专用", + "protocols": [ + "mac" + ], + "handshake": null, + "status": "ok", + "latency": 0.249, + "checked_at": "2026-08-12 18:36:17", + "source": "probe-2026-08-12" + }, + { + "ip": "121.36.248.138", + "name": "MAC专用", + "protocols": [ + "mac" + ], + "handshake": null, + "status": "ok", + "latency": 0.436, + "checked_at": "2026-08-12 18:36:17", + "source": "probe-2026-08-12" + }, + { + "ip": "123.60.47.136", + "name": "MAC专用", + "protocols": [ + "mac" + ], + "handshake": null, + "status": "ok", + "latency": 0.476, + "checked_at": "2026-08-12 18:36:17", + "source": "probe-2026-08-12" + }, + { + "ip": "116.205.135.205", + "name": "MAC EX专用", + "protocols": [ + "ex" + ], + "handshake": "login", + "status": "ok", + "latency": 0.192, + "checked_at": "2026-08-12 18:36:17", + "source": "probe-2026-08-12" + }, + { + "ip": "121.37.232.167", + "name": "MAC EX专用", + "protocols": [ + "ex" + ], + "handshake": "login", + "status": "ok", + "latency": 0.194, + "checked_at": "2026-08-12 18:36:17", + "source": "probe-2026-08-12" + }, + { + "ip": "116.205.143.214", + "name": "通达信扩展市场广州1", + "protocols": [ + "ex" + ], + "handshake": "setup", + "status": "ok", + "latency": 0.192, + "checked_at": "2026-08-12 18:36:17", + "source": "probe-2026-08-12" + }, + { + "ip": "s1", + "name": "s1", + "protocols": [ + "standard" + ], + "status": "unknown", + "latency": null, + "fast_pool": false, + "checked_date": "2026-08-19", + "checked_at": "2026-08-19 15:34:45" + } + ], + "checked_date": "2026-09-08" +} \ No newline at end of file diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..59f3822 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,331 @@ +"""统一接口 TdxData 测试(mock 连接,不连真实网络)+ 分页调度器测试""" +from __future__ import annotations + +import asyncio +import datetime +import unittest.mock as mock + +import pytest + +from pytdxdata.api import TdxData +from pytdxdata.models import KlinePeriod, Market, MinuteBar, SecurityBar +from pytdxdata.protocol.commands.mac_tick import MacTickChartCmd +from pytdxdata.scheduler.paginator import PageSpec, Paginator + + +def _make_bar(i: int) -> SecurityBar: + return SecurityBar(market=1, code="600000", open=10.0, high=10.1, low=9.9, + close=10.0 + i * 0.01, vol=1000, amount=10000, + year=2026, month=8, day=1 + i // 240, hour=9, minute=i % 60) + + +# ---------- Paginator ---------- + +class FakePoolConn: + """模拟连接:返回按start切片的数据。""" + def __init__(self, total_bars: int, page_size: int, fail_pages: set[int] | None = None): + self.total = total_bars + self.page_size = page_size + self.fail_pages = fail_pages or set() + self.executed: list[tuple[int, int]] = [] + + async def execute(self, cmd): + start, count = cmd # cmd 即 (start, count) 元组 + self.executed.append((start, count)) + if start // self.page_size in self.fail_pages: + raise ConnectionError("mock失败") + return [_make_bar(i) for i in range(start, min(start + count, self.total))] + + +class FakePool: + def __init__(self, conns: list[FakePoolConn]): + self.conns = conns + self.released: list = [] + self._acquire_idx = 0 + + async def acquire_many(self, n, *, timeout=None): + return self.conns[:n] + + async def acquire(self, *, timeout=None): + pc = self.conns[self._acquire_idx % len(self.conns)] + self._acquire_idx += 1 + return pc + + async def release(self, pc, *, broken=False): + self.released.append((pc, broken)) + + +@pytest.mark.asyncio +async def test_paginator_parallel_assembly(): + """3连接并行拉2400根(3页×800),按序拼接。""" + conns = [FakePoolConn(2400, 800) for _ in range(3)] + pool = FakePool(conns) + pag = Paginator(pool, concurrency=3) # type: ignore[arg-type] + + def factory(s, c): + return (s, c) + + result = await pag.fetch_all(factory, PageSpec(page_size=800, total=2400)) + assert len(result) == 2400 + # 按序:前800根是0-799 + assert result[0].close == 10.0 + assert result[799].close == pytest.approx(10.0 + 799 * 0.01) + assert result[800].close == pytest.approx(10.0 + 800 * 0.01) + # 每个连接都用了(并发分页发生) + assert all(len(c.executed) >= 1 for c in conns) + + +@pytest.mark.asyncio +async def test_paginator_retry_and_strict(): + """第1页失败→strict=True抛错。""" + conns = [FakePoolConn(2400, 800, fail_pages={0}) for _ in range(3)] + pool = FakePool(conns) + pag = Paginator(pool, concurrency=3, retries=1, strict=True) # type: ignore[arg-type] + + def factory(s, c): + return (s, c) + + with pytest.raises(ConnectionError): + await pag.fetch_all(factory, PageSpec(page_size=800, total=2400)) + + +@pytest.mark.asyncio +async def test_paginator_strict_false(): + conns = [FakePoolConn(2400, 800, fail_pages={0}) for _ in range(3)] + pool = FakePool(conns) + pag = Paginator(pool, concurrency=3, retries=0, strict=False) # type: ignore[arg-type] + + def factory(s, c): + return (s, c) + + result = await pag.fetch_all(factory, PageSpec(page_size=800, total=2400)) + assert len(result) == 1600 # 第0页留空,1-2页成功 + + +# ---------- TdxData 统一接口 ---------- + +class FakeApiConn: + """模拟 execute:按命令类型返回。""" + def __init__(self, server="s1"): + self.server = server + self.host = server + self.alive_flag = True + + @property + def alive(self): + return self.alive_flag + + async def connect(self): + pass + + async def execute(self, cmd): + cname = type(cmd).__name__ + if "GetSecurityBarsCmd" in cname: + return [_make_bar(i) for i in range(cmd.start, cmd.start + cmd.count)] + if "GetSecurityQuotesCmd" in cname: + return [] + if "MacTransactionCmd" in cname: + return [] + if "GetSecurityCountCmd" in cname: + return 100 + if "GetXdxrInfoCmd" in cname: + return [] + if "GetFinanceInfoCmd" in cname: + return None + return [] + + async def close(self): + self.alive_flag = False + + +@pytest.mark.asyncio +async def test_tdxdata_get_kline_with_cache(): + """get_kline:第一次回源,第二次缓存命中(不重复请求)。""" + with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \ + mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \ + mock.patch("pytdxdata.pool.pool.TdxConnection", + side_effect=lambda h, p=7709, t=10.0: FakeApiConn(h)): + td = TdxData(cache_dir=None, pool_min=1, pool_max=2) + await td.start() + try: + bars1 = await td.get_kline(1, "600000", KlinePeriod.DAY, count=100) + bars2 = await td.get_kline(1, "600000", KlinePeriod.DAY, count=100) + assert len(bars1) == 100 + assert len(bars2) == 100 + assert bars1[0] == bars2[0] + finally: + await td.close() + + +class MinuteApiConn(FakeApiConn): + """捕获 get_minute 实际下发的命令(类级,跨实例共享)。""" + + last_minute_cmd = None + + async def execute(self, cmd): + if "MacTickChartCmd" in type(cmd).__name__: + MinuteApiConn.last_minute_cmd = cmd + return [MinuteBar(price=9.14, vol=553.0, hour=9, minute=30)] + return await super().execute(cmd) + + +@pytest.mark.asyncio +async def test_get_minute_default_uses_history_cmd(): + """date=None 走 MAC 通道 MacTickChartCmd(今天):0x122D当日分时。""" + MinuteApiConn.last_minute_cmd = None + with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \ + mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \ + mock.patch("pytdxdata.pool.pool.TdxConnection", + side_effect=lambda h, p=7709, t=10.0: MinuteApiConn(h)): + td = TdxData(cache_dir=None, pool_min=1, pool_max=2, ex_servers=[]) + await td.start() + try: + bars = await td.get_minute(0, "000001") + assert len(bars) == 1 + assert bars[0].price == 9.14 + cmd = MinuteApiConn.last_minute_cmd + assert isinstance(cmd, MacTickChartCmd) + today = datetime.date.today() + assert cmd.date == today.year * 10000 + today.month * 100 + today.day + finally: + await td.close() + + +@pytest.mark.asyncio +async def test_tdxdata_get_security_count(): + with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \ + mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \ + mock.patch("pytdxdata.pool.pool.TdxConnection", + side_effect=lambda h, p=7709, t=10.0: FakeApiConn(h)): + td = TdxData(cache_dir=None, pool_min=1, pool_max=2) + await td.start() + try: + n = await td.get_security_count(0) + assert n == 100 + finally: + await td.close() + + +# ---------- Paginator start_offset ---------- + +@pytest.mark.asyncio +async def test_paginator_start_offset(): + """start_offset 让分页从指定偏移开始,跳过前面的数据。""" + conns = [FakePoolConn(5000, 800) for _ in range(3)] + pool = FakePool(conns) + pag = Paginator(pool, concurrency=3) # type: ignore[arg-type] + + def factory(s, c): + return (s, c) + + # 从偏移 4000 开始拉 800 根(total=4800) + result = await pag.fetch_all(factory, PageSpec( + page_size=800, total=4800, start_offset=4000)) + assert len(result) == 800 + assert result[0].close == pytest.approx(10.0 + 4000 * 0.01) + # 验证没有拉取 offset < 4000 的数据 + starts_used = [s for conn in conns for s, _ in conn.executed] + assert all(s >= 4000 for s in starts_used) + + +@pytest.mark.asyncio +async def test_paginator_start_offset_default_zero(): + """不传 start_offset 时行为不变(从0开始)。""" + conns = [FakePoolConn(2400, 800) for _ in range(3)] + pool = FakePool(conns) + pag = Paginator(pool, concurrency=3) # type: ignore[arg-type] + + def factory(s, c): + return (s, c) + + result = await pag.fetch_all(factory, PageSpec(page_size=800, total=2400)) + assert len(result) == 2400 + assert result[0].close == 10.0 # 从 offset 0 开始 + + +# ---------- get_kline 日期过滤 ---------- + +class DateBarApiConn(FakeApiConn): + """支持日期范围探测的模拟连接(按偏移返回带日期的K线)。""" + # 类级可调:总共多少条、每条的日期序列 + total_bars: int = 500 + + async def execute(self, cmd): + cname = type(cmd).__name__ + if "GetSecurityBarsCmd" in cname: + start, count = cmd.start, cmd.count + result = [] + for i in range(start, min(start + count, self.total_bars)): + # 每条间隔1天:偏移0=2026-01-01, 偏移1=2026-01-02, ... + dt = datetime.datetime(2026, 1, 1) + datetime.timedelta(days=i) + result.append(SecurityBar( + market=1, code="600000", open=10.0, high=10.1, low=9.9, + close=10.0 + i * 0.01, vol=1000, amount=10000, + year=dt.year, month=dt.month, day=dt.day, + hour=9, minute=30)) + return result + return await super().execute(cmd) + + +@pytest.mark.asyncio +async def test_get_kline_start_date_filters(): + """get_kline(start_date=...) 只返回该日期之后的K线。""" + DateBarApiConn.total_bars = 500 + with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \ + mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \ + mock.patch("pytdxdata.pool.pool.TdxConnection", + side_effect=lambda h, p=7709, t=10.0: DateBarApiConn(h)): + td = TdxData(cache_dir=None, pool_min=1, pool_max=2) + await td.start() + try: + bars = await td.get_kline( + 1, "600000", KlinePeriod.DAY, + start_date=datetime.datetime(2026, 3, 1)) + # 2026-01-01 偏移0 → 2026-03-01 偏移59,所以 >= 偏移59 + assert len(bars) > 0 + assert bars[0].year == 2026 and bars[0].month == 3 and bars[0].day == 1 + finally: + await td.close() + + +@pytest.mark.asyncio +async def test_get_kline_date_range(): + """get_kline(start_date=..., end_date=...) 返回日期范围内的K线。""" + DateBarApiConn.total_bars = 500 + with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \ + mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \ + mock.patch("pytdxdata.pool.pool.TdxConnection", + side_effect=lambda h, p=7709, t=10.0: DateBarApiConn(h)): + td = TdxData(cache_dir=None, pool_min=1, pool_max=2) + await td.start() + try: + bars = await td.get_kline( + 1, "600000", KlinePeriod.DAY, + start_date=datetime.datetime(2026, 2, 1), + end_date=datetime.datetime(2026, 2, 28)) + assert len(bars) == 28 + assert bars[0].month == 2 and bars[0].day == 1 + assert bars[-1].month == 2 and bars[-1].day == 28 + finally: + await td.close() + + +@pytest.mark.asyncio +async def test_get_kline_since(): + """get_kline_since 便捷方法:返回指定日期之后的所有K线。""" + DateBarApiConn.total_bars = 500 + with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \ + mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \ + mock.patch("pytdxdata.pool.pool.TdxConnection", + side_effect=lambda h, p=7709, t=10.0: DateBarApiConn(h)): + td = TdxData(cache_dir=None, pool_min=1, pool_max=2) + await td.start() + try: + bars = await td.get_kline_since( + 1, "600000", KlinePeriod.DAY, datetime.datetime(2026, 6, 1)) + assert len(bars) > 0 + assert bars[0].year == 2026 and bars[0].month == 6 and bars[0].day == 1 + # 最后一条应该是偏移499=2027-04-16 + assert bars[-1].year == 2027 + finally: + await td.close() diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..fd5798e --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,144 @@ +"""缓存单元测试:TTL/LRU/双层/single-flight""" +from __future__ import annotations + +import asyncio +import tempfile +from pathlib import Path + +import pytest + +from pytdxdata.cache.disk import DiskCache +from pytdxdata.cache.key import CacheKey +from pytdxdata.cache.manager import CacheManager +from pytdxdata.cache.memory import MemoryCache +from pytdxdata.cache.ttl import ttl_for + + +# ---------- CacheKey ---------- + +def test_key_roundtrip(): + k = CacheKey("kline", 1, "600000", "DAY", 0, 2400, "QFQ") + s = k.to_str() + assert CacheKey.from_str(s) == k + + +def test_quote_key_order_independent(): + a = CacheKey.quote_key([(0, "000001"), (1, "600000")]) + b = CacheKey.quote_key([(1, "600000"), (0, "000001")]) + assert a.to_str() == b.to_str() + + +# ---------- TTL ---------- + +def test_ttl_quote(): + assert ttl_for("quote") == 5.0 + + +def test_ttl_kline_minute(): + assert ttl_for("kline", period="MIN_1") == 60.0 + + +def test_ttl_kline_day(): + t = ttl_for("kline", period="DAY") + assert 0 < t <= 12 * 3600 + + +def test_ttl_hist_transaction_permanent(): + assert ttl_for("transaction", date=20260101) == 0.0 + + +def test_ttl_today_transaction(): + assert ttl_for("transaction") == 60.0 + + +# ---------- MemoryCache ---------- + +def test_memory_ttl_expiry(): + mc = MemoryCache() + mc.set("k", b"v", ttl=0.1) + assert mc.get("k") == b"v" + import time + time.sleep(0.15) + assert mc.get("k") is None + + +def test_memory_lru_eviction(): + mc = MemoryCache(max_entries=2) + mc.set("a", b"1", 10) + mc.set("b", b"2", 10) + mc.set("c", b"3", 10) + assert len(mc) <= 2 + assert mc.get("a") is None # 最旧被逐出 + + +def test_memory_permanent(): + mc = MemoryCache() + mc.set("k", b"v", ttl=0) # 永久 + assert mc.get("k") == b"v" + + +# ---------- CacheManager + single-flight ---------- + +@pytest.mark.asyncio +async def test_manager_double_layer(): + tmp = Path(tempfile.mkdtemp()) / "cache.db" + mem = MemoryCache() + disk = DiskCache(tmp) + mgr = CacheManager(mem, disk) + await mgr.start() + + key = CacheKey("kline", 1, "600000", "DAY", 0, 800) + fetch_count = 0 + + async def fetcher(): + nonlocal fetch_count + fetch_count += 1 + return b"payload" + + v1 = await mgr.get_or_fetch(key, fetcher) + v2 = await mgr.get_or_fetch(key, fetcher) + assert v1 == v2 == b"payload" + assert fetch_count == 1 # 第二次命中缓存 + + # 磁盘持久:新管理器读同一磁盘库 + mgr2 = CacheManager(MemoryCache(), DiskCache(tmp)) + await mgr2.start() + v3 = await mgr2.get(key) + assert v3 == b"payload" + await mgr.close() + await mgr2.close() + + +@pytest.mark.asyncio +async def test_manager_single_flight(): + mem = MemoryCache() + mgr = CacheManager(mem, None) + key = CacheKey("quote", extra="0:000001") + fetch_count = 0 + + async def fetcher(): + nonlocal fetch_count + fetch_count += 1 + await asyncio.sleep(0.05) + return b"x" + + await asyncio.gather( + mgr.get_or_fetch(key, fetcher), + mgr.get_or_fetch(key, fetcher), + mgr.get_or_fetch(key, fetcher), + ) + assert fetch_count == 1 # single-flight 只回源一次 + await mgr.close() + + +@pytest.mark.asyncio +async def test_manager_invalidate(): + mem = MemoryCache() + mgr = CacheManager(mem, None) + key = CacheKey("kline", 1, "600000", "DAY", 0, 800) + await mgr.set(key, b"data") + assert await mgr.get(key) == b"data" + n = await mgr.invalidate("kline", market=1, code="600000") + assert n >= 1 + assert await mgr.get(key) is None + await mgr.close() diff --git a/tests/test_cli_files.py b/tests/test_cli_files.py new file mode 100644 index 0000000..210efdb --- /dev/null +++ b/tests/test_cli_files.py @@ -0,0 +1,158 @@ +"""CLI 服务器文件命令测试:block / industry / financial-list / financial。 + +全部离线:用 FakeTD 提供异步桩,不发任何网络请求。 +""" +from __future__ import annotations + +from pytdxdata.cli import cmd_block, cmd_financial, cmd_financial_list, cmd_industry +from pytdxdata.models import FinancialFileInfo, FinancialRecord, IndustryInfo, TdxBlock + +_BLOCKS = [ + TdxBlock(name="人工智能", category=2, count=3, codes=["300308", "002415", "688111"]), + TdxBlock(name="白酒", category=0, count=1, codes=["600519"]), +] +_INDUSTRY = { + "600000": IndustryInfo(code="600000", tdx_industry="880305", sw_industry="801780"), + "000001": IndustryInfo(code="000001", tdx_industry="880471", sw_industry=""), +} +_FILES = [FinancialFileInfo(filename="gpcw20260331.zip", hash="abc123", filesize=1234567)] +_RECORDS = [ + FinancialRecord(code="000001", market=0, report_date=20260331, fields=[1.0, 2.5, 3.5, 4.0]), + FinancialRecord(code="600000", market=1, report_date=20260331, fields=[9.0, 8.0]), +] + + +class FakeTD: + """最小异步桩:只实现 4 个服务器文件方法。""" + + def __init__(self, blocks=None, industry=None, files=None, records=None): + self.blocks = _BLOCKS if blocks is None else blocks + self.industry = _INDUSTRY if industry is None else industry + self.files = _FILES if files is None else files + self.records = _RECORDS if records is None else records + + async def get_block_parsed(self, filename): + return self.blocks + + async def get_industry_map(self): + return self.industry + + async def get_financial_file_infos(self): + return self.files + + async def get_financial_records_parsed(self, filename): + return self.records + + +# --------------------------------------------------------------------------- # +# block +# --------------------------------------------------------------------------- # + +async def test_cmd_block_basic(capsys): + await cmd_block(FakeTD(), "block_gn.dat", 30, False) + out = capsys.readouterr().out + assert "block_gn.dat 板块文件 (2 个板块)" in out + assert "概念" in out and "人工智能" in out # category=2 → 概念 + assert "行业" in out and "白酒" in out # category=0 → 行业 + assert "300308" in out + assert "其余" not in out + + +async def test_cmd_block_limit_and_codes(capsys): + await cmd_block(FakeTD(), "block_gn.dat", 1, True) + out = capsys.readouterr().out + assert "人工智能" in out + assert "白酒" not in out # 被 --limit 1 截掉 + assert "其余 1 个板块省略" in out + assert "300308 002415 688111" in out # --codes 展开完整成分 + + +async def test_cmd_block_unknown_category_and_empty(capsys): + weird = [TdxBlock(name="神秘", category=9, count=0, codes=[])] + await cmd_block(FakeTD(blocks=weird), "block_x.dat", 30, False) + assert "?" in capsys.readouterr().out + + await cmd_block(FakeTD(blocks=[]), "block_gn.dat", 30, False) + assert "0 个板块" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- # +# industry +# --------------------------------------------------------------------------- # + +async def test_cmd_industry_all(capsys): + await cmd_industry(FakeTD(), [], 30, None) + out = capsys.readouterr().out + assert "行业分类 tdxhy.cfg (共 2 只)" in out + assert "600000" in out and "880305" in out and "801780" in out + assert "880471" in out and "-" in out # 申万缺失 → "-" + + +async def test_cmd_industry_codes_and_missing(capsys): + await cmd_industry(FakeTD(), ["600000", "999999"], 30, None) + out = capsys.readouterr().out + assert "600000" in out and "通达信=880305" in out + assert "999999" in out and "未收录" in out + + +async def test_cmd_industry_search_and_limit(capsys): + await cmd_industry(FakeTD(), [], 30, "880305") + out = capsys.readouterr().out + assert "匹配 '880305' 1 条" in out + assert "600000" in out and "000001" not in out + + await cmd_industry(FakeTD(), [], 1, None) + assert "其余 1 条省略" in capsys.readouterr().out + + await cmd_industry(FakeTD(industry={}), [], 30, None) + assert "共 0 只" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- # +# financial-list +# --------------------------------------------------------------------------- # + +async def test_cmd_financial_list(capsys): + await cmd_financial_list(FakeTD(), 30) + out = capsys.readouterr().out + assert "gpcw.txt, 共 1 个" in out + assert "gpcw20260331.zip" in out and "abc123" in out and "1234567" in out + assert "tdx financial gpcw20260331.zip" in out + + +async def test_cmd_financial_list_limit_and_empty(capsys): + extra = _FILES + [FinancialFileInfo(filename="gpcw20251231.zip", hash="def", filesize=1)] + await cmd_financial_list(FakeTD(files=extra), 1) + assert "其余 1 个省略" in capsys.readouterr().out + + await cmd_financial_list(FakeTD(files=[]), 30) + assert "共 0 个" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- # +# financial +# --------------------------------------------------------------------------- # + +async def test_cmd_financial_basic(capsys): + await cmd_financial(FakeTD(), "gpcw20260331.zip", None, 5, 4) + out = capsys.readouterr().out + assert "gpcw20260331.zip 财报记录 (2 条)" in out + assert "市场=深" in out and "市场=沪" in out + assert "报告期=20260331" in out + assert "1 2.5 3.5 4" in out + + +async def test_cmd_financial_filter_and_fields_zero(capsys): + await cmd_financial(FakeTD(), "gpcw20260331.zip", "600000", 5, 0) + out = capsys.readouterr().out + assert "(1 条)" in out + assert "600000" in out and "000001" not in out + assert "字段数=2" in out + + +async def test_cmd_financial_limit_and_empty(capsys): + await cmd_financial(FakeTD(), "gpcw20260331.zip", None, 1, 4) + assert "其余 1 条省略" in capsys.readouterr().out + + await cmd_financial(FakeTD(records=[]), "gpcw20260331.zip", None, 5, 4) + assert "(0 条)" in capsys.readouterr().out diff --git a/tests/test_codec_files.py b/tests/test_codec_files.py new file mode 100644 index 0000000..2bc0f18 --- /dev/null +++ b/tests/test_codec_files.py @@ -0,0 +1,319 @@ +"""服务器文件解析器测试:板块 .dat / 行业 tdxhy.cfg / 历史专业财报 gpcw。 + +全部离线构造二进制样本,不连网络。 +""" +from __future__ import annotations + +import io +import struct +import unittest.mock as mock +import zipfile + +import pytest + +from pytdxdata.api import TdxData +from pytdxdata.codec.block import infer_block_category, parse_block_dat +from pytdxdata.codec.financial import parse_financial_dat, parse_financial_file_list +from pytdxdata.codec.industry import parse_tdxhy_cfg +from pytdxdata.models import FinancialFileInfo, FinancialRecord, IndustryInfo, TdxBlock + +# --------------------------------------------------------------------------- # +# 样本构造工具 +# --------------------------------------------------------------------------- # + +_HEADER = 384 +_RECORD = 2813 + + +def _make_block_dat(blocks: list[tuple[str, list[str], int, int]]) -> bytes: + """构造板块 .dat:blocks = [(name, codes, declared_count, type), ...]""" + out = bytearray(_HEADER) + out += struct.pack(" bytes: + """构造 gpcw*.dat:index_entries = [(code_bytes, market, offset), ...]""" + count = len(index_entries) if max_count is None else max_count + idx = b"".join(struct.pack("<6sBL", cb, m, off) for cb, m, off in index_entries) + header = struct.pack("<1hI1H3L", 0, header_report_date, count, 0, report_size, 0) + return header + idx + data_blob + + +def _make_financial_zip(dat_bytes: bytes, inner_name: str = "gpcw.dat") -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr(inner_name, dat_bytes) + return buf.getvalue() + + +# --------------------------------------------------------------------------- # +# parse_block_dat +# --------------------------------------------------------------------------- # + +def test_parse_block_dat_basic(): + data = _make_block_dat([ + ("银行", ["000001", "600036"], 2, 1), + ("房地产", ["000002"], 1, 1), + ]) + blocks = parse_block_dat(data, "block_zs.dat") + assert len(blocks) == 2 + assert isinstance(blocks[0], TdxBlock) + assert blocks[0].name == "银行" + assert blocks[0].category == 0 + assert blocks[0].count == 2 + assert blocks[0].codes == ["000001", "600036"] + assert blocks[1].name == "房地产" + assert blocks[1].codes == ["000002"] + + +@pytest.mark.parametrize( + ("filename", "expected"), + [ + ("block_zs.dat", 0), + ("block_gn.dat", 2), + ("block_fg.dat", 3), + ("mystery.dat", 0), + ("BLOCK_GN.DAT", 2), + ], +) +def test_infer_block_category(filename, expected): + assert infer_block_category(filename) == expected + + +def test_parse_block_dat_too_short(): + assert parse_block_dat(b"\x00" * 100) == [] + + +def test_parse_block_dat_count_exceeds_body(): + """声明 3 条但只带 1 条记录 -> 只解析出 1 条。""" + data = _make_block_dat([("银行", ["000001"], 1, 1)]) + data = data[: _HEADER] + struct.pack(" 空槽位被跳过。""" + data = _make_block_dat([("银行", ["000001", "600036"], 5, 1)]) + blocks = parse_block_dat(data, "block_zs.dat") + assert blocks[0].count == 5 + assert blocks[0].codes == ["000001", "600036"] + + +# --------------------------------------------------------------------------- # +# parse_tdxhy_cfg +# --------------------------------------------------------------------------- # + +def test_parse_tdxhy_cfg_basic(): + content = "0|000001|A01|||S01\n1|600036|A02|||S02\n".encode("gbk") + result = parse_tdxhy_cfg(content) + assert set(result) == {"000001", "600036"} + assert isinstance(result["000001"], IndustryInfo) + assert result["000001"].tdx_industry == "A01" + assert result["000001"].sw_industry == "S01" + + +def test_parse_tdxhy_cfg_short_line_skipped(): + """不足 3 列的行被跳过。""" + content = "0|000001\nbadline\n".encode("gbk") + assert parse_tdxhy_cfg(content) == {} + + +def test_parse_tdxhy_cfg_non_six_digit_skipped(): + content = "0|SH600036|A01|||S01\n0|60003|A01\n0|abcdef|A01|||S01\n".encode("gbk") + assert parse_tdxhy_cfg(content) == {} + + +def test_parse_tdxhy_cfg_missing_sw_column(): + """只有 5 列时申万行业为空串。""" + content = "0|000001|A01||\n".encode("gbk") + result = parse_tdxhy_cfg(content) + assert result["000001"].tdx_industry == "A01" + assert result["000001"].sw_industry == "" + + +def test_parse_tdxhy_cfg_empty(): + assert parse_tdxhy_cfg(b"") == {} + + +# --------------------------------------------------------------------------- # +# parse_financial_file_list +# --------------------------------------------------------------------------- # + +def test_parse_financial_file_list_basic(): + data = b"gpcw20260331.zip,abcdef012345,1024\ngpcw20251231.zip,deadbeef,2048\n" + infos = parse_financial_file_list(data) + assert len(infos) == 2 + assert isinstance(infos[0], FinancialFileInfo) + assert infos[0].filename == "gpcw20260331.zip" + assert infos[0].hash == "abcdef012345" + assert infos[0].filesize == 1024 + + +def test_parse_financial_file_list_skips_bad_lines(): + data = b"badline\nx,y,notanint\nok.zip,abc,10\n\n" + infos = parse_financial_file_list(data) + assert [i.filename for i in infos] == ["ok.zip"] + + +def test_parse_financial_file_list_empty(): + assert parse_financial_file_list(b"") == [] + + +# --------------------------------------------------------------------------- # +# parse_financial_dat +# --------------------------------------------------------------------------- # + +def _valid_two_record_dat() -> bytes: + # data 起点 = 20(header) + 2*11(index) = 42 + blob = struct.pack("<2f", 1.5, 2.5) + struct.pack("<2f", 3.0, 4.0) + return _make_dat_raw( + [(b"600000", 1, 42), (b"000001", 0, 50)], blob + ) + + +def test_parse_financial_dat_explicit_date(): + recs = parse_financial_dat(_valid_two_record_dat(), report_date=20260101) + assert len(recs) == 2 + assert isinstance(recs[0], FinancialRecord) + assert recs[0].code == "600000" + assert recs[0].market == 1 + assert recs[0].report_date == 20260101 + assert recs[0].fields == pytest.approx([1.5, 2.5]) + assert recs[1].code == "000001" + assert recs[1].market == 0 + assert recs[1].fields == pytest.approx([3.0, 4.0]) + + +def test_parse_financial_dat_fallback_date_from_header(): + recs = parse_financial_dat(_valid_two_record_dat(), report_date=0) + assert recs[0].report_date == 20260331 + + +def test_parse_financial_dat_too_short(): + assert parse_financial_dat(b"\x00" * 10) == [] + + +def test_parse_financial_dat_zero_report_size(): + data = _make_dat_raw([(b"600000", 1, 31)], b"", report_size=0) + assert parse_financial_dat(data) == [] + + +def test_parse_financial_dat_skips_zero_offset_and_empty_code(): + blob = struct.pack("<2f", 1.0, 2.0) + # 第 1 条 offset=0 跳过;第 2 条 code 全 0 跳过 + data = _make_dat_raw([(b"600000", 1, 0), (b"\x00" * 6, 1, 31)], blob) + assert parse_financial_dat(data) == [] + + +def test_parse_financial_dat_offset_out_of_range(): + data = _make_dat_raw([(b"600000", 1, 9999)], struct.pack("<2f", 1.0, 2.0)) + assert parse_financial_dat(data) == [] + + +def test_parse_financial_dat_max_count_breaks(): + """max_count 大于实际索引 -> 越界即 break。""" + blob = struct.pack("<2f", 1.0, 2.0) + data = _make_dat_raw([(b"600000", 1, 31)], blob, max_count=5) + recs = parse_financial_dat(data) + assert len(recs) == 1 + + +# --------------------------------------------------------------------------- # +# TdxData 解析接口 +# --------------------------------------------------------------------------- # + +def _new_tdxdata() -> TdxData: + with mock.patch("pytdxdata.api.get_standard_hosts", return_value=["s1"]), \ + mock.patch("pytdxdata.api.get_mac_hosts", return_value=["m1"]), \ + mock.patch("pytdxdata.api.get_ex_hosts", return_value=[]): + return TdxData(cache_dir=None, pool_min=1, pool_max=2, ex_servers=[]) + + +@pytest.mark.asyncio +async def test_api_get_block_parsed(): + td = _new_tdxdata() + raw = _make_block_dat([("银行", ["000001"], 1, 1)]) + with mock.patch.object(td, "get_block_info", new=mock.AsyncMock(return_value=raw)): + blocks = await td.get_block_parsed("block_gn.dat") + assert len(blocks) == 1 + assert blocks[0].category == 2 # gn -> 概念 + + +@pytest.mark.asyncio +async def test_api_get_industry_map(): + td = _new_tdxdata() + raw = "0|000001|A01|||S01\n".encode("gbk") + patched = mock.AsyncMock(return_value=raw) + with mock.patch.object(td, "get_report_file", new=patched): + result = await td.get_industry_map() + patched.assert_awaited_once_with("tdxhy.cfg") + assert result["000001"].sw_industry == "S01" + + +@pytest.mark.asyncio +async def test_api_get_financial_file_infos(): + td = _new_tdxdata() + raw = b"gpcw20260331.zip,abc,1024\n" + patched = mock.AsyncMock(return_value=raw) + with mock.patch.object(td, "get_report_file", new=patched): + infos = await td.get_financial_file_infos() + patched.assert_awaited_once_with("tdxfin/gpcw.txt") + assert infos[0].filesize == 1024 + + +@pytest.mark.asyncio +async def test_api_get_financial_records_parsed(): + td = _new_tdxdata() + zip_bytes = _make_financial_zip(_valid_two_record_dat()) + with mock.patch.object(td, "get_report_file", + new=mock.AsyncMock(return_value=zip_bytes)): + recs = await td.get_financial_records_parsed("tdxfin/gpcw20260331.zip") + assert len(recs) == 2 + assert recs[0].report_date == 20260331 # 取自文件名 + + +@pytest.mark.asyncio +async def test_api_get_financial_records_parsed_empty_zip_data(): + td = _new_tdxdata() + with mock.patch.object(td, "get_report_file", + new=mock.AsyncMock(return_value=b"")): + assert await td.get_financial_records_parsed("tdxfin/gpcw20260331.zip") == [] + + +@pytest.mark.asyncio +async def test_api_get_financial_records_parsed_no_dat_member(): + td = _new_tdxdata() + zip_bytes = _make_financial_zip(b"whatever", inner_name="readme.txt") + with mock.patch.object(td, "get_report_file", + new=mock.AsyncMock(return_value=zip_bytes)): + assert await td.get_financial_records_parsed("nodate.zip") == [] + + +@pytest.mark.asyncio +async def test_api_get_financial_records_parsed_no_date_in_name(): + """文件名无 8 位日期 -> 回退用 .dat 头部报告期。""" + td = _new_tdxdata() + zip_bytes = _make_financial_zip(_valid_two_record_dat()) + with mock.patch.object(td, "get_report_file", + new=mock.AsyncMock(return_value=zip_bytes)): + recs = await td.get_financial_records_parsed("gpcw.zip") + assert recs[0].report_date == 20260331 diff --git a/tests/test_pool.py b/tests/test_pool.py new file mode 100644 index 0000000..8e71b0c --- /dev/null +++ b/tests/test_pool.py @@ -0,0 +1,120 @@ +"""连接池/健康分单元测试(mock服务器,不连真实网络)""" +from __future__ import annotations + +import asyncio +import sys +import unittest.mock as mock + +import pytest + +from pytdxdata.pool.health import HealthEngine +from pytdxdata.pool.pool import DynamicPool, PoolTimeoutError + + +# ---------- HealthEngine ---------- + +def test_health_default_score(): + h = HealthEngine() + assert h.score("server1") == 1.0 + + +def test_health_failure_penalty(): + h = HealthEngine() + h.record_failure("s1") + assert h.score("s1") == 0.5 + h.record_failure("s1") + h.record_failure("s1") + assert h.in_cooldown("s1") # 连续3次失败进入冷却 + + +def test_health_success_recovery(): + h = HealthEngine() + h.record_failure("s1") + h.record_success("s1") + assert h.score("s1") == pytest.approx(0.7) + assert not h.in_cooldown("s1") + + +def test_health_rank(): + h = HealthEngine() + h.record_success("fast", latency=0.05) + h.record_success("slow", latency=0.5) + ranked = h.rank(["slow", "fast"]) + assert ranked[0] == "fast" + + +# ---------- DynamicPool ---------- + +class FakeConn: + def __init__(self, host): + self.host = host + self.alive_flag = True + + @property + def alive(self): + return self.alive_flag + + async def connect(self): + pass + + async def ping(self): + return 0.01 + + async def close(self): + self.alive_flag = False + + async def execute(self, cmd): + return "ok" + + +@pytest.fixture +def fake_pool(): + with mock.patch("pytdxdata.pool.pool.TdxConnection", side_effect=lambda h, p=7709, t=10.0: FakeConn(h)): + pool = DynamicPool(["s1", "s2"], min_size=1, max_size=3, per_server_cap=2, + idle_ttl=0.2, scale_down_grace=0, reaper_interval=0.05) + yield pool + + +@pytest.mark.asyncio +async def test_pool_acquire_release(fake_pool): + await fake_pool.start() + pc = await fake_pool.acquire() + assert pc.server in ("s1", "s2") + assert fake_pool.stats.borrowed == 1 + await fake_pool.release(pc) + assert fake_pool.stats.borrowed == 0 + await fake_pool.close() + + +@pytest.mark.asyncio +async def test_pool_acquire_many(fake_pool): + await fake_pool.start() + conns = await fake_pool.acquire_many(2) + assert len(conns) == 2 + servers = {c.server for c in conns} + for c in conns: + await fake_pool.release(c) + await fake_pool.close() + + +@pytest.mark.asyncio +async def test_pool_broken_release(fake_pool): + await fake_pool.start() + pc = await fake_pool.acquire() + await fake_pool.release(pc, broken=True) + assert fake_pool.stats.total <= fake_pool.min_size + await fake_pool.close() + + +@pytest.mark.asyncio +async def test_pool_scale_down(fake_pool): + """空闲连接超时后缩容到 min_size。""" + await fake_pool.start() + pc1 = await fake_pool.acquire() + await fake_pool.release(pc1) + pc2 = await fake_pool.acquire() + await fake_pool.release(pc2) + total_before = fake_pool.stats.total + await asyncio.sleep(0.3) # 等 reaper 缩容 + assert fake_pool.stats.total <= total_before + await fake_pool.close() diff --git a/tests/test_protocol.py b/tests/test_protocol.py new file mode 100644 index 0000000..8cd60e4 --- /dev/null +++ b/tests/test_protocol.py @@ -0,0 +1,178 @@ +"""协议层单元测试:帧/价格差分/浮点/命令解析(黄金字节向量,对照 easy-tdx 已验证实现)""" +from __future__ import annotations + +import struct + +import pytest + +from pytdxdata.protocol.frame import FrameHeader, decompress_body, parse_header +from pytdxdata.protocol.price import get_price, put_price, decode_ohlc_diffs +from pytdxdata.protocol.volume import decode_volume +from pytdxdata.protocol.commands.ex_proto import GetExInstrumentBarsCmd +from pytdxdata.protocol.commands.kline import GetSecurityBarsCmd +from pytdxdata.protocol.commands.minute import GetHistoryMinuteTimeDataCmd +from pytdxdata.protocol.commands.transaction import MacTransactionCmd + + +# ---------- 变长有符号整数 ---------- + +@pytest.mark.parametrize("value", [0, 1, -1, 63, 64, -64, 127, -128, 1000, -100000, 2**20]) +def test_price_roundtrip(value: int): + enc = put_price(value) + dec, pos = get_price(enc, 0) + assert dec == value + assert pos == len(enc) + + +def test_price_negative_flag(): + # 首字节 bit6=符号 + enc = put_price(-5) + assert enc[0] & 0x40 + dec, _ = get_price(enc, 0) + assert dec == -5 + + +# ---------- OHLC 差分还原 ---------- + +def test_decode_ohlc_diffs(): + # 手工构造:open_diff=100, close_diff=-20, high_diff=10, low_diff=-30 + body = bytearray() + for v in (100, -20, 10, -30): + body.extend(put_price(v)) + od, cd, hd, ld, pos = decode_ohlc_diffs(body, 0) + assert (od, cd, hd, ld) == (100, -20, 10, -30) + assert pos == len(body) + + +# ---------- 自定义4字节浮点 ---------- + +def test_decode_volume_zero(): + assert decode_volume(0) == 0.0 + + +def test_decode_volume_known(): + # 经验值:0x00000001 应为极小正数(非0) + assert decode_volume(1) > 0 + + +# ---------- 响应帧头 ---------- + +def test_frame_header(): + # magic=7654321, 未压缩: zipsize==unzipsize + buf = struct.pack(" bytes: + """构造 0x23FF 响应体:20B头(count@18) + 每条32B(4B时间 + 28B行情)。""" + body = bytearray(20 + len(raw_dates) * 32) + struct.pack_into("