fix(pool): 重写当日测速缓存:全量候选 + 用户级缓存 + 真实数据探针
原实现在 v0.5.0 已随包发布,但有三处会"静默失效"的缺陷,等于没真正生效: - 候选集自指:连接池把自己已过滤后的成员当探针输入,实测只重测了 9 台, 另外 20+ 台候选(含快节点)永远测不到。改为由 probe_cache 自行枚举 全量候选(包内清单 ∪ 代码常量 ∪ env),不再接受调用方传入的"当前池成员"。 - 写回安装目录:结果写回 src/pytdxdata/servers.json,容器/系统级安装下只读会 静默失败、多用户共用一个安装会互相覆盖,且污染随包分发的数据。改为写入 用户级缓存目录(Win %LOCALAPPDATA% / macOS ~/Library/Caches / Linux $XDG_CACHE_HOME), 运行时绝不再改写包内清单。 - 探测太弱:只做握手,测不出「握手正常但 K 线空响应」的坏节点。改为发真实业务 命令并要求返回非空(标准 GetSecurityBarsCmd / MAC MacKlineCmd / EX GetExInstrumentBarsCmd), EX 池(4 台,7727)也一并纳入。 无感保证:后台 task 不阻塞 start()、日志全 debug 级、绝不抛异常(取消除外)、 原子写 tmp + os.replace、进程内单例 + asyncio.Lock 单飞;全不可达时不写缓存 (否则"今天全挂"会被缓存一整天)。close() 会取消并 await 后台探针, 避免解释器退出时报 "Task was destroyed but it is pending"。 新增环境变量 PYPYTDX_DATA_CACHE_DIR(覆盖缓存目录)、 PYPYTDX_DATA_DISABLE_PROBE=1(关闭测速)。新增 tests/test_probe_cache.py(37 项)。
This commit is contained in:
1 parent
3e0beb27a6
commit
18d9e9a006
9 files changed
+941
-134
No files matched your search
+178
-13
@@ -1,12 +1,28 @@
|
||||
"""服务器清单与默认配置
|
||||
|
||||
服务器清单存于包内 servers.json,由 server_probe.py 导入时自动探测可用性与延时。
|
||||
config.py 加载时只采用 status="ok" 的服务器;环境变量可覆盖;代码内嵌清单为兜底。
|
||||
服务器清单存于包内 ``servers.json`` —— 它是**随包分发的基线清单,运行期只读**。
|
||||
每日首次启动的后台测速结果写入**用户级缓存目录**,运行期优先采用当日实测结果。
|
||||
|
||||
主机选取优先级(每个协议独立)::
|
||||
|
||||
环境变量 PYPYTDX_DATA_*_HOSTS > 当日测速缓存 > 包内清单 fast_pool
|
||||
> 包内清单 ok > 代码内嵌常量
|
||||
|
||||
相关环境变量:
|
||||
|
||||
- ``PYPYTDX_DATA_CACHE_DIR`` 覆盖用户级缓存目录(默认见 :func:`probe_cache_dir`)
|
||||
- ``PYPYTDX_DATA_DISABLE_PROBE`` 置真值时既不读当日缓存也不测速(CI / 测试用,保证确定性)
|
||||
|
||||
为什么不把测速结果写回包内 ``servers.json``:安装目录可能是只读的(容器 / 系统级
|
||||
site-packages / pipx),且多用户共用一份安装时彼此覆盖,运行期改写分发数据本身也不妥。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_PORT = 7709
|
||||
@@ -14,19 +30,29 @@ DEFAULT_TIMEOUT = 10.0
|
||||
|
||||
_SERVERS_FILE = Path(__file__).resolve().parent / "servers.json" # 包内清单(随包分发)
|
||||
|
||||
# 当日测速缓存(结构变更时递增,旧版本缓存自动失效)
|
||||
PROBE_CACHE_VERSION = 1
|
||||
_PROBE_CACHE_NAME = "hosts-today.json"
|
||||
|
||||
|
||||
# ---------- 包内清单(只读基线) ----------
|
||||
def _read_servers_json() -> dict:
|
||||
"""读取包内清单;文件缺失 / 内容损坏 / 顶层非对象 → 空 dict。"""
|
||||
try:
|
||||
cfg = json.loads(_SERVERS_FILE.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
return cfg if isinstance(cfg, dict) else {}
|
||||
|
||||
|
||||
def _load_ok_hosts(protocol: str, *, fast_only: bool = False) -> list[str]:
|
||||
"""从 servers.json 加载 status=ok 且支持指定协议的服务器。
|
||||
|
||||
fast_only=True 时只返回当日测速入池(fast_pool)的服务器(排名前N或延迟<阈值),
|
||||
fast_only=True 时只返回测速入池(fast_pool)的服务器(排名前N或延迟<阈值),
|
||||
用于动态连接池优先使用优质节点。
|
||||
"""
|
||||
try:
|
||||
cfg = json.loads(_SERVERS_FILE.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return []
|
||||
out = []
|
||||
for s in cfg.get("servers", []):
|
||||
for s in _read_servers_json().get("servers", []):
|
||||
if s.get("status") != "ok":
|
||||
continue
|
||||
if protocol not in s.get("protocols", []):
|
||||
@@ -38,13 +64,90 @@ def _load_ok_hosts(protocol: str, *, fast_only: bool = False) -> list[str]:
|
||||
|
||||
|
||||
def _load_ex_handshakes() -> dict[str, str]:
|
||||
try:
|
||||
cfg = json.loads(_SERVERS_FILE.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
return {s["ip"]: s.get("handshake", "setup") for s in cfg.get("servers", [])
|
||||
"""包内清单里 EX 服务器的握手模式。"""
|
||||
return {s["ip"]: s.get("handshake", "setup")
|
||||
for s in _read_servers_json().get("servers", [])
|
||||
if "ex" in s.get("protocols", []) and s.get("status") == "ok"}
|
||||
|
||||
|
||||
# ---------- 用户级当日测速缓存 ----------
|
||||
def probe_cache_dir() -> Path:
|
||||
"""用户级缓存目录(保证可写,绝不指向安装目录)。
|
||||
|
||||
``PYPYTDX_DATA_CACHE_DIR`` 可覆盖(测试指向临时目录)。按平台约定:
|
||||
|
||||
- Windows: ``%LOCALAPPDATA%\\pytdxdata``
|
||||
- macOS: ``~/Library/Caches/pytdxdata``
|
||||
- 其他: ``$XDG_CACHE_HOME/pytdxdata`` 或 ``~/.cache/pytdxdata``
|
||||
"""
|
||||
env = (os.environ.get("PYPYTDX_DATA_CACHE_DIR") or "").strip()
|
||||
if env:
|
||||
return Path(env).expanduser()
|
||||
if os.name == "nt":
|
||||
base = os.environ.get("LOCALAPPDATA") or os.environ.get("APPDATA")
|
||||
return (Path(base) if base else Path.home() / "AppData" / "Local") / "pytdxdata"
|
||||
if sys.platform == "darwin":
|
||||
return Path.home() / "Library" / "Caches" / "pytdxdata"
|
||||
base = os.environ.get("XDG_CACHE_HOME")
|
||||
return (Path(base) if base else Path.home() / ".cache") / "pytdxdata"
|
||||
|
||||
|
||||
def probe_cache_file() -> Path:
|
||||
"""当日测速缓存文件路径(每次调用重新解析,便于测试改环境变量后立即生效)。"""
|
||||
return probe_cache_dir() / _PROBE_CACHE_NAME
|
||||
|
||||
|
||||
def probe_disabled() -> bool:
|
||||
"""是否已通过 ``PYPYTDX_DATA_DISABLE_PROBE`` 关闭当日测速缓存。"""
|
||||
return os.environ.get("PYPYTDX_DATA_DISABLE_PROBE", "").strip().lower() in {
|
||||
"1", "true", "yes", "on",
|
||||
}
|
||||
|
||||
|
||||
def load_probe_cache(today: str | None = None,
|
||||
path: str | Path | None = None) -> dict | None:
|
||||
"""读取**当日**测速缓存;缺失 / 损坏 / 非今日 / 版本不符 → ``None``。
|
||||
|
||||
返回非 None 即表示"今天已经测过速了",调用方应直接复用而不再刷新。
|
||||
``path`` 可显式指定缓存文件(默认 :func:`probe_cache_file`),便于注入与测试。
|
||||
"""
|
||||
if probe_disabled():
|
||||
return None
|
||||
target = Path(path) if path is not None else probe_cache_file()
|
||||
try:
|
||||
cfg = json.loads(target.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not isinstance(cfg, dict) or cfg.get("version") != PROBE_CACHE_VERSION:
|
||||
return None
|
||||
if str(cfg.get("date", "")) != (today or date.today().isoformat()):
|
||||
return None
|
||||
results = cfg.get("results")
|
||||
if not isinstance(results, dict) or not results:
|
||||
return None
|
||||
return cfg
|
||||
|
||||
|
||||
def _load_cached_hosts(protocol: str) -> list[str]:
|
||||
"""当日测速缓存中该协议**实测可达**的主机,按延迟升序(越快越靠前)。"""
|
||||
cfg = load_probe_cache()
|
||||
if cfg is None:
|
||||
return []
|
||||
rows: list[tuple[float, str]] = []
|
||||
for ip, per_proto in cfg["results"].items():
|
||||
if not isinstance(per_proto, dict):
|
||||
continue
|
||||
lat = per_proto.get(protocol)
|
||||
if isinstance(lat, bool) or not isinstance(lat, (int, float)):
|
||||
continue
|
||||
if lat < 0:
|
||||
continue
|
||||
rows.append((float(lat), str(ip)))
|
||||
rows.sort(key=lambda kv: (kv[0], kv[1]))
|
||||
return [ip for _, ip in rows]
|
||||
|
||||
|
||||
# ---------- 代码内嵌兜底清单 ----------
|
||||
# 标准行情服务器 —— 真实K线探测验证可用(2026-08-12, GetSecurityBarsCmd 600000 日K)
|
||||
# 27台可用 = pytdxwrapper券商服务器(国泰君安/中信短线宝/华林/安信等) + 通达信官方主站
|
||||
# 来源①: pytdxwrapper config.py TDX_HOSTS(券商+官方, 51台探测27可用)
|
||||
@@ -91,10 +194,66 @@ EX_HANDSHAKES: dict[str, str] = {
|
||||
}
|
||||
|
||||
|
||||
# ---------- 测速候选枚举 ----------
|
||||
def is_valid_host(host: object) -> bool:
|
||||
"""是否是可用作服务器地址的合法主机(IPv4/IPv6 字面量,或含点的域名)。
|
||||
|
||||
用于挡住测试 mock 混进来的伪主机名(如 ``"s1"``)被写进缓存 / 清单。
|
||||
"""
|
||||
if not isinstance(host, str):
|
||||
return False
|
||||
h = host.strip()
|
||||
if not h or len(h) > 253 or any(c.isspace() for c in h):
|
||||
return False
|
||||
try:
|
||||
ipaddress.ip_address(h)
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
return "." in h and all(
|
||||
part and part[0] != "-" and part[-1] != "-" and part.replace("-", "").isalnum()
|
||||
for part in h.split(".")
|
||||
)
|
||||
|
||||
|
||||
def all_probe_candidates() -> dict[str, list[str]]:
|
||||
"""[测速用] 返回 ``{协议: [候选主机]}`` 全量清单(已校验、去重、保序)。
|
||||
|
||||
来源 = 包内清单申报的协议 ∪ 代码内嵌常量 ∪ 环境变量覆盖。
|
||||
|
||||
与 ``get_*_hosts()`` 的关键区别:这里**不做 status / 延迟过滤**。
|
||||
测速必须能看到全部候选,否则"上次慢、这次快"的节点永远捞不回来,
|
||||
形成自指循环(只重测已在池里的少数节点,其余记录一直过期)。
|
||||
"""
|
||||
out: dict[str, list[str]] = {"standard": [], "mac": [], "ex": []}
|
||||
for s in _read_servers_json().get("servers", []):
|
||||
ip = s.get("ip")
|
||||
for protocol in s.get("protocols", []):
|
||||
if protocol in out:
|
||||
out[protocol].append(ip)
|
||||
out["standard"] += STANDARD_HOSTS
|
||||
out["mac"] += MAC_HOSTS
|
||||
out["ex"] += EX_HOSTS
|
||||
for protocol, var in (("standard", "PYPYTDX_DATA_STANDARD_HOSTS"),
|
||||
("mac", "PYPYTDX_DATA_MAC_HOSTS"),
|
||||
("ex", "PYPYTDX_DATA_EX_HOSTS")):
|
||||
env = os.environ.get(var)
|
||||
if env:
|
||||
out[protocol] += env.split(",")
|
||||
return {
|
||||
protocol: [h for h in dict.fromkeys(hosts) if is_valid_host(h)]
|
||||
for protocol, hosts in out.items()
|
||||
}
|
||||
|
||||
|
||||
# ---------- 对外取主机 ----------
|
||||
def get_standard_hosts() -> list[str]:
|
||||
env = os.environ.get("PYPYTDX_DATA_STANDARD_HOSTS")
|
||||
if env:
|
||||
return [h.strip() for h in env.split(",") if h.strip()]
|
||||
cached = _load_cached_hosts("standard")
|
||||
if cached:
|
||||
return cached
|
||||
fast = _load_ok_hosts("standard", fast_only=True)
|
||||
if fast:
|
||||
return fast
|
||||
@@ -106,6 +265,9 @@ def get_mac_hosts() -> list[str]:
|
||||
env = os.environ.get("PYPYTDX_DATA_MAC_HOSTS")
|
||||
if env:
|
||||
return [h.strip() for h in env.split(",") if h.strip()]
|
||||
cached = _load_cached_hosts("mac")
|
||||
if cached:
|
||||
return cached
|
||||
fast = _load_ok_hosts("mac", fast_only=True)
|
||||
if fast:
|
||||
return fast
|
||||
@@ -117,6 +279,9 @@ def get_ex_hosts() -> list[str]:
|
||||
env = os.environ.get("PYPYTDX_DATA_EX_HOSTS")
|
||||
if env:
|
||||
return [h.strip() for h in env.split(",") if h.strip()]
|
||||
cached = _load_cached_hosts("ex")
|
||||
if cached:
|
||||
return cached
|
||||
from_json = _load_ok_hosts("ex")
|
||||
return from_json or list(EX_HOSTS)
|
||||
|
||||
|
||||
@@ -6,8 +6,9 @@
|
||||
- 动态:支持运行时 add() 新服务器;支持后台全量测速入库(当日首次启动)。
|
||||
- 单连接约束(核心需求):per_server_cap 默认 1 —— 一台服务器只允许建立 1 个连接,
|
||||
不允许多开,避免对单一券商服务器造成并发压力被限流/封禁。
|
||||
- 后台测速(核心需求):当日首次启动时,在后台对全量候选服务器测速,按
|
||||
「排名前 FAST_POOL_TOP_N 或 延迟 < FAST_POOL_MAX_LATENCY」筛选入库,结果写回 servers.json。
|
||||
- 后台测速(核心需求):当日首次启动时,在后台对**全量候选**服务器测速(覆盖标准/MAC/EX
|
||||
三个协议,而不是只测当前池内节点),结果缓存到用户级缓存目录;当天已有缓存则跳过、不再刷新。
|
||||
全程不阻塞 start()、不打印、不抛错(对用户无感)。详见 pool/probe_cache.py。
|
||||
- 弹性:空闲连接超时后由 reaper 缩容回 min_size;支持 stats 统计。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
@@ -19,7 +20,7 @@ import time
|
||||
|
||||
from .connection import TdxConnection
|
||||
from .health import HealthEngine
|
||||
from .probe_cache import get_probe_cache
|
||||
from .probe_cache import schedule_probe
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -101,6 +102,7 @@ class DynamicPool:
|
||||
self._lock = asyncio.Lock()
|
||||
self._cond = asyncio.Condition(self._lock)
|
||||
self._reaper_task: asyncio.Task | None = None
|
||||
self._probe_task: asyncio.Task | None = None
|
||||
|
||||
# ---------- 生命周期 ----------
|
||||
async def start(self) -> None:
|
||||
@@ -108,13 +110,11 @@ class DynamicPool:
|
||||
return
|
||||
self._started = True
|
||||
|
||||
# 需求①:当日首次启动 → 后台全量测速入库(不阻塞主流程)
|
||||
# 需求①:当日首次启动 → 后台全量测速(不阻塞主流程;当天已有缓存则自动跳过)
|
||||
# 候选清单由 probe_cache 自行枚举全量主机(本池的 servers 只是其中一个子集,
|
||||
# 不能直接当作测速输入,否则只会重测已在池内的少数节点、其余记录永久过期)。
|
||||
if self.auto_probe:
|
||||
hosts = list(self.servers)
|
||||
asyncio.create_task(
|
||||
get_probe_cache().ensure_probe(hosts),
|
||||
name="pytdxdata-daily-probe",
|
||||
)
|
||||
self._probe_task = schedule_probe()
|
||||
|
||||
targets = self._pick_servers(self.min_size)
|
||||
for host in targets:
|
||||
@@ -175,6 +175,15 @@ class DynamicPool:
|
||||
if self._reaper_task is not None:
|
||||
self._reaper_task.cancel()
|
||||
self._reaper_task = None
|
||||
# 后台测速仍可能持有 socket;取消并等它退场,避免事件循环关闭时冒出
|
||||
# "Task was destroyed but it is pending"(那会破坏"无感")
|
||||
if self._probe_task is not None:
|
||||
self._probe_task.cancel()
|
||||
try:
|
||||
await self._probe_task
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
self._probe_task = None
|
||||
async with self._lock:
|
||||
for pc in list(self._conns):
|
||||
try:
|
||||
|
||||
+272
-112
@@ -1,162 +1,322 @@
|
||||
"""当日首次启动的全量服务器测速与入库缓存。
|
||||
"""当日首次启动的全量服务器测速与缓存(对用户完全无感)。
|
||||
|
||||
设计要点:
|
||||
- 仅在"当日首次启动"时触发一次后台全量测速(按本地日期判断是否已探测)。
|
||||
- 测速采用真实握手连通(TdxConnection.test),比单纯 ping 更能反映通达信可用性。
|
||||
- 入库规则:延迟排名前 FAST_POOL_TOP_N(10) **或** 延迟 < FAST_POOL_MAX_LATENCY(0.2s) 的服务器
|
||||
才标记为 status="ok" 并写入 servers.json,其余降权(保留但不优先)。
|
||||
- 结果持久化到 servers.json,供 config / 连接池直接使用。
|
||||
行为
|
||||
----
|
||||
- **当日首次启动**触发一次后台全量测速;当天已有缓存则直接复用,**不再刷新**。
|
||||
- 测速覆盖**全部候选**(包内清单 ∪ 代码常量 ∪ env 覆盖,见
|
||||
:func:`pytdxdata.config.all_probe_candidates`),而不是只测当前已在池里的少数节点 ——
|
||||
否则"上次慢、这次快"的节点永远捞不回来。
|
||||
- 用**真实业务命令**判定可达(标准: GetSecurityBarsCmd / MAC: MacKlineCmd /
|
||||
EX: GetExInstrumentBarsCmd)并要求返回非空数据,而不是只做握手回显:
|
||||
存在"握手正常但 K线空响应"的坏节点(见 ``config.py`` 注释),只有真数据能测出来。
|
||||
- 结果写入**用户级缓存目录**(见 :func:`pytdxdata.config.probe_cache_file`),
|
||||
不写安装目录:site-packages 在容器 / 系统级安装 / pipx 下可能只读,
|
||||
多用户共用一份安装时还会互相覆盖。
|
||||
- 结果被 :func:`pytdxdata.config.get_standard_hosts` 等按**实测延迟升序**采用。
|
||||
|
||||
无感保证
|
||||
--------
|
||||
- 不打印任何东西;日志一律 ``debug`` 级(默认不输出)。
|
||||
- 不抛异常(取消除外);失败只落 debug 日志,下次启动自然重试。
|
||||
- 由连接池以后台 task 方式调用,不阻塞 ``start()`` 与任何用户请求。
|
||||
|
||||
一致性与并发
|
||||
------------
|
||||
- 进程内单例 + ``asyncio.Lock``:标准 / MAC / EX 三个池共用一次真正测速。
|
||||
- 原子写入(tmp + ``os.replace``):进程被杀或多进程并发都不会读到半截 JSON。
|
||||
- 全部不可达时**不写缓存**,避免把"全挂"状态缓存一整天。
|
||||
|
||||
开关
|
||||
----
|
||||
- ``PYPYTDX_DATA_DISABLE_PROBE=1`` 关闭(CI / 测试用,保证确定性)。
|
||||
- ``PYPYTDX_DATA_CACHE_DIR`` 覆盖缓存目录(测试指向临时目录)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from ..config import _SERVERS_FILE
|
||||
from ..config import (
|
||||
PROBE_CACHE_VERSION,
|
||||
all_probe_candidates,
|
||||
get_ex_handshakes,
|
||||
is_valid_host,
|
||||
load_probe_cache,
|
||||
probe_cache_file,
|
||||
probe_disabled,
|
||||
)
|
||||
from ..models import Adjust, KlinePeriod
|
||||
from ..protocol.commands.ex_proto import GetExInstrumentBarsCmd
|
||||
from ..protocol.commands.kline import GetSecurityBarsCmd
|
||||
from ..protocol.commands.mac import MacKlineCmd
|
||||
from .connection import TdxConnection
|
||||
from .ex_connection import ExTdxConnection
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# 入池阈值
|
||||
FAST_POOL_TOP_N = 10 # 延迟排名前 N
|
||||
FAST_POOL_MAX_LATENCY = 0.2 # 延迟 < 200ms(秒)也可入池
|
||||
# 入池阈值(仅用于缓存里的 fast 诊断标记;运行时按实测延迟整体排序,不做硬切)
|
||||
FAST_POOL_TOP_N = 10
|
||||
FAST_POOL_MAX_LATENCY = 0.2
|
||||
# 探针参数
|
||||
PROBE_PORT = 7709
|
||||
EX_PORT = 7727
|
||||
PROBE_TIMEOUT = 3.0
|
||||
PROBE_CONCURRENCY = 24
|
||||
PROBE_CONCURRENCY = 12
|
||||
PROBE_BUDGET = 60.0 # 整轮测速硬上限(秒),防止极端网络下长期占着后台
|
||||
|
||||
ProbeFn = Callable[[str, float], Awaitable[float | None]]
|
||||
|
||||
|
||||
# ---------- 三个协议各自的真实数据探针 ----------
|
||||
async def _probe_standard(host: str, timeout: float) -> float | None:
|
||||
"""标准通道 7709:取 600000 日K,非空才算可达。"""
|
||||
conn = TdxConnection(host, PROBE_PORT, timeout)
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
await conn.connect()
|
||||
bars = await conn.execute(GetSecurityBarsCmd(1, "600000", KlinePeriod.DAY, 0, 100))
|
||||
return time.monotonic() - t0 if bars else None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
async def _probe_mac(host: str, timeout: float) -> float | None:
|
||||
"""MAC 通道 7709:MAC 帧与标准帧不同,用 MacKlineCmd 取日K。"""
|
||||
conn = TdxConnection(host, PROBE_PORT, timeout)
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
await conn.connect()
|
||||
bars = await conn.execute(
|
||||
MacKlineCmd(1, "600000", KlinePeriod.DAY, Adjust.NONE, 0, 100)
|
||||
)
|
||||
return time.monotonic() - t0 if bars else None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
async def _probe_ex(host: str, timeout: float) -> float | None:
|
||||
"""EX 扩展市场 7727:取港股 00700,握手模式按主机取(login / setup)。"""
|
||||
conn = ExTdxConnection(host, EX_PORT, timeout, get_ex_handshakes().get(host, "setup"))
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
await conn.connect()
|
||||
bars = await conn.execute(GetExInstrumentBarsCmd(31, "00700", 0, 0, 5))
|
||||
return time.monotonic() - t0 if bars else None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
_DEFAULT_PROBERS: dict[str, ProbeFn] = {
|
||||
"standard": _probe_standard,
|
||||
"mac": _probe_mac,
|
||||
"ex": _probe_ex,
|
||||
}
|
||||
|
||||
|
||||
class ProbeCache:
|
||||
"""管理当日测速缓存与 servers.json 入库。
|
||||
"""当日全量测速的进程内协调器。
|
||||
|
||||
一台服务器不论协议(standard/mac/ex),测速结果共享:
|
||||
只要任一端口可达即视为该主机可用,并以最快端口延迟作为延迟基准。
|
||||
一台主机不论申报几个协议,都按协议分别探测;缓存里记录
|
||||
``{host: {protocol: 延迟或 None}}``,``None`` 表示该协议本次不可达。
|
||||
|
||||
参数(后 5 个主要供测试注入):
|
||||
cache_file: 显式指定缓存文件;``None`` 时按 config 的平台规则解析。
|
||||
probers: 协议 → 探针函数;默认使用真实网络探针。
|
||||
today_fn: 返回 ``YYYY-MM-DD``;测试可注入假日期来验证跨日重测。
|
||||
"""
|
||||
|
||||
def __init__(self, servers_file: Path | None = None) -> None:
|
||||
self._file = servers_file or _SERVERS_FILE
|
||||
def __init__(self, cache_file: Path | str | None = None, *,
|
||||
timeout: float = PROBE_TIMEOUT,
|
||||
concurrency: int = PROBE_CONCURRENCY,
|
||||
budget: float = PROBE_BUDGET,
|
||||
probers: dict[str, ProbeFn] | None = None,
|
||||
today_fn: Callable[[], str] | None = None) -> None:
|
||||
self._file = Path(cache_file) if cache_file is not None else None
|
||||
self._timeout = timeout
|
||||
self._concurrency = concurrency
|
||||
self._budget = budget
|
||||
self._probers = dict(probers or _DEFAULT_PROBERS)
|
||||
self._today_fn = today_fn or (lambda: date.today().isoformat())
|
||||
self._lock = asyncio.Lock()
|
||||
self._last_probe_date: str | None = None
|
||||
self._probed_today = False
|
||||
self._handled = False # 本进程是否已处理过今日测速(含"已有缓存,跳过")
|
||||
self._task: asyncio.Task | None = None
|
||||
|
||||
# ---------- 路径与状态 ----------
|
||||
def cache_file(self) -> Path:
|
||||
"""缓存文件路径(构造时显式传入的优先,否则按平台规则解析)。"""
|
||||
return self._file if self._file is not None else probe_cache_file()
|
||||
|
||||
def today(self) -> str:
|
||||
return self._today_fn()
|
||||
|
||||
def read_cache(self) -> dict | None:
|
||||
"""读取当日缓存;``None`` 表示"今天还没测过"。"""
|
||||
return load_probe_cache(self.today(), self.cache_file())
|
||||
|
||||
# ---- 公开 API ----
|
||||
def needs_probe_today(self) -> bool:
|
||||
"""是否有必要在今日执行一次全量测速(文件不存在 / 日期不符 / 无 ok 项)。"""
|
||||
if not self._file.exists():
|
||||
return True
|
||||
try:
|
||||
cfg = json.loads(self._file.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return True
|
||||
checked = str(cfg.get("checked_date", ""))
|
||||
today = date.today().isoformat()
|
||||
if checked != today:
|
||||
return True
|
||||
servers = cfg.get("servers", [])
|
||||
if not any(s.get("status") == "ok" for s in servers):
|
||||
return True
|
||||
return False
|
||||
"""今日是否仍需测速(缓存缺失 / 非今日 / 损坏 / 版本不符 / 无任何可用项)。"""
|
||||
if probe_disabled():
|
||||
return False
|
||||
return self.read_cache() is None
|
||||
|
||||
async def ensure_probe(self, hosts: list[str]) -> None:
|
||||
"""若今日尚未测速,则后台执行一次全量测速并入库(幂等)。"""
|
||||
def reset(self) -> None:
|
||||
"""清除"本进程已处理"标记(仅供测试复用单例)。"""
|
||||
self._handled = False
|
||||
|
||||
# ---------- 主入口 ----------
|
||||
async def ensure_probe(self, hosts: list[str] | None = None) -> None:
|
||||
"""幂等入口:当日尚未测速则测一次;当天已有缓存则立即返回。
|
||||
|
||||
- 不阻塞调用方:正常由连接池以后台 task 调用。
|
||||
- 绝不抛异常(``CancelledError`` 除外):测速 / 写盘失败只落 debug 日志。
|
||||
- ``hosts``:为兼容旧签名保留,作为**额外的标准通道候选**并入,
|
||||
且必须通过 :func:`~pytdxdata.config.is_valid_host` 校验(挡住 mock 脏数据)。
|
||||
"""
|
||||
async with self._lock:
|
||||
if self._probed_today:
|
||||
if self._handled:
|
||||
return
|
||||
self._handled = True # 先占位:并发调用方不必排队等网络
|
||||
if not self.needs_probe_today():
|
||||
self._probed_today = True
|
||||
log.debug("当日测速缓存已存在,跳过测速: %s", self.cache_file())
|
||||
return
|
||||
self._probed_today = True
|
||||
|
||||
# 在锁外执行(耗时网络 IO),允许并发进入的调用方共享本次结果
|
||||
try:
|
||||
await self._run_full_probe(hosts)
|
||||
except Exception as e: # 测速失败不能阻塞主流程
|
||||
log.warning("后台全量测速失败,沿用既有清单: %s", str(e)[:80])
|
||||
payload = await self._run_probe(hosts)
|
||||
if payload is not None:
|
||||
self._write(payload)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # 测速绝不能影响主流程
|
||||
log.debug("后台测速失败,沿用既有清单: %s", str(exc)[:120])
|
||||
|
||||
# ---- 内部实现 ----
|
||||
async def _run_full_probe(self, hosts: list[str]) -> None:
|
||||
from .connection import TdxConnection
|
||||
# ---------- 内部实现 ----------
|
||||
async def _run_probe(self, extra_hosts: list[str] | None = None) -> dict | None:
|
||||
"""全量并发测速,返回待写入的缓存 payload;无任何可达主机时返回 None。"""
|
||||
candidates = all_probe_candidates()
|
||||
if extra_hosts:
|
||||
extra = [h.strip() for h in extra_hosts
|
||||
if isinstance(h, str) and is_valid_host(h)]
|
||||
if extra:
|
||||
candidates["standard"] = list(dict.fromkeys(candidates["standard"] + extra))
|
||||
jobs = [(proto, ip) for proto, ips in candidates.items() for ip in ips]
|
||||
if not jobs:
|
||||
log.debug("无测速候选,跳过")
|
||||
return None
|
||||
|
||||
log.info("开始当日全量服务器测速 (%d 台)...", len(hosts))
|
||||
sem = asyncio.Semaphore(PROBE_CONCURRENCY)
|
||||
results: dict[str, float | None] = {}
|
||||
log.debug("开始当日全量测速: %d 台主机 / %d 个协议探针",
|
||||
len({ip for _, ip in jobs}), len(jobs))
|
||||
sem = asyncio.Semaphore(self._concurrency)
|
||||
# {host: {protocol: 延迟或 None}}
|
||||
results: dict[str, dict[str, float | None]] = defaultdict(dict)
|
||||
|
||||
async def _probe_one(host: str) -> None:
|
||||
async def _one(protocol: str, host: str) -> None:
|
||||
async with sem:
|
||||
try:
|
||||
# 使用连接自带的 ping():建连+握手+发1条命令测往返延迟
|
||||
conn = TdxConnection(host, PROBE_PORT, PROBE_TIMEOUT)
|
||||
dt = await asyncio.wait_for(conn.ping(), timeout=PROBE_TIMEOUT + 1.0)
|
||||
results[host] = dt # 失败时为 None
|
||||
lat = await asyncio.wait_for(
|
||||
self._probers[protocol](host, self._timeout),
|
||||
timeout=self._timeout + 2.0,
|
||||
)
|
||||
except Exception:
|
||||
results[host] = None
|
||||
lat = None
|
||||
results[host][protocol] = lat
|
||||
|
||||
await asyncio.gather(*(_probe_one(h) for h in hosts))
|
||||
|
||||
ok_hosts = {h: lt for h, lt in results.items() if lt is not None}
|
||||
# 按延迟升序排名
|
||||
ranked = sorted(ok_hosts.items(), key=lambda kv: kv[1])
|
||||
fast = set()
|
||||
for i, (h, lt) in enumerate(ranked):
|
||||
if i < FAST_POOL_TOP_N or lt < FAST_POOL_MAX_LATENCY:
|
||||
fast.add(h)
|
||||
|
||||
log.info("测速完成: 可用 %d 台, 入池 %d 台 (前%d或<%dms)",
|
||||
len(ok_hosts), len(fast), FAST_POOL_TOP_N,
|
||||
int(FAST_POOL_MAX_LATENCY * 1000))
|
||||
self._write_results(results, fast)
|
||||
|
||||
def _write_results(self, results: dict[str, float | None], fast: set[str]) -> None:
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
cfg = json.loads(self._file.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
cfg = {"servers": []}
|
||||
servers = cfg.get("servers", [])
|
||||
by_ip = {s.get("ip"): s for s in servers}
|
||||
await asyncio.wait_for(
|
||||
asyncio.gather(*(_one(p, ip) for p, ip in jobs)),
|
||||
timeout=self._budget,
|
||||
)
|
||||
except TimeoutError:
|
||||
# 超时:采用已完成的部分结果(部分可用也强于完全没有)
|
||||
log.debug("测速整体超过 %.0fs 预算,采用已完成的部分结果", self._budget)
|
||||
except Exception as exc:
|
||||
log.debug("测速过程异常: %s", str(exc)[:120])
|
||||
elapsed = time.monotonic() - t0
|
||||
|
||||
today = date.today().isoformat()
|
||||
now = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
checked_any = False
|
||||
for host, lt in results.items():
|
||||
entry = by_ip.get(host)
|
||||
if entry is None:
|
||||
entry = {"ip": host, "name": host, "protocols": ["standard"]}
|
||||
servers.append(entry)
|
||||
by_ip[host] = entry
|
||||
if lt is not None:
|
||||
checked_any = True
|
||||
entry["status"] = "ok" if host in fast else "slow"
|
||||
entry["latency"] = round(lt, 3)
|
||||
entry["fast_pool"] = host in fast
|
||||
else:
|
||||
# 探测不可达:不降级已有 ok(避免误杀),仅标记本次不可达
|
||||
entry.setdefault("status", "unknown")
|
||||
entry["latency"] = None
|
||||
entry["fast_pool"] = False
|
||||
entry["checked_date"] = today
|
||||
entry["checked_at"] = now
|
||||
responsive = {
|
||||
ip: {p: lat for p, lat in per.items() if lat is not None}
|
||||
for ip, per in results.items()
|
||||
}
|
||||
responsive = {ip: per for ip, per in responsive.items() if per}
|
||||
if not responsive:
|
||||
log.debug("测速结束: 无任何可达主机,不写缓存(下次启动重试)")
|
||||
return None
|
||||
|
||||
# 若有 ok 服务器写入可入池日期;否则保留今日以免反复重试
|
||||
cfg["checked_date"] = today
|
||||
cfg["note"] = (
|
||||
f"当日全量测速入库({today}): 入池={len(fast)}台 "
|
||||
f"(排名前{FAST_POOL_TOP_N}或延迟<{int(FAST_POOL_MAX_LATENCY*1000)}ms)"
|
||||
def _ordered(protocol: str) -> list[str]:
|
||||
rows = [(per[protocol], ip) for ip, per in responsive.items()
|
||||
if protocol in per]
|
||||
rows.sort(key=lambda kv: (kv[0], kv[1]))
|
||||
return [ip for _, ip in rows]
|
||||
|
||||
hosts_by_proto = {p: _ordered(p) for p in ("standard", "mac", "ex")}
|
||||
overall = sorted(
|
||||
(min(per.values()), ip) for ip, per in responsive.items()
|
||||
)
|
||||
if not checked_any:
|
||||
# 全部不可达时不写 checked_date,下次启动重试
|
||||
cfg.pop("checked_date", None)
|
||||
fast = {ip for i, (lat, ip) in enumerate(overall)
|
||||
if i < FAST_POOL_TOP_N or lat < FAST_POOL_MAX_LATENCY}
|
||||
|
||||
self._file.write_text(
|
||||
json.dumps(cfg, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
log.info("测速结果已写入 %s", self._file)
|
||||
log.debug("测速完成: 候选 %d 台, 可达 %d 台, fast %d 台, 耗时 %.2fs",
|
||||
len({ip for _, ip in jobs}), len(responsive), len(fast), elapsed)
|
||||
return {
|
||||
"version": PROBE_CACHE_VERSION,
|
||||
"date": self.today(),
|
||||
"probed_at": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"elapsed": round(elapsed, 3),
|
||||
# 每个协议的完整探测结果(含 None = 本次不可达),便于事后诊断
|
||||
"results": {ip: dict(per) for ip, per in results.items() if per},
|
||||
# 按延迟升序的可达主机(运行时即按此顺序取用)
|
||||
"hosts": hosts_by_proto,
|
||||
# 诊断用:符合"前N名或<200ms"的可达主机
|
||||
"fast": {p: [h for h in ips if h in fast]
|
||||
for p, ips in hosts_by_proto.items()},
|
||||
}
|
||||
|
||||
def _write(self, payload: dict) -> None:
|
||||
"""原子写入:先写临时文件再 ``os.replace``,避免读到半截 JSON。"""
|
||||
target = self.cache_file()
|
||||
tmp = target.with_name(f"{target.name}.tmp{os.getpid()}")
|
||||
try:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
os.replace(tmp, target)
|
||||
log.debug("当日测速结果已缓存: %s", target)
|
||||
except OSError as exc:
|
||||
log.debug("测速结果写入失败(忽略): %s", str(exc)[:120])
|
||||
try:
|
||||
tmp.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# 进程级单例(跨池共享,确保当日仅测一次)
|
||||
# 进程级单例(标准 / MAC / EX 三池共享,确保当日只真正测一次)
|
||||
_probe_cache = ProbeCache()
|
||||
|
||||
|
||||
def get_probe_cache() -> ProbeCache:
|
||||
return _probe_cache
|
||||
|
||||
|
||||
def schedule_probe(hosts: list[str] | None = None) -> asyncio.Task | None:
|
||||
"""[连接池用] 把当日测速挂到后台并返回 Task,供调用方持有 / 取消。
|
||||
|
||||
返回 ``None`` 表示当前没有运行中的事件循环(此时无法起后台任务)。
|
||||
"""
|
||||
try:
|
||||
return asyncio.create_task(
|
||||
get_probe_cache().ensure_probe(hosts), name="pytdxdata-daily-probe"
|
||||
)
|
||||
except RuntimeError:
|
||||
return None
|
||||
Reference in new issue
Block a user