feat(reconnect): 引入服务器健康分引擎 + K线空数据故障转移

彻底解决通达信服务器"跳来跳去"问题:

1. 新增 _health.py 健康分引擎:失败乘性降权(×0.5)、连续失败3次进
   120s 冷却、成功加性恢复(+0.2)。rank_by_health 按 latency/score
   (有效延迟)重排,冷却中的剔除。全健康时恒等映射,对既有测试零影响。

2. get_index_bars/get_security_bars 空数据时自动逐台换台(此前直接
   返回空 DataFrame,是日志"指数K线响应在第1/800条处被截断"后用户拿
   不到数据的根因)。复用泛化后的 _find_host_returning_data。

3. select_best_host_*/find_working_host_* 应用 rank_by_health 重排;
   空数据验证失败/异常时调 record_failure,命中调 record_success。

4. 8 个 _execute(A股/MAC/EX/MAC-EX × sync/async)统一注入健康分记录:
   成功 record_success、连接失败 record_failure。

5. security_bars 截断日志区分"首条即空(服务器无数据)"与"末尾截断"。

测试:26 个新增(15 health + 7 failover + 4 ex-client 健康分追踪),
全量 reconnect/failover/decode 回归通过,ruff/mypy 通过。
This commit is contained in:
Justin Gu
2026-07-13 15:59:28 +08:00
parent 5dd1b96818
commit 1f040afabf
10 changed files with 770 additions and 139 deletions
+125
View File
@@ -191,3 +191,128 @@ class TestMacExLoginRetriedOnConnectionError:
client._execute(GetExMarketsCmd())
# 关键:_login 异常被纳入重试,4 次都跑了(而非第 1 次就逃逸)
assert mock_sleep.call_count == len(_RETRY_DELAYS)
# --------------------------------------------------------------------------- #
# MAC/EX client 健康分联动(防 pattern-fix 回归)
# --------------------------------------------------------------------------- #
class TestExClientHealthTracking:
"""锁定修复:MAC/EX 的 _execute 必须像 A 股 client 一样写健康分。
此前只有 A 股 TdxClient/AsyncTdxClient 注入了 record_failure/record_success
MAC/EX 的 6 个 _execute 漏改(它们的服务器 IP 与 A 股不重叠,失败时不会被
降权,功能残缺)。本测试防止再次漏改。
"""
def test_ex_client_failure_records_health(self) -> None:
"""ExTdxClient 连接失败时应调 record_failure 降权当前 host。"""
from easy_tdx._health import reset_health
reset_health()
try:
with (
patch("easy_tdx.ex.client.ExTdxConnection") as mock_conn_cls,
patch("easy_tdx.ex.client.time.sleep"),
patch("easy_tdx.ex.client.select_best_host_sync", return_value=None),
):
mock_conn = MagicMock()
mock_conn.execute.side_effect = TdxConnectionError("down")
mock_conn_cls.return_value = mock_conn
client = ExTdxClient("ex-bad", auto_reconnect=True)
with pytest.raises(TdxConnectionError):
client._execute(GetExMarketsCmd())
from easy_tdx._health import get_score
# ex-bad 经历首次 + 4 次重试共 5 次失败,score 应远低于 1.0
assert get_score("ex-bad") < 1.0
finally:
reset_health()
def test_ex_client_success_records_health(self) -> None:
"""ExTdxClient 首次成功应调 record_successscore 保持 1.0)。"""
from easy_tdx._health import get_score, reset_health
reset_health()
try:
with patch("easy_tdx.ex.client.ExTdxConnection") as mock_conn_cls:
mock_conn = MagicMock()
mock_conn.execute.return_value = ["market"]
mock_conn_cls.return_value = mock_conn
client = ExTdxClient("ex-good", auto_reconnect=True)
result = client._execute(GetExMarketsCmd())
assert result == ["market"]
assert get_score("ex-good") == 1.0
finally:
reset_health()
def test_mac_ex_client_failure_records_health(self) -> None:
"""MacExClient(含 _login 重连路径)连接失败也应降权当前 host。"""
from easy_tdx._health import reset_health
reset_health()
try:
with (
patch("easy_tdx.ex.mac_client.ExTdxConnection") as mock_conn_cls,
patch("easy_tdx.ex.mac_client.time.sleep"),
patch("easy_tdx.ex.mac_client.select_best_host_sync", return_value=None),
):
mock_conn = MagicMock()
mock_conn.execute.side_effect = TdxConnectionError("down")
mock_conn_cls.return_value = mock_conn
client = MacExClient("macex-bad", auto_reconnect=True)
with pytest.raises(TdxConnectionError):
client._execute(GetExMarketsCmd())
from easy_tdx._health import get_score
assert get_score("macex-bad") < 1.0
finally:
reset_health()
def test_async_ex_client_failure_records_health(self) -> None:
"""AsyncExTdxClient 连接失败时也应降权。"""
from easy_tdx._health import reset_health
reset_health()
try:
async def main() -> None:
with patch("easy_tdx.ex.client.AsyncExTdxConnection") as mock_conn_cls:
mock_conn = MagicMock()
async def _execute(cmd: object) -> list[str]:
raise TdxConnectionError("down")
async def _noop() -> None:
return None
mock_conn.execute = _execute
mock_conn.close = _noop
mock_conn.connect = _noop
mock_conn_cls.return_value = mock_conn
client = AsyncExTdxClient("aex-bad", auto_reconnect=True, heartbeat_interval=0)
with (
patch("easy_tdx.ex.client.asyncio.sleep"),
patch(
"easy_tdx.ex.client.select_best_host_async",
new_callable=AsyncMock,
return_value=None,
),
):
with pytest.raises(TdxConnectionError):
await client._execute(GetExMarketsCmd())
asyncio.run(main())
from easy_tdx._health import get_score
assert get_score("aex-bad") < 1.0
finally:
reset_health()
+81 -10
View File
@@ -15,6 +15,7 @@ from unittest.mock import MagicMock, patch
import pytest
from easy_tdx._health import _FAILURE_DECAY, reset_health
from easy_tdx._reconnect import (
_FAILOVER_PING_THROTTLE_SEC,
_WORKING_HOST_MAX_ATTEMPTS,
@@ -27,6 +28,23 @@ from easy_tdx.exceptions import TdxConnectionError
from easy_tdx.models.bar import SecurityBar
from easy_tdx.models.enums import KlineCategory, Market
@pytest.fixture(autouse=True)
def _isolate_health_and_throttle():
"""每个测试前后重置健康分 + 节流时间戳,避免跨测试污染。
failover 与空数据转移现在会写健康分(record_failure/success),
若不重置,一个测试里降权的 host 会影响后续测试的 rank_by_health 排序。
"""
import easy_tdx._reconnect as r
r._last_failover_ts = 0.0
reset_health()
yield
reset_health()
r._last_failover_ts = 0.0
# --------------------------------------------------------------------------- #
# select_best_host_sync 单元逻辑
# --------------------------------------------------------------------------- #
@@ -400,19 +418,72 @@ class TestMarketStatEmptyFailover:
# --------------------------------------------------------------------------- #
# get_index_bars / get_security_bars 空数据故障转移
# (与 TestMarketStatEmptyFailover 对称:指数/板块指数 880xxx 并非所有服务器都提供)
# 健康分联动:select_best_host / find_working_host 感知健康分
# --------------------------------------------------------------------------- #
class TestIndexBarsEmptyFailover:
class TestHealthAwareFailover:
"""验证故障转移会读取/写入健康分:坏主机被降权后排序靠后。"""
def test_select_best_host_skips_cooldown_host(self) -> None:
"""冷却中的主机即使延迟最低,也不会被 select_best_host 选中。"""
from easy_tdx._health import record_failure
# host-fast 连续失败进入冷却
for _ in range(3):
record_failure("host-fast")
ping_fn = MagicMock(return_value=[("host-fast", 0.01), ("host-slow", 0.10)])
save_fn = MagicMock()
result = select_best_host_sync(
["host-fast", "host-slow", "cur"], ping_fn, save_fn, 7709, 1.0, "cur"
)
# host-fast 在冷却中被剔除,应选 host-slow
assert result == "host-slow"
save_fn.assert_called_once_with("host-slow")
def test_find_working_host_records_failure_on_empty(self) -> None:
"""候选返回空数据时记一次 failure(降权),下次轮询优先级下降。"""
from easy_tdx._health import get_score
ranked = [("empty-host", 0.01), ("good-host", 0.02)]
try_fn = MagicMock(side_effect=[False, True]) # empty 空,good 非空
save_fn = MagicMock()
result = find_working_host_sync(ranked, try_fn, save_fn, "cur")
assert result == "good-host"
# empty-host 被记一次失败,score < 1.0
assert get_score("empty-host") < 1.0
# good-host 被记成功,score = 1.0
assert get_score("good-host") == 1.0
def test_find_working_host_records_success_on_hit(self) -> None:
"""命中的主机 score 恢复到 1.0。"""
from easy_tdx._health import get_score, record_failure
# 先把 good-host 降权
record_failure("good-host")
assert get_score("good-host") < 1.0
ranked = [("good-host", 0.01)]
try_fn = MagicMock(return_value=True)
save_fn = MagicMock()
find_working_host_sync(ranked, try_fn, save_fn, "cur")
# 命中后 score 恢复(+0.2,但初始降权后 0.5+0.2=0.7,未到 1.0
# 关键是比失败前上升了)
assert get_score("good-host") > _FAILURE_DECAY
# --------------------------------------------------------------------------- #
# get_index_bars / get_security_bars 空数据故障转移
# (指数/板块指数 880xxx 并非所有服务器都提供,空时逐台实测切 host)
# --------------------------------------------------------------------------- #
class TestBarsEmptyFailover:
"""K 线空数据故障转移——验证 get_index_bars/get_security_bars 空时逐台实测切 host。"""
def setup_method(self) -> None:
import easy_tdx._reconnect as r
r._last_failover_ts = 0.0
def _make_bar(self) -> SecurityBar:
"""构造一根字段合法的日 K,让 get_index_bars 下游处理走通。"""
return SecurityBar(
@@ -478,7 +549,7 @@ class TestIndexBarsEmptyFailover:
with (
patch.object(client, "_execute", return_value=[bar]) as mock_exec,
patch.object(client, "_find_host_returning_bars") as mock_failover,
patch.object(client, "_find_host_returning_data") as mock_failover,
):
df = client.get_index_bars(Market.SH, "880008", KlineCategory.DAY, 0, 10)
@@ -492,7 +563,7 @@ class TestIndexBarsEmptyFailover:
with (
patch.object(client, "_execute", return_value=[]) as mock_exec,
patch.object(client, "_find_host_returning_bars") as mock_failover,
patch.object(client, "_find_host_returning_data") as mock_failover,
):
df = client.get_index_bars(Market.SH, "880008", KlineCategory.DAY, 0, 10)
+149
View File
@@ -0,0 +1,149 @@
"""服务器健康分(health score)引擎单元测试。
覆盖:
- record_failure 乘性衰减 + 连续失败触发冷却
- record_success 加性恢复 + 重置计数与冷却
- is_in_cooldown / get_score 读取语义
- rank_by_health:冷却剔除 + 有效延迟(latency/score)排序
- 全健康时 rank_by_health 近似恒等映射(向后兼容保证)
"""
from __future__ import annotations
import time
import pytest
from easy_tdx._health import (
_COOLDOWN_FAIL_THRESHOLD,
_FAILURE_DECAY,
_SUCCESS_RECOVER,
get_score,
is_in_cooldown,
rank_by_health,
record_failure,
record_success,
reset_health,
)
@pytest.fixture(autouse=True)
def _isolate_health():
"""每个测试前后清空健康记录,避免跨测试污染。"""
reset_health()
yield
reset_health()
# --------------------------------------------------------------------------- #
# record_failure / record_success
# --------------------------------------------------------------------------- #
class TestRecordFailure:
def test_single_failure_decays_score(self) -> None:
s = record_failure("h1")
assert s == pytest.approx(_FAILURE_DECAY)
assert get_score("h1") == pytest.approx(_FAILURE_DECAY)
def test_repeated_failure_decays_multiplicatively(self) -> None:
record_failure("h1")
record_failure("h1")
s = record_failure("h1")
assert s == pytest.approx(_FAILURE_DECAY**3)
def test_consecutive_failures_below_threshold_no_cooldown(self) -> None:
for _ in range(_COOLDOWN_FAIL_THRESHOLD - 1):
record_failure("h1")
assert not is_in_cooldown("h1")
def test_consecutive_failures_at_threshold_enters_cooldown(self) -> None:
for _ in range(_COOLDOWN_FAIL_THRESHOLD):
record_failure("h1")
assert is_in_cooldown("h1")
def test_score_never_drops_below_floor(self) -> None:
for _ in range(100):
record_failure("h1")
assert get_score("h1") > 0
class TestRecordSuccess:
def test_success_recovers_score_additively(self) -> None:
record_failure("h1") # score = 0.5
record_success("h1")
assert get_score("h1") == pytest.approx(_FAILURE_DECAY + _SUCCESS_RECOVER)
def test_success_caps_at_one(self) -> None:
record_success("h1")
record_success("h1")
assert get_score("h1") == pytest.approx(1.0)
def test_success_resets_consecutive_failures_and_cooldown(self) -> None:
for _ in range(_COOLDOWN_FAIL_THRESHOLD):
record_failure("h1")
assert is_in_cooldown("h1")
record_success("h1")
assert not is_in_cooldown("h1")
# 再次失败一次不应立即进冷却(计数已重置)
record_failure("h1")
assert not is_in_cooldown("h1")
# --------------------------------------------------------------------------- #
# is_in_cooldown
# --------------------------------------------------------------------------- #
class TestIsInCooldown:
def test_unknown_host_not_in_cooldown(self) -> None:
assert not is_in_cooldown("never-seen")
def test_cooldown_expires(self) -> None:
# 手动模拟过期:记录到阈值进入冷却后,快进时间戳
for _ in range(_COOLDOWN_FAIL_THRESHOLD):
record_failure("h1")
assert is_in_cooldown("h1")
# 直接篡改内部状态模拟冷却过期(避免真睡 120s)
from easy_tdx._health import _BOOK
with _BOOK.lock:
_BOOK.hosts["h1"].cooldown_until = time.monotonic() - 1.0
assert not is_in_cooldown("h1")
# --------------------------------------------------------------------------- #
# rank_by_health
# --------------------------------------------------------------------------- #
class TestRankByHealth:
def test_identity_when_all_healthy(self) -> None:
"""全健康时输出排序与输入一致(向后兼容关键保证)。"""
ranked = [("a", 0.01), ("b", 0.05), ("c", 0.10)]
assert rank_by_health(ranked) == ranked
def test_filters_out_cooldown_hosts(self) -> None:
for _ in range(_COOLDOWN_FAIL_THRESHOLD):
record_failure("bad")
ranked = [("bad", 0.01), ("good", 0.02)]
result = rank_by_health(ranked)
assert "bad" not in [h for h, _ in result]
assert result == [("good", 0.02)]
def test_low_score_host_pushed_back(self) -> None:
# b 延迟最低但 score 被打到很低,使其有效延迟(latency/score)反超 a。
# 2 次失败 → b score = 0.25;有效延迟 = 0.03/0.25 = 0.12。
# a 全健康,有效延迟 = 0.05/1.0 = 0.05 < 0.12 → a 应排前。
for _ in range(2):
record_failure("b") # 不进冷却(阈值 3),但 score 衰减到 0.25
ranked = [("b", 0.03), ("a", 0.05)]
result = rank_by_health(ranked)
assert result[0][0] == "a"
def test_empty_input(self) -> None:
assert rank_by_health([]) == []
def test_preserves_latency_order_among_equal_scores(self) -> None:
ranked = [("a", 0.01), ("b", 0.02), ("c", 0.03)]
assert rank_by_health(ranked) == ranked