mirror of
https://ghfast.top/https://github.com/aeroxw/easy-tdx.git
synced 2026-09-12 15:44:15 +08:00
feat(reconnect): 引入服务器健康分引擎 + K线空数据故障转移
彻底解决通达信服务器"跳来跳去"问题: 1. 新增 _health.py 健康分引擎:失败乘性降权(×0.5)、连续失败3次进 120s 冷却、成功加性恢复(+0.2)。rank_by_health 按 latency/score (有效延迟)重排,冷却中的剔除。全健康时恒等映射,对既有测试零影响。 2. get_index_bars/get_security_bars 空数据时自动逐台换台(此前直接 返回空 DataFrame,是日志"指数K线响应在第1/800条处被截断"后用户拿 不到数据的根因)。复用泛化后的 _find_host_returning_data。 3. select_best_host_*/find_working_host_* 应用 rank_by_health 重排; 空数据验证失败/异常时调 record_failure,命中调 record_success。 4. 8 个 _execute(A股/MAC/EX/MAC-EX × sync/async)统一注入健康分记录: 成功 record_success、连接失败 record_failure。 5. security_bars 截断日志区分"首条即空(服务器无数据)"与"末尾截断"。 测试:26 个新增(15 health + 7 failover + 4 ex-client 健康分追踪), 全量 reconnect/failover/decode 回归通过,ruff/mypy 通过。
This commit is contained in:
@@ -191,3 +191,128 @@ class TestMacExLoginRetriedOnConnectionError:
|
||||
client._execute(GetExMarketsCmd())
|
||||
# 关键:_login 异常被纳入重试,4 次都跑了(而非第 1 次就逃逸)
|
||||
assert mock_sleep.call_count == len(_RETRY_DELAYS)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# MAC/EX client 健康分联动(防 pattern-fix 回归)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestExClientHealthTracking:
|
||||
"""锁定修复:MAC/EX 的 _execute 必须像 A 股 client 一样写健康分。
|
||||
|
||||
此前只有 A 股 TdxClient/AsyncTdxClient 注入了 record_failure/record_success,
|
||||
MAC/EX 的 6 个 _execute 漏改(它们的服务器 IP 与 A 股不重叠,失败时不会被
|
||||
降权,功能残缺)。本测试防止再次漏改。
|
||||
"""
|
||||
|
||||
def test_ex_client_failure_records_health(self) -> None:
|
||||
"""ExTdxClient 连接失败时应调 record_failure 降权当前 host。"""
|
||||
from easy_tdx._health import reset_health
|
||||
|
||||
reset_health()
|
||||
try:
|
||||
with (
|
||||
patch("easy_tdx.ex.client.ExTdxConnection") as mock_conn_cls,
|
||||
patch("easy_tdx.ex.client.time.sleep"),
|
||||
patch("easy_tdx.ex.client.select_best_host_sync", return_value=None),
|
||||
):
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute.side_effect = TdxConnectionError("down")
|
||||
mock_conn_cls.return_value = mock_conn
|
||||
|
||||
client = ExTdxClient("ex-bad", auto_reconnect=True)
|
||||
with pytest.raises(TdxConnectionError):
|
||||
client._execute(GetExMarketsCmd())
|
||||
|
||||
from easy_tdx._health import get_score
|
||||
|
||||
# ex-bad 经历首次 + 4 次重试共 5 次失败,score 应远低于 1.0
|
||||
assert get_score("ex-bad") < 1.0
|
||||
finally:
|
||||
reset_health()
|
||||
|
||||
def test_ex_client_success_records_health(self) -> None:
|
||||
"""ExTdxClient 首次成功应调 record_success(score 保持 1.0)。"""
|
||||
from easy_tdx._health import get_score, reset_health
|
||||
|
||||
reset_health()
|
||||
try:
|
||||
with patch("easy_tdx.ex.client.ExTdxConnection") as mock_conn_cls:
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute.return_value = ["market"]
|
||||
mock_conn_cls.return_value = mock_conn
|
||||
|
||||
client = ExTdxClient("ex-good", auto_reconnect=True)
|
||||
result = client._execute(GetExMarketsCmd())
|
||||
assert result == ["market"]
|
||||
assert get_score("ex-good") == 1.0
|
||||
finally:
|
||||
reset_health()
|
||||
|
||||
def test_mac_ex_client_failure_records_health(self) -> None:
|
||||
"""MacExClient(含 _login 重连路径)连接失败也应降权当前 host。"""
|
||||
from easy_tdx._health import reset_health
|
||||
|
||||
reset_health()
|
||||
try:
|
||||
with (
|
||||
patch("easy_tdx.ex.mac_client.ExTdxConnection") as mock_conn_cls,
|
||||
patch("easy_tdx.ex.mac_client.time.sleep"),
|
||||
patch("easy_tdx.ex.mac_client.select_best_host_sync", return_value=None),
|
||||
):
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute.side_effect = TdxConnectionError("down")
|
||||
mock_conn_cls.return_value = mock_conn
|
||||
|
||||
client = MacExClient("macex-bad", auto_reconnect=True)
|
||||
with pytest.raises(TdxConnectionError):
|
||||
client._execute(GetExMarketsCmd())
|
||||
|
||||
from easy_tdx._health import get_score
|
||||
|
||||
assert get_score("macex-bad") < 1.0
|
||||
finally:
|
||||
reset_health()
|
||||
|
||||
def test_async_ex_client_failure_records_health(self) -> None:
|
||||
"""AsyncExTdxClient 连接失败时也应降权。"""
|
||||
from easy_tdx._health import reset_health
|
||||
|
||||
reset_health()
|
||||
try:
|
||||
|
||||
async def main() -> None:
|
||||
with patch("easy_tdx.ex.client.AsyncExTdxConnection") as mock_conn_cls:
|
||||
mock_conn = MagicMock()
|
||||
|
||||
async def _execute(cmd: object) -> list[str]:
|
||||
raise TdxConnectionError("down")
|
||||
|
||||
async def _noop() -> None:
|
||||
return None
|
||||
|
||||
mock_conn.execute = _execute
|
||||
mock_conn.close = _noop
|
||||
mock_conn.connect = _noop
|
||||
mock_conn_cls.return_value = mock_conn
|
||||
|
||||
client = AsyncExTdxClient("aex-bad", auto_reconnect=True, heartbeat_interval=0)
|
||||
with (
|
||||
patch("easy_tdx.ex.client.asyncio.sleep"),
|
||||
patch(
|
||||
"easy_tdx.ex.client.select_best_host_async",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
with pytest.raises(TdxConnectionError):
|
||||
await client._execute(GetExMarketsCmd())
|
||||
|
||||
asyncio.run(main())
|
||||
|
||||
from easy_tdx._health import get_score
|
||||
|
||||
assert get_score("aex-bad") < 1.0
|
||||
finally:
|
||||
reset_health()
|
||||
|
||||
+81
-10
@@ -15,6 +15,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from easy_tdx._health import _FAILURE_DECAY, reset_health
|
||||
from easy_tdx._reconnect import (
|
||||
_FAILOVER_PING_THROTTLE_SEC,
|
||||
_WORKING_HOST_MAX_ATTEMPTS,
|
||||
@@ -27,6 +28,23 @@ from easy_tdx.exceptions import TdxConnectionError
|
||||
from easy_tdx.models.bar import SecurityBar
|
||||
from easy_tdx.models.enums import KlineCategory, Market
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_health_and_throttle():
|
||||
"""每个测试前后重置健康分 + 节流时间戳,避免跨测试污染。
|
||||
|
||||
failover 与空数据转移现在会写健康分(record_failure/success),
|
||||
若不重置,一个测试里降权的 host 会影响后续测试的 rank_by_health 排序。
|
||||
"""
|
||||
import easy_tdx._reconnect as r
|
||||
|
||||
r._last_failover_ts = 0.0
|
||||
reset_health()
|
||||
yield
|
||||
reset_health()
|
||||
r._last_failover_ts = 0.0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# select_best_host_sync 单元逻辑
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -400,19 +418,72 @@ class TestMarketStatEmptyFailover:
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# get_index_bars / get_security_bars 空数据故障转移
|
||||
# (与 TestMarketStatEmptyFailover 对称:指数/板块指数 880xxx 并非所有服务器都提供)
|
||||
# 健康分联动:select_best_host / find_working_host 感知健康分
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestIndexBarsEmptyFailover:
|
||||
class TestHealthAwareFailover:
|
||||
"""验证故障转移会读取/写入健康分:坏主机被降权后排序靠后。"""
|
||||
|
||||
def test_select_best_host_skips_cooldown_host(self) -> None:
|
||||
"""冷却中的主机即使延迟最低,也不会被 select_best_host 选中。"""
|
||||
from easy_tdx._health import record_failure
|
||||
|
||||
# host-fast 连续失败进入冷却
|
||||
for _ in range(3):
|
||||
record_failure("host-fast")
|
||||
|
||||
ping_fn = MagicMock(return_value=[("host-fast", 0.01), ("host-slow", 0.10)])
|
||||
save_fn = MagicMock()
|
||||
result = select_best_host_sync(
|
||||
["host-fast", "host-slow", "cur"], ping_fn, save_fn, 7709, 1.0, "cur"
|
||||
)
|
||||
# host-fast 在冷却中被剔除,应选 host-slow
|
||||
assert result == "host-slow"
|
||||
save_fn.assert_called_once_with("host-slow")
|
||||
|
||||
def test_find_working_host_records_failure_on_empty(self) -> None:
|
||||
"""候选返回空数据时记一次 failure(降权),下次轮询优先级下降。"""
|
||||
from easy_tdx._health import get_score
|
||||
|
||||
ranked = [("empty-host", 0.01), ("good-host", 0.02)]
|
||||
try_fn = MagicMock(side_effect=[False, True]) # empty 空,good 非空
|
||||
save_fn = MagicMock()
|
||||
|
||||
result = find_working_host_sync(ranked, try_fn, save_fn, "cur")
|
||||
assert result == "good-host"
|
||||
# empty-host 被记一次失败,score < 1.0
|
||||
assert get_score("empty-host") < 1.0
|
||||
# good-host 被记成功,score = 1.0
|
||||
assert get_score("good-host") == 1.0
|
||||
|
||||
def test_find_working_host_records_success_on_hit(self) -> None:
|
||||
"""命中的主机 score 恢复到 1.0。"""
|
||||
from easy_tdx._health import get_score, record_failure
|
||||
|
||||
# 先把 good-host 降权
|
||||
record_failure("good-host")
|
||||
assert get_score("good-host") < 1.0
|
||||
|
||||
ranked = [("good-host", 0.01)]
|
||||
try_fn = MagicMock(return_value=True)
|
||||
save_fn = MagicMock()
|
||||
|
||||
find_working_host_sync(ranked, try_fn, save_fn, "cur")
|
||||
# 命中后 score 恢复(+0.2,但初始降权后 0.5+0.2=0.7,未到 1.0;
|
||||
# 关键是比失败前上升了)
|
||||
assert get_score("good-host") > _FAILURE_DECAY
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# get_index_bars / get_security_bars 空数据故障转移
|
||||
# (指数/板块指数 880xxx 并非所有服务器都提供,空时逐台实测切 host)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestBarsEmptyFailover:
|
||||
"""K 线空数据故障转移——验证 get_index_bars/get_security_bars 空时逐台实测切 host。"""
|
||||
|
||||
def setup_method(self) -> None:
|
||||
import easy_tdx._reconnect as r
|
||||
|
||||
r._last_failover_ts = 0.0
|
||||
|
||||
def _make_bar(self) -> SecurityBar:
|
||||
"""构造一根字段合法的日 K,让 get_index_bars 下游处理走通。"""
|
||||
return SecurityBar(
|
||||
@@ -478,7 +549,7 @@ class TestIndexBarsEmptyFailover:
|
||||
|
||||
with (
|
||||
patch.object(client, "_execute", return_value=[bar]) as mock_exec,
|
||||
patch.object(client, "_find_host_returning_bars") as mock_failover,
|
||||
patch.object(client, "_find_host_returning_data") as mock_failover,
|
||||
):
|
||||
df = client.get_index_bars(Market.SH, "880008", KlineCategory.DAY, 0, 10)
|
||||
|
||||
@@ -492,7 +563,7 @@ class TestIndexBarsEmptyFailover:
|
||||
|
||||
with (
|
||||
patch.object(client, "_execute", return_value=[]) as mock_exec,
|
||||
patch.object(client, "_find_host_returning_bars") as mock_failover,
|
||||
patch.object(client, "_find_host_returning_data") as mock_failover,
|
||||
):
|
||||
df = client.get_index_bars(Market.SH, "880008", KlineCategory.DAY, 0, 10)
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
"""服务器健康分(health score)引擎单元测试。
|
||||
|
||||
覆盖:
|
||||
- record_failure 乘性衰减 + 连续失败触发冷却
|
||||
- record_success 加性恢复 + 重置计数与冷却
|
||||
- is_in_cooldown / get_score 读取语义
|
||||
- rank_by_health:冷却剔除 + 有效延迟(latency/score)排序
|
||||
- 全健康时 rank_by_health 近似恒等映射(向后兼容保证)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from easy_tdx._health import (
|
||||
_COOLDOWN_FAIL_THRESHOLD,
|
||||
_FAILURE_DECAY,
|
||||
_SUCCESS_RECOVER,
|
||||
get_score,
|
||||
is_in_cooldown,
|
||||
rank_by_health,
|
||||
record_failure,
|
||||
record_success,
|
||||
reset_health,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_health():
|
||||
"""每个测试前后清空健康记录,避免跨测试污染。"""
|
||||
reset_health()
|
||||
yield
|
||||
reset_health()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# record_failure / record_success
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestRecordFailure:
|
||||
def test_single_failure_decays_score(self) -> None:
|
||||
s = record_failure("h1")
|
||||
assert s == pytest.approx(_FAILURE_DECAY)
|
||||
assert get_score("h1") == pytest.approx(_FAILURE_DECAY)
|
||||
|
||||
def test_repeated_failure_decays_multiplicatively(self) -> None:
|
||||
record_failure("h1")
|
||||
record_failure("h1")
|
||||
s = record_failure("h1")
|
||||
assert s == pytest.approx(_FAILURE_DECAY**3)
|
||||
|
||||
def test_consecutive_failures_below_threshold_no_cooldown(self) -> None:
|
||||
for _ in range(_COOLDOWN_FAIL_THRESHOLD - 1):
|
||||
record_failure("h1")
|
||||
assert not is_in_cooldown("h1")
|
||||
|
||||
def test_consecutive_failures_at_threshold_enters_cooldown(self) -> None:
|
||||
for _ in range(_COOLDOWN_FAIL_THRESHOLD):
|
||||
record_failure("h1")
|
||||
assert is_in_cooldown("h1")
|
||||
|
||||
def test_score_never_drops_below_floor(self) -> None:
|
||||
for _ in range(100):
|
||||
record_failure("h1")
|
||||
assert get_score("h1") > 0
|
||||
|
||||
|
||||
class TestRecordSuccess:
|
||||
def test_success_recovers_score_additively(self) -> None:
|
||||
record_failure("h1") # score = 0.5
|
||||
record_success("h1")
|
||||
assert get_score("h1") == pytest.approx(_FAILURE_DECAY + _SUCCESS_RECOVER)
|
||||
|
||||
def test_success_caps_at_one(self) -> None:
|
||||
record_success("h1")
|
||||
record_success("h1")
|
||||
assert get_score("h1") == pytest.approx(1.0)
|
||||
|
||||
def test_success_resets_consecutive_failures_and_cooldown(self) -> None:
|
||||
for _ in range(_COOLDOWN_FAIL_THRESHOLD):
|
||||
record_failure("h1")
|
||||
assert is_in_cooldown("h1")
|
||||
record_success("h1")
|
||||
assert not is_in_cooldown("h1")
|
||||
# 再次失败一次不应立即进冷却(计数已重置)
|
||||
record_failure("h1")
|
||||
assert not is_in_cooldown("h1")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# is_in_cooldown
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestIsInCooldown:
|
||||
def test_unknown_host_not_in_cooldown(self) -> None:
|
||||
assert not is_in_cooldown("never-seen")
|
||||
|
||||
def test_cooldown_expires(self) -> None:
|
||||
# 手动模拟过期:记录到阈值进入冷却后,快进时间戳
|
||||
for _ in range(_COOLDOWN_FAIL_THRESHOLD):
|
||||
record_failure("h1")
|
||||
assert is_in_cooldown("h1")
|
||||
# 直接篡改内部状态模拟冷却过期(避免真睡 120s)
|
||||
from easy_tdx._health import _BOOK
|
||||
|
||||
with _BOOK.lock:
|
||||
_BOOK.hosts["h1"].cooldown_until = time.monotonic() - 1.0
|
||||
assert not is_in_cooldown("h1")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# rank_by_health
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestRankByHealth:
|
||||
def test_identity_when_all_healthy(self) -> None:
|
||||
"""全健康时输出排序与输入一致(向后兼容关键保证)。"""
|
||||
ranked = [("a", 0.01), ("b", 0.05), ("c", 0.10)]
|
||||
assert rank_by_health(ranked) == ranked
|
||||
|
||||
def test_filters_out_cooldown_hosts(self) -> None:
|
||||
for _ in range(_COOLDOWN_FAIL_THRESHOLD):
|
||||
record_failure("bad")
|
||||
ranked = [("bad", 0.01), ("good", 0.02)]
|
||||
result = rank_by_health(ranked)
|
||||
assert "bad" not in [h for h, _ in result]
|
||||
assert result == [("good", 0.02)]
|
||||
|
||||
def test_low_score_host_pushed_back(self) -> None:
|
||||
# b 延迟最低但 score 被打到很低,使其有效延迟(latency/score)反超 a。
|
||||
# 2 次失败 → b score = 0.25;有效延迟 = 0.03/0.25 = 0.12。
|
||||
# a 全健康,有效延迟 = 0.05/1.0 = 0.05 < 0.12 → a 应排前。
|
||||
for _ in range(2):
|
||||
record_failure("b") # 不进冷却(阈值 3),但 score 衰减到 0.25
|
||||
ranked = [("b", 0.03), ("a", 0.05)]
|
||||
result = rank_by_health(ranked)
|
||||
assert result[0][0] == "a"
|
||||
|
||||
def test_empty_input(self) -> None:
|
||||
assert rank_by_health([]) == []
|
||||
|
||||
def test_preserves_latency_order_among_equal_scores(self) -> None:
|
||||
ranked = [("a", 0.01), ("b", 0.02), ("c", 0.03)]
|
||||
assert rank_by_health(ranked) == ranked
|
||||
Reference in New Issue
Block a user