mirror of
https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git
synced 2026-09-12 23:44:16 +08:00
shy3130
5618b4ef1d
线上曾出现并发 LazyFrame.collect 触发 polars streaming 执行器死锁, 叠加 _write_lock 区间内做重活, 放大为全站请求冻结。本次按触发缩小、 爆炸半径收缩、自动恢复三层布防: - polars_guard: BoundedSemaphore 并发闸 (总闸 4 + 后台车道 2, 后台 先拿子闸再拿总闸防死锁); repository 18 处 collect 按交互/后台分级接入 - repository 写锁区间收缩: 分区合并移出锁外, 锁内 (mtime_ns,size) 指纹校验 + 3 次乐观重试, 失败回退锁内合并; 5 处 _write_lock 重构 - watchdog: 周期探测 collect 闸与全局写锁, 连续 2 次失败退出交由 supervisor 拉起 (可配置, 默认开) - polars >=1.44,<1.45 (1.44.1); 附并发压测脚本 scripts/stress_polars_concurrency.py 供复现验证 另: config 新增 polars_collect_permits / watchdog_* / strategy_run_all_workers / strategy_run_all_first_return_s 旋钮 (后两者供后续 run_all 优化提交使用, 默认保持旧行为基准)。
55 lines
1.5 KiB
Python
55 lines
1.5 KiB
Python
"""看门狗触发逻辑测试 (不真退出进程)。"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
from app.watchdog import HealthWatchdog, default_probe
|
|
|
|
|
|
async def _run_watchdog(probe_results, *, threshold=2, interval=0.01, timeout=0.2):
|
|
exits: list[int] = []
|
|
idx = 0
|
|
|
|
def probe() -> None:
|
|
nonlocal idx
|
|
if idx < len(probe_results):
|
|
result = probe_results[idx]
|
|
idx += 1
|
|
if isinstance(result, BaseException):
|
|
raise result
|
|
# 脚本耗尽后恒为成功
|
|
|
|
wd = HealthWatchdog(
|
|
probe,
|
|
exit_cb=exits.append,
|
|
interval_s=interval,
|
|
probe_timeout_s=timeout,
|
|
failure_threshold=threshold,
|
|
)
|
|
wd.start()
|
|
for _ in range(50):
|
|
await asyncio.sleep(0.02)
|
|
if exits or wd._task.done():
|
|
break
|
|
await wd.stop()
|
|
return exits
|
|
|
|
|
|
async def test_consecutive_failures_trigger_exit() -> None:
|
|
exits = await _run_watchdog([RuntimeError("wedge"), TimeoutError("wedge")])
|
|
assert exits == [70]
|
|
|
|
|
|
async def test_success_resets_failure_counter() -> None:
|
|
# 失败 1 次 → 成功 → 再失败 1 次: 未达连续阈值, 不退出。
|
|
exits = await _run_watchdog([RuntimeError("slow"), None, RuntimeError("slow")])
|
|
assert exits == []
|
|
|
|
|
|
async def test_default_probe_passes_on_healthy_resources() -> None:
|
|
import threading
|
|
|
|
lock = threading.Lock()
|
|
default_probe(lock) # 不抛即通过
|
|
default_probe(None)
|