mirror of
https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git
synced 2026-09-12 17:54:15 +08:00
perf(polars): 并发闸+写锁收缩+看门狗三层防死锁, 升级 polars 1.44
线上曾出现并发 LazyFrame.collect 触发 polars streaming 执行器死锁, 叠加 _write_lock 区间内做重活, 放大为全站请求冻结。本次按触发缩小、 爆炸半径收缩、自动恢复三层布防: - polars_guard: BoundedSemaphore 并发闸 (总闸 4 + 后台车道 2, 后台 先拿子闸再拿总闸防死锁); repository 18 处 collect 按交互/后台分级接入 - repository 写锁区间收缩: 分区合并移出锁外, 锁内 (mtime_ns,size) 指纹校验 + 3 次乐观重试, 失败回退锁内合并; 5 处 _write_lock 重构 - watchdog: 周期探测 collect 闸与全局写锁, 连续 2 次失败退出交由 supervisor 拉起 (可配置, 默认开) - polars >=1.44,<1.45 (1.44.1); 附并发压测脚本 scripts/stress_polars_concurrency.py 供复现验证 另: config 新增 polars_collect_permits / watchdog_* / strategy_run_all_workers / strategy_run_all_first_return_s 旋钮 (后两者供后续 run_all 优化提交使用, 默认保持旧行为基准)。
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
"""_write_lock 锁区间瘦身的回归测试。
|
||||
|
||||
背景: polars 并发执行存在死锁风险 (app.polars_guard), 若 polars 读/合并/排序
|
||||
悬死在 _write_lock 内, 全局写锁被永久持有 → 所有写路径排队冻结 (2026-09-07
|
||||
线上全站冻结事故的放大器)。乐观并发模式把重活移到锁外, 此处验证:
|
||||
- 并发 upsert 不丢行 (乐观重试的正确性);
|
||||
- 合并计算期间 _write_lock 可被其他线程获取 (重活确实不在锁内)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.tickflow.repository import DataStore, KlineRepository
|
||||
|
||||
|
||||
def _frame(symbols: list[str], dt: date = date(2026, 9, 7)) -> pl.DataFrame:
|
||||
n = len(symbols)
|
||||
return pl.DataFrame({
|
||||
"symbol": symbols,
|
||||
"date": [dt] * n,
|
||||
"close": [10.0 + i for i in range(n)],
|
||||
})
|
||||
|
||||
|
||||
def test_concurrent_upserts_do_not_lose_rows(tmp_path: Path) -> None:
|
||||
repo = KlineRepository(DataStore(tmp_path))
|
||||
|
||||
groups = [[f"{i:03d}{j:04d}.SZ" for j in range(8)] for i in range(6)]
|
||||
errors: list[BaseException] = []
|
||||
|
||||
def worker(symbols: list[str]) -> None:
|
||||
try:
|
||||
for _ in range(3): # 每线程多轮写, 提高乐观重试路径命中
|
||||
repo.merge_live_daily_asset("stock", _frame(symbols))
|
||||
except BaseException as exc:
|
||||
errors.append(exc)
|
||||
|
||||
threads = [threading.Thread(target=worker, args=(g,), daemon=True) for g in groups]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join(timeout=30)
|
||||
assert not errors, errors
|
||||
assert all(not t.is_alive() for t in threads)
|
||||
|
||||
out = tmp_path / "kline_daily" / "date=2026-09-07" / "part.parquet"
|
||||
final = pl.read_parquet(out)
|
||||
assert final["symbol"].n_unique() == sum(len(g) for g in groups) # 无一丢失
|
||||
assert final["symbol"].to_list() == sorted(final["symbol"].to_list())
|
||||
|
||||
|
||||
def test_heavy_merge_runs_outside_write_lock(tmp_path: Path, monkeypatch) -> None:
|
||||
repo = KlineRepository(DataStore(tmp_path))
|
||||
# 预置旧分区内容, 让 upsert 走「读旧 + concat 合并」路径。
|
||||
repo.merge_live_daily_asset("stock", _frame(["000001.SZ"]))
|
||||
|
||||
merge_entered = threading.Event()
|
||||
original_concat = pl.concat
|
||||
|
||||
def slow_concat(*args, **kwargs):
|
||||
merge_entered.set()
|
||||
import time
|
||||
|
||||
time.sleep(0.4) # 模拟重合并耗时; 期间 _write_lock 必须是空闲的
|
||||
return original_concat(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(pl, "concat", slow_concat)
|
||||
|
||||
done = threading.Event()
|
||||
|
||||
def upsert() -> None:
|
||||
repo.merge_live_daily_asset("stock", _frame(["000002.SZ"]))
|
||||
done.set()
|
||||
|
||||
t = threading.Thread(target=upsert, daemon=True)
|
||||
t.start()
|
||||
assert merge_entered.wait(timeout=5), "合并路径未被触发"
|
||||
|
||||
acquired = repo._write_lock.acquire(timeout=1.0)
|
||||
assert acquired, "合并计算期间 _write_lock 被占用 — 重活仍在锁内"
|
||||
repo._write_lock.release()
|
||||
|
||||
assert done.wait(timeout=5)
|
||||
t.join(timeout=5)
|
||||
out = tmp_path / "kline_daily" / "date=2026-09-07" / "part.parquet"
|
||||
assert pl.read_parquet(out)["symbol"].to_list() == ["000001.SZ", "000002.SZ"]
|
||||
Reference in New Issue
Block a user