perf(polars): 并发闸+写锁收缩+看门狗三层防死锁, 升级 polars 1.44

线上曾出现并发 LazyFrame.collect 触发 polars streaming 执行器死锁,
叠加 _write_lock 区间内做重活, 放大为全站请求冻结。本次按触发缩小、
爆炸半径收缩、自动恢复三层布防:

- polars_guard: BoundedSemaphore 并发闸 (总闸 4 + 后台车道 2, 后台
  先拿子闸再拿总闸防死锁); repository 18 处 collect 按交互/后台分级接入
- repository 写锁区间收缩: 分区合并移出锁外, 锁内 (mtime_ns,size)
  指纹校验 + 3 次乐观重试, 失败回退锁内合并; 5 处 _write_lock 重构
- watchdog: 周期探测 collect 闸与全局写锁, 连续 2 次失败退出交由
  supervisor 拉起 (可配置, 默认开)
- polars >=1.44,<1.45 (1.44.1); 附并发压测脚本
  scripts/stress_polars_concurrency.py 供复现验证

另: config 新增 polars_collect_permits / watchdog_* /
strategy_run_all_workers / strategy_run_all_first_return_s 旋钮
(后两者供后续 run_all 优化提交使用, 默认保持旧行为基准)。
This commit is contained in:
shy3130
2026-09-07 15:25:41 +08:00
parent 58b161b16d
commit 5618b4ef1d
12 changed files with 2545 additions and 1931 deletions
@@ -0,0 +1,126 @@
"""polars 并发死锁复现压测。
模拟 2026-09-07 线上事故的触发形态: 多个线程并发对同一 parquet 目录做
lazy scan + filter + collect (页面首屏多路读), 叠加后台线程的批量重计算 —
全部绕过 polars_guard 闸, 直接裸调 collect。
判定: worker 线程持续完成 collect 即健康; 若超过宽限期没有任何完成
(0 进度推进), 判定死锁复现, 退出码 1。
用法:
uv run python scripts/stress_polars_concurrency.py [--duration 180] [--threads 8]
"""
from __future__ import annotations
import argparse
import threading
import time
from pathlib import Path
from tempfile import TemporaryDirectory
import polars as pl
def _build_dataset(root: Path, symbols: int = 300, days: int = 120) -> None:
"""构造 ~数十万行、按日期分区的 parquet 目录 (模拟 enriched 布局)。"""
dates = pl.date_range(
__import__("datetime").date(2026, 1, 1),
__import__("datetime").date(2026, 12, 31),
"1d",
eager=True,
).to_list()[:days]
for d in dates:
df = pl.DataFrame({
"symbol": [f"{i:06d}.SZ" for i in range(symbols)],
"date": [d] * symbols,
"close": [10.0 + (i % 37) for i in range(symbols)],
"volume": [1_000.0 * (i + 1) for i in range(symbols)],
})
out = root / f"date={d.isoformat()}" / "part.parquet"
out.parent.mkdir(parents=True, exist_ok=True)
df.write_parquet(out)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--duration", type=float, default=180.0)
parser.add_argument("--threads", type=int, default=8)
parser.add_argument("--grace", type=float, default=60.0)
args = parser.parse_args()
stop = threading.Event()
stats = {"collects": 0, "last_done": time.monotonic()}
lock = threading.Lock()
with TemporaryDirectory(prefix="polars_stress_") as tmp:
root = Path(tmp) / "kline"
print("building dataset ...", flush=True)
_build_dataset(root)
glob = str(root / "**" / "*.parquet")
def worker(kind: str) -> None:
i = 0
while not stop.is_set():
i += 1
if kind == "interactive":
df = (
pl.scan_parquet(glob)
.filter((pl.col("symbol") == f"{(i * 7) % 300:06d}.SZ"))
.sort("date")
.collect()
)
else:
df = (
pl.scan_parquet(glob)
.filter(pl.col("volume") > 100_000.0)
.group_by("symbol")
.agg(pl.col("close").mean().alias("avg_close"))
.sort("symbol")
.collect()
)
del df
with lock:
stats["collects"] += 1
stats["last_done"] = time.monotonic()
print(f"stressing: {args.threads} threads for {args.duration:.0f}s "
f"(polars {pl.__version__})", flush=True)
threads = [
threading.Thread(
target=worker,
args=("interactive" if i % 2 == 0 else "background",),
daemon=True,
)
for i in range(args.threads)
]
for t in threads:
t.start()
deadline = time.monotonic() + args.duration
wedged = False
while time.monotonic() < deadline:
time.sleep(5.0)
with lock:
idle = time.monotonic() - stats["last_done"]
done = stats["collects"]
print(f" progress: {done} collects, idle {idle:.1f}s", flush=True)
if idle > args.grace:
wedged = True
break
stop.set()
for t in threads:
t.join(timeout=10)
alive = [t for t in threads if t.is_alive()]
with lock:
total = stats["collects"]
if wedged or alive:
print(f"RESULT: DEADLOCK REPRODUCED — wedged={wedged}, "
f"stuck_threads={len(alive)}/{args.threads}, total_collects={total}", flush=True)
return 1
print(f"RESULT: HEALTHY — {total} collects completed, all threads exited", flush=True)
return 0
if __name__ == "__main__":
raise SystemExit(main())