fix: parallel scan pickle bug — pass strategy file path instead of class to child processes

This commit is contained in:
Justin Gu
2026-06-11 04:24:09 +08:00
parent ddcb9d4425
commit c9ed57e66d
5 changed files with 133 additions and 8 deletions
+9
View File
@@ -1308,6 +1308,15 @@ ruff format --check src/ tests/ # format check
## Changelog
### 1.9.9 (2026-06-11)
**Bug 修复** — 修复并发扫描(`--workers`)在动态加载策略时静默返回空结果的问题。
- **根因**`ProcessPoolExecutor` 将动态 `importlib` 加载的策略类 pickle 序列化后发送到子进程,子进程无法反序列化(模块未注册到 `sys.modules`),异常被 `except` 静默吞掉
- **修复**`_scan_parallel` 改为传递策略文件路径(字符串),子进程内通过 `_load_strategy_class` 自行加载策略类
- 新增 `_get_strategy_file` 辅助函数:从类方法 `co_filename` 反查策略文件路径
- 新增回归测试 `TestParallelPickleFix`
### 1.9.8 (2026-06-11)
**CI 修复** — 修复 CI 流水线 ruff 和 pytest 配置问题。
+1 -1
View File
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "easy-tdx"
version = "1.9.8"
version = "1.9.9"
description = "通达信 TCP 协议行情数据客户端,支持在线行情、离线数据读取与写入同步"
readme = "README.md"
requires-python = ">=3.10"
+2 -2
View File
@@ -77,8 +77,8 @@ def scan(
if workers > 0:
click.echo(f"并发: {workers} 进程", err=True)
if workers > 0 and cache_file:
click.echo("注意: 并发模式暂不支持增量缓存,--cache 参数将被忽略", err=True)
if cache_file:
click.echo("注意: 并发模式暂不支持增量缓存,--cache 仅串行模式生效", err=True)
if cache_file and workers <= 0:
click.echo(f"缓存: {cache_file}", err=True)
from .scanner import SignalScanner
+85 -5
View File
@@ -201,9 +201,13 @@ class SignalScanner:
"""并发扫描(ProcessPoolExecutor)。"""
import concurrent.futures
# 构建参数:每个任务需要的独立数据
# 策略类不可跨进程 pickle(动态 importlib 加载的类子进程无法解析),
# 改为传递策略文件路径,子进程自行加载
strategy_file = _get_strategy_file(self._strategy_cls)
# 构建参数:每个任务需要的独立数据(全部为可 pickle 的基础类型)
tasks = [
(str(filepath), market, code, self._strategy_cls, self._cash, self._commission)
(str(filepath), market, code, strategy_file, self._cash, self._commission)
for filepath, market, code in files
]
@@ -452,26 +456,29 @@ def _scan_one_file(
filepath: str,
market: str,
code: str,
strategy_cls: type[Strategy],
strategy_file: str,
cash: float,
commission: float,
) -> ScanResult | None:
"""顶层扫描函数(供 ProcessPoolExecutor 调用)。
将实例方法逻辑提取为独立函数,避免 pickle 实例方法的问题
在子进程内动态加载策略类,避免跨进程 pickle 序列化失败
逻辑与 SignalScanner._scan_one 完全一致。
Args:
filepath: .day 文件路径字符串
market: 市场代码(SZ/SH
code: 6 位股票代码
strategy_cls: Strategy 子类
strategy_file: 策略文件路径(子进程内动态加载)
cash: 初始资金
commission: 佣金率
Returns:
ScanResult 如果触发信号,否则 None
"""
# 子进程内加载策略类(每次调用都重新加载,开销可忽略)
strategy_cls = _load_strategy_class(strategy_file)
bars = read_daily_bars(filepath)
if len(bars) < 30:
return None
@@ -503,3 +510,76 @@ def _scan_one_file(
signal_date=signal_date,
last_close=last_close,
)
def _get_strategy_file(strategy_cls: type) -> str:
"""获取策略类所在的文件路径。
按优先级尝试:sys.modules → 类方法 co_filename → inspect.getfile。
适用于标准 import 和 importlib 动态加载的模块。
Args:
strategy_cls: Strategy 子类
Returns:
策略文件路径字符串
"""
import sys
# 1. 从 sys.modules 查找(适用于标准 import 加载的模块)
mod = sys.modules.get(strategy_cls.__module__)
if mod is not None and hasattr(mod, "__file__") and mod.__file__:
return mod.__file__
# 2. 从类自身定义的方法的 code object 反查文件路径
# (适用于 importlib 动态加载的模块,__module__ 是临时名但方法保留了源文件信息)
for attr_name in ("init", "next", "on_bar", "on_tick"):
method = strategy_cls.__dict__.get(attr_name)
if method is not None and hasattr(method, "__code__"):
filepath = method.__code__.co_filename
if filepath and not filepath.startswith("<"):
return filepath
# 3. 任意自定义方法
for attr_val in strategy_cls.__dict__.values():
if callable(attr_val) and hasattr(attr_val, "__code__"):
filepath = attr_val.__code__.co_filename
if filepath and not filepath.startswith("<"):
return filepath
raise ValueError(
f"策略类 {strategy_cls.__name__} 无法定位源文件路径,"
"并发模式(--workers)仅支持从 .py 文件加载的策略"
)
def _load_strategy_class(strategy_file: str) -> type[Strategy]:
"""在子进程内动态加载策略类。
与 CLI 的 _load_strategy 逻辑一致,提取为顶层函数以便子进程调用。
Args:
strategy_file: 策略文件路径
Returns:
Strategy 子类
"""
import importlib.util
file_path = Path(strategy_file)
spec = importlib.util.spec_from_file_location("strategy_module", file_path)
if spec is None or spec.loader is None:
return None # type: ignore[return-value]
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
for attr_name in dir(module):
obj = getattr(module, attr_name)
try:
if isinstance(obj, type) and issubclass(obj, Strategy) and obj is not Strategy:
return obj
except TypeError:
pass
return None # type: ignore[return-value]
+36
View File
@@ -115,6 +115,42 @@ class TestConcurrentScan:
assert progress[-1][2] == "done"
class TestParallelPickleFix:
"""回归测试:并发模式从策略文件加载(修复 pickle 序列化失败)。"""
def test_parallel_with_file_strategy(self, vipdoc: Path) -> None:
"""从 .py 文件加载的策略在并发模式下应正常工作。"""
# 使用项目自带的策略文件
strategy_path = Path("strategies/macd_cross.py")
if not strategy_path.exists():
pytest.skip("strategies/macd_cross.py not found")
import importlib.util
from easy_tdx.backtest.strategy import Strategy
spec = importlib.util.spec_from_file_location("strat", strategy_path)
assert spec is not None and spec.loader is not None
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
cls = None
for name in dir(mod):
obj = getattr(mod, name)
try:
if isinstance(obj, type) and issubclass(obj, Strategy) and obj is not Strategy:
cls = obj
break
except TypeError:
pass
assert cls is not None, "No Strategy subclass found in macd_cross.py"
scanner = SignalScanner(cls, vipdoc_path=vipdoc)
# 并发模式不应抛出异常(修复前会因为 pickle 失败而静默返回空列表)
results = scanner.scan(universe="all", workers=2)
# 结果应为列表(可能为空,取决于策略信号)
assert isinstance(results, list)
class TestIncrementalScan:
"""测试增量扫描."""