Merge pull request #233 from SeerGlaucus/fix/etf-backtest-turnover-rate

fix(backtest): ETF矩阵回测因缺失换手率字段报错

backend/app/backtest/matrix.py

@@ -1427,7 +1427,18 @@ def _populate_matrix_derived_arrays(
     if "turnover_rate" in wanted_fields and "turnover_rate" not in parquet_fields:
         float_shares = fields.get("float_shares")
         if float_shares is None:
+            # 非股票资产 (etf/index) 无股本数据: instruments 无 float_shares 列,
+            # 也无法从 parquet 读到 turnover_rate (数据源不提供, ETF 无换手率口径)。
+            # 此时矩阵中该字段保持全 NaN 列 (matrix_fields 已占位), 与运行期
+            # _optional_field 的降级语义一致, 供不需要换手率的策略正常回测。
+            # 若本应有股本 (vector_fields 含 float_shares) 却取不到值, 才是数据
+            # 异常, 由 _resolve_matrix_storage_fields 的 vector 装载路径显式失败。
+            if "float_shares" in vector_fields:
                 raise ValueError("matrix turnover_rate requires float_shares")
+            logger.info(
+                "turnover_rate unavailable (asset has no float_shares); keeping NaN column"
+            )
+        else:
             _write_turnover_rate_matrix(
                 fields["turnover_rate"],
                 arrays["volume"],

backend/app/backtest/strategy.py

@@ -487,20 +487,30 @@ _SHARE_CAP_FILTER_KEYS = (
     "float_cap_max",
 )
 
+# 换手率界同样依赖股本派生字段 (turnover_rate ← float_shares):
+# 非股票资产 (etf/index) 没有股本数据, 若保留非 None 的换手率界,
+# _basic_filter_dependencies 会解析出 turnover_rate 字段需求,
+# 矩阵缓存档构建时因无 float_shares 而失败 (matrix turnover_rate requires
+# float_shares)。与市值界同一族问题, 必须一并中和。
+_TURNOVER_FILTER_KEYS = (
+    "turnover_min",
+    "turnover_max",
+)
+
 
 def _basic_filter_for_asset(basic_filter: dict, asset_type: str) -> dict:
-    """非股票资产没有股本数据 (etf/index 维表只有 symbol/name), 市值与流通
-    市值界对它们既无意义也不可满足: 依赖解析前先置 None, 避免解析出
-    total_shares/float_shares 字段需求导致矩阵加载直接失败。
+    """非股票资产没有股本数据 (etf/index 维表只有 symbol/name), 市值、流通
+    市值与换手率界对它们既无意义也不可满足: 依赖解析前先置 None, 避免解析出
+    total_shares/float_shares/turnover_rate 字段需求导致矩阵加载直接失败。
 
     运行期过滤无需同步修改 —— polars 侧有列守卫 (engine._basic_filter_expr),
     矩阵侧 _optional_field 对缺失字段返回全 NaN 且 _apply_bound 跳过全 NaN
-    界, 二者对缺失股本列本就降级为 no-op。
+    界, 二者对缺失股本/换手率列本就降级为 no-op。
     """
     if asset_type == "stock" or not basic_filter:
         return basic_filter
     sanitized = dict(basic_filter)
-    for key in _SHARE_CAP_FILTER_KEYS:
+    for key in (*_SHARE_CAP_FILTER_KEYS, *_TURNOVER_FILTER_KEYS):
         sanitized[key] = None
     return sanitized
 

backend/tests/test_matrix_etf_share_fields.py

@@ -7,18 +7,29 @@ _resolve_matrix_storage_fields 抛
 "matrix parquet fields unavailable: ['float_shares', 'total_shares']"
 (用户反馈: ETF 因子挖掘死于「准备共享撮合矩阵」阶段)。非股票资产必须在
 依赖解析前中和市值界, 股票行为保持不变。
+
+换手率同族问题: common_filter 还强制 turnover_min: 0.0, 若不同时中和
+turnover_min/max, 依赖解析会进一步要求 turnover_rate, 而 ETF enriched
+窄表无 turnover_rate 列且无股本可派生 —— 旧代码在
+_populate_matrix_derived_arrays 直接抛
+"matrix turnover_rate requires float_shares", 使任何 ETF 矩阵回测
+(含内置 ETF 策略) 全部失败。非股票资产在字段派生阶段应将缺失的
+turnover_rate 降级为全 NaN 列 (与运行期 _optional_field 语义一致),
+股票行为保持不变。
 """
 from __future__ import annotations
 
 from datetime import date
 from pathlib import Path
 
+import numpy as np
 import polars as pl
 import pyarrow.dataset as pads
 import pytest
 
 from app.backtest.matrix import (
     _normalize_matrix_cache_fields,
+    _populate_matrix_derived_arrays,
     _resolve_matrix_storage_fields,
 )
 from app.backtest.strategy import (
@@ -139,3 +150,112 @@ def test_share_fields_still_unavailable_without_sanitization(tmp_path):
             frozenset({"name", "total_shares", "float_shares"}),
             _etf_instruments(),
         )
+
+
+def _etf_dataset_without_turnover(tmp_path: Path) -> pads.Dataset:
+    """真实 ETF enriched 窄表: 只有 OHLCV 基础列, 无 turnover_rate。"""
+    pl.DataFrame({
+        "symbol": ["510300.SH", "510500.SH"],
+        "date": [date(2024, 1, 2)] * 2,
+        "open": [4.0, 6.0],
+        "high": [4.1, 6.1],
+        "low": [3.9, 5.9],
+        "close": [4.0, 6.0],
+        "volume": [100.0, 200.0],
+        "amount": [400.0, 1200.0],
+        "raw_close": [4.0, 6.0],
+        "raw_high": [4.1, 6.1],
+        "raw_low": [3.9, 5.9],
+    }).write_parquet(tmp_path / "part.parquet")
+    return pads.dataset(str(tmp_path / "part.parquet"), format="parquet")
+
+
+def test_etf_turnover_rate_degrades_to_nan_without_float_shares(tmp_path):
+    """ETF enriched 无 turnover_rate 列且维表无 float_shares 时,
+    派生阶段不得抛 "matrix turnover_rate requires float_shares":
+    turnover_rate 保持全 NaN 列 (与运行期 _optional_field 降级语义一致),
+    使不依赖换手率的 ETF 矩阵回测可以正常构建。"""
+    engine = _engine()
+    plan = _resolve_research_plan(engine, asset_type="etf")
+    assert "turnover_rate" in plan.matrix_columns  # 依赖链仍会请求该字段
+
+    dataset = _etf_dataset_without_turnover(tmp_path)
+    parquet_fields, matrix_fields, vector_fields = _resolve_matrix_storage_fields(
+        dataset,
+        frozenset({"close", "turnover_rate"}),
+        _etf_instruments(),
+    )
+    assert "turnover_rate" in matrix_fields
+    assert vector_fields == []  # 无股本可派生
+
+    shape = (2, 2)
+    arrays = {
+        "volume": np.array([[100.0, 200.0], [150.0, 250.0]], dtype=np.float32),
+    }
+    fields = {
+        "turnover_rate": np.full(shape, np.nan, dtype=np.float32),
+        "close": np.array([[4.0, 6.0], [4.2, 6.3]], dtype=np.float32),
+    }
+    seen = np.ones(shape, dtype=bool)
+    # 修复前: 抛 "matrix turnover_rate requires float_shares"
+    names, _limits = _populate_matrix_derived_arrays(
+        ["510300.SH", "510500.SH"],
+        arrays,
+        fields,
+        frozenset({"close", "turnover_rate"}),
+        _etf_instruments(),
+        seen,
+        parquet_fields=parquet_fields,
+        vector_fields=vector_fields,
+    )
+    assert names == ["沪深300ETF", "中证500ETF"]
+    assert np.isnan(fields["turnover_rate"]).all()  # 降级为 NaN 列
+
+
+def test_stock_turnover_rate_still_derived_when_float_shares_present(tmp_path):
+    """对照: 股票场景若维表提供 float_shares, turnover_rate 仍按
+    volume*10000/float_shares 正常派生 —— 修复只对"无股本资产"降级为 NaN,
+    不得影响有股本数据的派生路径。"""
+    engine = _engine()
+    plan = _resolve_research_plan(engine, asset_type="stock")
+    assert "turnover_rate" in plan.matrix_columns
+
+    stock_inst = pl.DataFrame({
+        "symbol": ["510300.SH", "510500.SH"],
+        "name": ["沪深300ETF", "中证500ETF"],
+        "code": ["510300", "510500"],
+        "asset_type": ["stock", "stock"],
+        "float_shares": [1.0e6, 2.0e6],  # 测试用极小股本, 便于断言非 NaN
+    })
+    dataset = _etf_dataset_without_turnover(tmp_path)  # parquet 无 turnover_rate
+    parquet_fields, matrix_fields, vector_fields = _resolve_matrix_storage_fields(
+        dataset,
+        frozenset({"close", "turnover_rate", "float_shares"}),
+        stock_inst,
+    )
+    assert "turnover_rate" in matrix_fields
+    assert vector_fields == ["float_shares"]  # 有股本 -> 走派生
+
+    shape = (2, 2)
+    arrays = {
+        "volume": np.array([[100.0, 200.0], [150.0, 250.0]], dtype=np.float32),
+    }
+    fields = {
+        "turnover_rate": np.full(shape, np.nan, dtype=np.float32),
+        "close": np.array([[4.0, 6.0], [4.2, 6.3]], dtype=np.float32),
+    }
+    seen = np.ones(shape, dtype=bool)
+    _names, _limits = _populate_matrix_derived_arrays(
+        ["510300.SH", "510500.SH"],
+        arrays,
+        fields,
+        frozenset({"close", "turnover_rate", "float_shares"}),
+        stock_inst,
+        seen,
+        parquet_fields=parquet_fields,
+        vector_fields=vector_fields,
+    )
+    # volume(手)*10000/float_shares: 100*10000/1e6 = 1.0
+    assert np.isfinite(fields["turnover_rate"]).all()
+    assert float(fields["turnover_rate"][0, 0]) == pytest.approx(1.0)
+    assert float(fields["turnover_rate"][1, 0]) == pytest.approx(1.5)