From ba83e0241b896c6515d15c74fc5968031b38ab3b Mon Sep 17 00:00:00 2001 From: SeerGlaucus Date: Thu, 3 Sep 2026 01:30:05 +0800 Subject: [PATCH] =?UTF-8?q?fix(backtest):=20ETF=E7=9F=A9=E9=98=B5=E5=9B=9E?= =?UTF-8?q?=E6=B5=8B=E4=B8=8D=E5=9B=A0=E7=BC=BA=E5=A4=B1=E6=8D=A2=E6=89=8B?= =?UTF-8?q?=E7=8E=87=E5=AD=97=E6=AE=B5=E8=80=8C=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 非股票资产(etf/index)无股本数据(维表仅 symbol/name/code/asset_type), 数据源也不提供换手率; 但矩阵缓存档 common_filter 强制 turnover_min:0.0, 依赖解析仍为 ETF 请求 turnover_rate(需 float_shares 派生), 导致矩阵缓存 构建失败。实测内置 low_volatility_leader 也报 "matrix turnover_rate requires float_shares"。 - _basic_filter_for_asset 将 turnover_min/max 一并中和(与市值界同族, 上次修复遗漏) - _populate_matrix_derived_arrays 对无股本的 etf/index 把缺失的 turnover_rate 降级为全 NaN 列(与运行期 _optional_field 语义一致), 股票派生路径行为不变 - 新增回归测试: ETF 无股本时降级 NaN 不报错; 有 float_shares 时仍正常派生 --- backend/app/backtest/matrix.py | 23 +++- backend/app/backtest/strategy.py | 20 ++- backend/tests/test_matrix_etf_share_fields.py | 120 ++++++++++++++++++ 3 files changed, 152 insertions(+), 11 deletions(-) diff --git a/backend/app/backtest/matrix.py b/backend/app/backtest/matrix.py index a70c69f..bcf0198 100644 --- a/backend/app/backtest/matrix.py +++ b/backend/app/backtest/matrix.py @@ -1427,12 +1427,23 @@ def _populate_matrix_derived_arrays( if "turnover_rate" in wanted_fields and "turnover_rate" not in parquet_fields: float_shares = fields.get("float_shares") if float_shares is None: - raise ValueError("matrix turnover_rate requires float_shares") - _write_turnover_rate_matrix( - fields["turnover_rate"], - arrays["volume"], - float_shares, - ) + # 非股票资产 (etf/index) 无股本数据: instruments 无 float_shares 列, + # 也无法从 parquet 读到 turnover_rate (数据源不提供, ETF 无换手率口径)。 + # 此时矩阵中该字段保持全 NaN 列 (matrix_fields 已占位), 与运行期 + # _optional_field 的降级语义一致, 供不需要换手率的策略正常回测。 + # 若本应有股本 (vector_fields 含 float_shares) 却取不到值, 才是数据 + # 异常, 由 _resolve_matrix_storage_fields 的 vector 装载路径显式失败。 + if "float_shares" in vector_fields: + raise ValueError("matrix turnover_rate requires float_shares") + logger.info( + "turnover_rate unavailable (asset has no float_shares); keeping NaN column" + ) + else: + _write_turnover_rate_matrix( + fields["turnover_rate"], + arrays["volume"], + float_shares, + ) return names, latest_limits diff --git a/backend/app/backtest/strategy.py b/backend/app/backtest/strategy.py index 7a5d83d..7540c20 100644 --- a/backend/app/backtest/strategy.py +++ b/backend/app/backtest/strategy.py @@ -487,20 +487,30 @@ _SHARE_CAP_FILTER_KEYS = ( "float_cap_max", ) +# 换手率界同样依赖股本派生字段 (turnover_rate ← float_shares): +# 非股票资产 (etf/index) 没有股本数据, 若保留非 None 的换手率界, +# _basic_filter_dependencies 会解析出 turnover_rate 字段需求, +# 矩阵缓存档构建时因无 float_shares 而失败 (matrix turnover_rate requires +# float_shares)。与市值界同一族问题, 必须一并中和。 +_TURNOVER_FILTER_KEYS = ( + "turnover_min", + "turnover_max", +) + def _basic_filter_for_asset(basic_filter: dict, asset_type: str) -> dict: - """非股票资产没有股本数据 (etf/index 维表只有 symbol/name), 市值与流通 - 市值界对它们既无意义也不可满足: 依赖解析前先置 None, 避免解析出 - total_shares/float_shares 字段需求导致矩阵加载直接失败。 + """非股票资产没有股本数据 (etf/index 维表只有 symbol/name), 市值、流通 + 市值与换手率界对它们既无意义也不可满足: 依赖解析前先置 None, 避免解析出 + total_shares/float_shares/turnover_rate 字段需求导致矩阵加载直接失败。 运行期过滤无需同步修改 —— polars 侧有列守卫 (engine._basic_filter_expr), 矩阵侧 _optional_field 对缺失字段返回全 NaN 且 _apply_bound 跳过全 NaN - 界, 二者对缺失股本列本就降级为 no-op。 + 界, 二者对缺失股本/换手率列本就降级为 no-op。 """ if asset_type == "stock" or not basic_filter: return basic_filter sanitized = dict(basic_filter) - for key in _SHARE_CAP_FILTER_KEYS: + for key in (*_SHARE_CAP_FILTER_KEYS, *_TURNOVER_FILTER_KEYS): sanitized[key] = None return sanitized diff --git a/backend/tests/test_matrix_etf_share_fields.py b/backend/tests/test_matrix_etf_share_fields.py index 2c3a0a6..ad69944 100644 --- a/backend/tests/test_matrix_etf_share_fields.py +++ b/backend/tests/test_matrix_etf_share_fields.py @@ -7,18 +7,29 @@ _resolve_matrix_storage_fields 抛 "matrix parquet fields unavailable: ['float_shares', 'total_shares']" (用户反馈: ETF 因子挖掘死于「准备共享撮合矩阵」阶段)。非股票资产必须在 依赖解析前中和市值界, 股票行为保持不变。 + +换手率同族问题: common_filter 还强制 turnover_min: 0.0, 若不同时中和 +turnover_min/max, 依赖解析会进一步要求 turnover_rate, 而 ETF enriched +窄表无 turnover_rate 列且无股本可派生 —— 旧代码在 +_populate_matrix_derived_arrays 直接抛 +"matrix turnover_rate requires float_shares", 使任何 ETF 矩阵回测 +(含内置 ETF 策略) 全部失败。非股票资产在字段派生阶段应将缺失的 +turnover_rate 降级为全 NaN 列 (与运行期 _optional_field 语义一致), +股票行为保持不变。 """ from __future__ import annotations from datetime import date from pathlib import Path +import numpy as np import polars as pl import pyarrow.dataset as pads import pytest from app.backtest.matrix import ( _normalize_matrix_cache_fields, + _populate_matrix_derived_arrays, _resolve_matrix_storage_fields, ) from app.backtest.strategy import ( @@ -139,3 +150,112 @@ def test_share_fields_still_unavailable_without_sanitization(tmp_path): frozenset({"name", "total_shares", "float_shares"}), _etf_instruments(), ) + + +def _etf_dataset_without_turnover(tmp_path: Path) -> pads.Dataset: + """真实 ETF enriched 窄表: 只有 OHLCV 基础列, 无 turnover_rate。""" + pl.DataFrame({ + "symbol": ["510300.SH", "510500.SH"], + "date": [date(2024, 1, 2)] * 2, + "open": [4.0, 6.0], + "high": [4.1, 6.1], + "low": [3.9, 5.9], + "close": [4.0, 6.0], + "volume": [100.0, 200.0], + "amount": [400.0, 1200.0], + "raw_close": [4.0, 6.0], + "raw_high": [4.1, 6.1], + "raw_low": [3.9, 5.9], + }).write_parquet(tmp_path / "part.parquet") + return pads.dataset(str(tmp_path / "part.parquet"), format="parquet") + + +def test_etf_turnover_rate_degrades_to_nan_without_float_shares(tmp_path): + """ETF enriched 无 turnover_rate 列且维表无 float_shares 时, + 派生阶段不得抛 "matrix turnover_rate requires float_shares": + turnover_rate 保持全 NaN 列 (与运行期 _optional_field 降级语义一致), + 使不依赖换手率的 ETF 矩阵回测可以正常构建。""" + engine = _engine() + plan = _resolve_research_plan(engine, asset_type="etf") + assert "turnover_rate" in plan.matrix_columns # 依赖链仍会请求该字段 + + dataset = _etf_dataset_without_turnover(tmp_path) + parquet_fields, matrix_fields, vector_fields = _resolve_matrix_storage_fields( + dataset, + frozenset({"close", "turnover_rate"}), + _etf_instruments(), + ) + assert "turnover_rate" in matrix_fields + assert vector_fields == [] # 无股本可派生 + + shape = (2, 2) + arrays = { + "volume": np.array([[100.0, 200.0], [150.0, 250.0]], dtype=np.float32), + } + fields = { + "turnover_rate": np.full(shape, np.nan, dtype=np.float32), + "close": np.array([[4.0, 6.0], [4.2, 6.3]], dtype=np.float32), + } + seen = np.ones(shape, dtype=bool) + # 修复前: 抛 "matrix turnover_rate requires float_shares" + names, _limits = _populate_matrix_derived_arrays( + ["510300.SH", "510500.SH"], + arrays, + fields, + frozenset({"close", "turnover_rate"}), + _etf_instruments(), + seen, + parquet_fields=parquet_fields, + vector_fields=vector_fields, + ) + assert names == ["沪深300ETF", "中证500ETF"] + assert np.isnan(fields["turnover_rate"]).all() # 降级为 NaN 列 + + +def test_stock_turnover_rate_still_derived_when_float_shares_present(tmp_path): + """对照: 股票场景若维表提供 float_shares, turnover_rate 仍按 + volume*10000/float_shares 正常派生 —— 修复只对"无股本资产"降级为 NaN, + 不得影响有股本数据的派生路径。""" + engine = _engine() + plan = _resolve_research_plan(engine, asset_type="stock") + assert "turnover_rate" in plan.matrix_columns + + stock_inst = pl.DataFrame({ + "symbol": ["510300.SH", "510500.SH"], + "name": ["沪深300ETF", "中证500ETF"], + "code": ["510300", "510500"], + "asset_type": ["stock", "stock"], + "float_shares": [1.0e6, 2.0e6], # 测试用极小股本, 便于断言非 NaN + }) + dataset = _etf_dataset_without_turnover(tmp_path) # parquet 无 turnover_rate + parquet_fields, matrix_fields, vector_fields = _resolve_matrix_storage_fields( + dataset, + frozenset({"close", "turnover_rate", "float_shares"}), + stock_inst, + ) + assert "turnover_rate" in matrix_fields + assert vector_fields == ["float_shares"] # 有股本 -> 走派生 + + shape = (2, 2) + arrays = { + "volume": np.array([[100.0, 200.0], [150.0, 250.0]], dtype=np.float32), + } + fields = { + "turnover_rate": np.full(shape, np.nan, dtype=np.float32), + "close": np.array([[4.0, 6.0], [4.2, 6.3]], dtype=np.float32), + } + seen = np.ones(shape, dtype=bool) + _names, _limits = _populate_matrix_derived_arrays( + ["510300.SH", "510500.SH"], + arrays, + fields, + frozenset({"close", "turnover_rate", "float_shares"}), + stock_inst, + seen, + parquet_fields=parquet_fields, + vector_fields=vector_fields, + ) + # volume(手)*10000/float_shares: 100*10000/1e6 = 1.0 + assert np.isfinite(fields["turnover_rate"]).all() + assert float(fields["turnover_rate"][0, 0]) == pytest.approx(1.0) + assert float(fields["turnover_rate"][1, 0]) == pytest.approx(1.5)