diff --git a/README.md b/README.md index 8680a44..f70b6f6 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,8 @@ | 🔀 **能力路由** | 多数据集(日K/除权/实时/分钟/盘口/财务,持续扩展)按源能力独立路由,任选组合 | [custom-data-source.md](./docs/custom-data-source.md) | | 🔍 **选股引擎** | 25 个内置策略 + 分钟策略 + 自定义信号 + AI 生成,Polars 毫秒级扫全 A 股 | [strategy.md](./docs/strategy.md) | | 📊 **指标流水线** | MA/EMA/MACD/RSI/KDJ/布林/量比等 68 列指标与信号,一次扫表落盘 enriched Parquet | [features.md](./docs/features.md) | -| 🧪 **回测研究** | 因子/策略/分钟回测 + 财务快照因子(点时口径),T+1/费用/滑点约束,结果可导出 | [features.md](./docs/features.md) | +| 🧪 **回测研究** | 因子/策略/分钟回测 + 财务快照因子(点时口径),T+1/费用/滑点约束,评分策略附带因子归因 | [features.md](./docs/features.md) | +| 🔬 **因子平台** | DSL 自定义因子(编辑器 25 算子点选/试算/版本) + 检验/组合,与策略双向联动(一键生成策略/触发器引用因子/回测归因) | [factor-platform-plan.md](./docs/factor-platform-plan.md) | | ⛏️ **因子挖掘** | 嵌套样本外搜索多因子排名组合,与自有策略对照,候选库显式发布、永不自动上线 | [mining.md](./docs/mining.md) | | 🌡️ **市场环境** | 情绪周期 6 阶段(连板梯队驱动)+ 概念/行业主线排名,与 5 档环境分并存 | [market-phase.md](./docs/market-phase.md) | | 🚨 **异动监控** | 竞价/盘中/偏移三类异动一页覆盖:同花顺风向标 + 当日信号聚合 + 交易所偏离值口径 | — | @@ -66,10 +67,11 @@ - **策略** Screener — Polars 毫秒级扫描全 A 股,日线/分钟策略统一单池,按策略声明周期自动路由执行 - **回测** Backtest — 四种研究视图: - **因子回测** — IC/IR、分层收益、多空组合,62+ 因子目录先筛掉无效指标 - - **策略回测** — 净值曲线、回撤、夏普、胜率,T+1/手续费/滑点/止损,SSE 流式进度 + - **策略回测** — 净值曲线、回撤、夏普、胜率、盈亏比、蒙卡回撤,T+1/手续费/滑点/止损,SSE 流式进度;评分因子策略附带「因子归因」(胜/败单入场信号日因子对比) - **分钟策略回测** — 逐交易日回放信号、分钟收盘入场,分钟级成交明细 - **验证** — 参数敏感性与滚动样本外 - 研究闭环:结果导出 CSV(概要/净值/交易明细/分标的统计) → 保存候选 → **一键载入复测** +- **因子** Factors — 检验/因子库/编辑器/组合四 tab:IC·分层·Newey-West 检验、自定义 DSL 因子(25 算子点选、双语字段、我的因子模板)、版本与生命周期管理;因子库可**一键生成排名策略**,策略触发器可直接引用因子条件 - **挖掘** Mining — 嵌套样本外因子与策略挖掘:训练区间因子方向重估 + 相关性去重 + 多因子排名组合搜索,自有策略作对照轨;候选入库,显式确认后才发布,永不自动上线 **📈 个股与板块分析** @@ -81,6 +83,8 @@ **🔔 监控与复盘** - **监控中心** Monitor — 策略/个股信号/价格/异动四类规则,支持自选分组作用域,盘中实时弹窗 + 语音播报(播报个股名称与信号) + 触发记录持久化 +- **持仓提醒** Lots — 记录个股/ETF 买入批次,自动生成止盈止损/到期监控规则 +- **信号库** Signals — 内置预计算信号 + 自定义条件信号(含因子条件与 AI 生成),供策略触发器/回测/监控统一取用 - **异动监控** Abnormal Moves — 按交易时间线三 tab: - **竞价异动** — 同花顺盘前风向标(含当日/次日真实收益对照、追高风险标记)+ 全市场竞价扫描(待采集任务) - **盘中异动** — 涨停/炸板/翘板/跌停/新高/新低/放量当日信号聚合,零新增采集 @@ -90,7 +94,7 @@ **🗄️ 数据与扩展** - **数据** Data — 本地数据画像与同步状态(维表/日K/除权/Enriched/指数/ETF/分钟K/财务),盘后管道与历史扩展 - **扩展分析** (动态菜单) — 把任意第三方/扩展数据字段配成一级菜单,与内置数据同台分析 -- **设置** Settings — 数据源与能力检测(能力路由矩阵、档位徽章)、AI 接口、实时监控、扩展页面、信号库、菜单与系统设置 +- **设置** Settings — 数据源与能力检测(能力路由矩阵、档位徽章)、AI 接口、实时监控、扩展页面、菜单与系统设置 @@ -346,37 +350,32 @@ fork同时请点个star哦,欢迎 Issue 和 PR。 --- -## 💬 交流群 - -欢迎加入交流群,一起讨论交流。作者个人维护的部分个性化接口,统一公布在群公告中,供大家免费使用。 - -交流群二维码 - ---- - -## ❤️ 支持项目 +## ❤️ 支持项目 / 💬 交流群
-如果这个项目对你有帮助,欢迎请作者喝杯咖啡 ☕ - - - - - - - + +
微信赞赏支付宝
微信赞赏码 · 感谢道友支持 愿一路长红支付宝收款码 · 打开支付宝扫一扫 + ❤️ 支持项目
+ 如果这个项目对你有帮助,欢迎请作者喝杯咖啡 ☕ + + + +
微信赞赏码 · 感谢道友支持 愿一路长红
愿道友一路长红 📈
+
+ 💬 交流群
+ 欢迎加入交流群,一起讨论交流
个性化接口统一公布在群公告,免费使用
+ + +
交流群二维码 · 个人维护的个性化接口见群公告
+
-愿道友一路长红 📈 -
-> 打赏完全自愿,金额不限;不用于购买任何功能、数据权限、投资建议 -> > 作者精力有限,优先响应赞助回馈,希望理解 --- diff --git a/backend/app/__init__.py b/backend/app/__init__.py index b5cb0c0..6104be4 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -2,7 +2,7 @@ import sys -__version__ = "0.2.2" +__version__ = "0.2.3" # Windows 默认 stdout/stderr 编码为 GBK(cp936),TickFlow SDK 内部输出含 emoji 的 # 指数/标的名称(如 \U0001f193)时会抛 UnicodeEncodeError,导致请求失败。 diff --git a/backend/app/api/backtest.py b/backend/app/api/backtest.py index e1d104c..e1dfd94 100644 --- a/backend/app/api/backtest.py +++ b/backend/app/api/backtest.py @@ -128,9 +128,9 @@ class FactorColumnsResponse(BaseModel): @router.get("/factor/columns") def factor_columns(): - """返回可用的因子列列表。""" - from app.backtest.factor import FACTOR_COLUMNS - return {"columns": FACTOR_COLUMNS} + """返回可用的因子列列表 (含运行期注册的自定义/复合因子)。""" + from app.factors.registry import factor_columns_view + return {"columns": factor_columns_view()} class FactorBacktestRequest(BaseModel): @@ -149,9 +149,10 @@ class FactorBacktestRequest(BaseModel): @router.post("/factor/run") def factor_run(req: FactorBacktestRequest, request: Request): """因子回测 — IC/IR 分析 + 分层回测。""" - from app.backtest.factor import FACTOR_COLUMNS, FactorBacktestService, FactorConfig + from app.backtest.factor import FactorBacktestService, FactorConfig + from app.factors.registry import factor_columns_view - if req.factor_name not in {item["id"] for item in FACTOR_COLUMNS}: + if req.factor_name not in {item["id"] for item in factor_columns_view()}: raise HTTPException(status_code=400, detail=f"不支持的因子: {req.factor_name}") engine = _get_engine(request) @@ -184,7 +185,7 @@ def factor_run(req: FactorBacktestRequest, request: Request): class FactorBatchRequest(BaseModel): - factor_names: list[str] = Field(..., min_length=1, max_length=64) + factor_names: list[str] = Field(..., min_length=1, max_length=96) # 目录 77 + 自定义余量 symbols: list[str] | None = None start: date | None = None end: date | None = None @@ -200,13 +201,13 @@ class FactorBatchRequest(BaseModel): def factor_batch(req: FactorBatchRequest, request: Request): """批量筛选因子, 同一批次只加载并计算一次数据面板。""" from app.backtest.factor import ( - FACTOR_COLUMNS, FactorBacktestService, FactorBatchConfig, ) + from app.factors.registry import factor_columns_view factor_names = list(dict.fromkeys(req.factor_names)) - allowed = {item["id"] for item in FACTOR_COLUMNS} + allowed = {item["id"] for item in factor_columns_view()} invalid = [name for name in factor_names if name not in allowed] if invalid: raise HTTPException(status_code=400, detail=f"不支持的因子: {', '.join(invalid)}") diff --git a/backend/app/api/factors.py b/backend/app/api/factors.py new file mode 100644 index 0000000..0879732 --- /dev/null +++ b/backend/app/api/factors.py @@ -0,0 +1,454 @@ +"""因子注册表 API — 因子库 (P1) + 公式校验/试算 (P2) + 自定义/复合因子 CRUD (P3)。""" +from __future__ import annotations + +from datetime import date, timedelta + +import polars as pl +from fastapi import APIRouter, HTTPException, Query, Request +from pydantic import BaseModel, Field + +from app.factors import store +from app.factors.dsl import FACTOR_COLUMN, compile_formula +from app.factors.registry import all_factors, unregister_factor + +router = APIRouter(prefix="/api/factors", tags=["factors"]) + + +@router.get("") +def list_factors(asset_type: str | None = Query(default=None, pattern="^(stock|etf)$")) -> dict: + """注册表因子列表; asset_type 过滤适用资产 (财务因子仅股票)。""" + specs = all_factors(asset_type=asset_type) + return { + "factors": [ + { + "id": spec.id, + "label": spec.label, + "group": spec.group, + "kind": spec.kind, + "version": spec.version, + "formula": spec.formula_text, + "direction": spec.direction, + "unit": spec.unit, + "warmup_bars": spec.warmup_bars, + "pit": spec.pit, + "asset_types": sorted(spec.asset_types), + "stability": spec.stability, + "scale_free": spec.scale_free, + "dependencies": sorted(spec.dependencies), + } + for spec in specs + ] + } + + +class FormulaValidateRequest(BaseModel): + formula: str = Field(..., min_length=1, max_length=2000) + + +def _compiled_payload(compiled) -> dict: + return { + "ok": compiled.ok, + "errors": [error.to_dict() for error in compiled.errors], + "dependencies": sorted(compiled.dependencies), + "referenced_factors": sorted(compiled.referenced_factors), + "warmup_bars": compiled.warmup_bars, + "cross_sectional": compiled.cross_sectional, + } + + +@router.post("/validate") +def validate_formula(req: FormulaValidateRequest) -> dict: + """公式校验: 语法/语义/窗口纪律/依赖推导, 编译期 fail-closed。""" + return _compiled_payload(compile_formula(req.formula)) + + +class FormulaTrialRequest(FormulaValidateRequest): + asset_type: str = Field(default="stock", pattern="^(stock|etf)$") + days: int = Field(default=40, ge=20, le=120) + + +@router.post("/trial") +def trial_formula(req: FormulaTrialRequest, request: Request) -> dict: + """公式试算: 最近 N 个交易日截面 Rank IC 快照 (复用回测面板与虚拟因子物化路径)。""" + compiled = compile_formula(req.formula) + if not compiled.ok: + raise HTTPException(status_code=400, detail={"errors": [error.to_dict() for error in compiled.errors]}) + + from app.api.backtest import _get_engine + + # 交易日 → 自然日换算 (A股年均 243 交易日 ≈ 1.48 自然日/交易日), 留 buffer + calendar_days = int((compiled.warmup_bars + req.days) * 1.6) + 15 + start = date.today() - timedelta(days=calendar_days) + # 面板基础物理列 (load_panel 只返回 parquet 物理列, 因子列由补算路径生成) + base_columns = ["symbol", "date", "open", "high", "low", "close", "volume", "amount", "turnover_rate"] + if "consecutive_limit_ups" in compiled.dependencies: + base_columns.append("consecutive_limit_ups") + engine = _get_engine(request) + panel = engine.load_panel(None, start, date.today(), columns=base_columns, asset_type=req.asset_type) + if panel.is_empty(): + raise HTTPException(status_code=400, detail="当前数据目录无可用历史数据, 无法试算") + + # 复用检验引擎同一条补算路径 (compute_indicators + 虚拟因子物化), 禁止第二套计算逻辑 + from app.backtest.factor import FactorBacktestService + + physical = set(panel.columns) + to_compute = set(compiled.referenced_factors) | { + dep for dep in compiled.dependencies if dep not in physical + } + if to_compute: + panel = FactorBacktestService._compute_missing_factors(panel, to_compute) + + if compiled.frame_transform is None: + raise HTTPException(status_code=500, detail="编译产物缺少帧变换") + prepared = compiled.frame_transform(panel) + if prepared is None: + raise HTTPException( + status_code=400, + detail={"errors": [{ + "code": "E013", "message": "依赖列不可用: 面板缺少公式所需列", + "position": {"offset": 0, "line": 1}, + "detail": {"missing": sorted((compiled.dependencies | compiled.referenced_factors) - set(panel.columns))}, + }]}, + ) + + total_rows = panel.height + frame = ( + prepared + .with_columns( + (pl.col("close").shift(-1).over("symbol") / pl.col("close") - 1.0).alias("_next_return") + ) + .filter(pl.col(FACTOR_COLUMN).is_not_null()) + .unique(subset=["symbol", "date"], keep="last") + .sort(["symbol", "date"]) + ) + non_null_rows = frame.height + if non_null_rows == 0: + return { + "ok": True, "n_dates": 0, "null_ratio": 1.0, + "ic_mean": None, "ic_std": None, "ir": None, "ic_win_rate": None, + "ic_series": [], "message": "试算区间内公式输出全为空 (检查预热窗口与数据范围)", + } + + ic_frame = ( + frame.filter(pl.col("_next_return").is_not_null()) + .group_by("date") + .agg( + pl.corr(pl.col(FACTOR_COLUMN).rank(method="average"), pl.col("_next_return").rank(method="average")).alias("ic"), + pl.len().alias("n_symbols"), + ) + .filter(pl.col("ic").is_not_null()) + .sort("date") + .tail(req.days) + ) + ic_series = [ + {"date": str(row["date"]), "ic": round(row["ic"], 4), "n_symbols": row["n_symbols"]} + for row in ic_frame.to_dicts() + ] + if ic_frame.is_empty(): + return { + "ok": True, "n_dates": 0, "null_ratio": round(1.0 - non_null_rows / max(total_rows, 1), 4), + "ic_mean": None, "ic_std": None, "ir": None, "ic_win_rate": None, + "ic_series": [], "message": "无有效 IC 截面 (需每期 ≥2 只标的)", + } + stats = ic_frame.select( + pl.col("ic").mean().alias("mean"), + pl.col("ic").std(ddof=0).alias("std"), + (pl.col("ic") > 0).mean().alias("win"), + ).row(0, named=True) + ic_std = stats["std"] + # Newey-West t (lag=1): 与检验页同源口径, 样本过少时不给 (fail-closed) + t_newey_west = None + if ic_frame.height >= 5: + from app.backtest.stats_v2 import newey_west_t + + values = ic_frame["ic"].to_numpy() + nw = newey_west_t(values, lag=1) + if nw is not None: + t_newey_west = round(float(nw[0]), 3) + return { + "ok": True, + "n_dates": ic_frame.height, + "null_ratio": round(1.0 - non_null_rows / max(total_rows, 1), 4), + "ic_mean": round(stats["mean"], 4), + "ic_std": None if ic_std is None else round(ic_std, 4), + "ir": None if not ic_std or ic_std == 0 else round(stats["mean"] / ic_std, 3), + "ic_win_rate": round(stats["win"], 4), + "t_newey_west": t_newey_west, + "ic_series": ic_series, + } + + +# ── 自定义/复合因子 CRUD (P3) ────────────────────────────── + + +class CustomFactorCreateRequest(BaseModel): + id: str | None = Field(default=None, max_length=48) + label: str = Field(..., min_length=1, max_length=32) + group: str = Field(default="自定义", max_length=16) + formula: str = Field(..., min_length=1, max_length=2000) + description: str = Field(default="", max_length=500) + direction: str = Field(default="none", pattern="^(high|low|none)$") + + +class CompositeFactorCreateRequest(BaseModel): + id: str | None = Field(default=None, max_length=48) + label: str = Field(..., min_length=1, max_length=32) + group: str = Field(default="组合", max_length=16) + members: dict[str, float] = Field(..., min_length=2, max_length=8) + description: str = Field(default="", max_length=500) + direction: str = Field(default="none", pattern="^(high|low|none)$") + + +def _data_dir(request: Request): + from pathlib import Path + + data_dir = getattr(getattr(request.app.state, "repo", None), "store", None) + root = getattr(data_dir, "data_dir", None) if data_dir is not None else None + if root is None: + raise HTTPException(status_code=500, detail="数据目录不可用") + return Path(root) + + +def _slugify_id(label: str, prefix: str) -> str: + base = "".join(ch if ch.isascii() and (ch.isalnum() or ch == "_") else "_" for ch in label.lower()) + candidate = f"{prefix}_{base}".strip("_")[:44] + import re + + candidate = re.sub(r"_+", "_", candidate) + return candidate or f"{prefix}_f" + + +def _resolve_id(requested: str | None, label: str, prefix: str) -> str: + return requested.strip() if requested and requested.strip() else _slugify_id(label, prefix) + + +def _next_version(data_dir, factor_id: str) -> int: + for definition in store.load_all(data_dir): + if str(definition.get("id")) == factor_id: + return int(definition.get("version", 1)) + 1 + return 1 + + +def _trial_nonempty(request: Request, formula: str, asset_type: str = "stock") -> None: + """保存前置校验: 公式在最近 40 个交易日有非空输出 (设计 §3.5, fail-closed)。""" + compiled = compile_formula(formula) + if not compiled.ok: + raise HTTPException(status_code=400, detail={"errors": [e.to_dict() for e in compiled.errors]}) + from app.api.backtest import _get_engine + from app.backtest.factor import FactorBacktestService + + calendar_days = int((compiled.warmup_bars + 40) * 1.6) + 15 + base_columns = ["symbol", "date", "open", "high", "low", "close", "volume", "amount", "turnover_rate"] + engine = _get_engine(request) + panel = engine.load_panel(None, date.today() - timedelta(days=calendar_days), date.today(), columns=base_columns, asset_type=asset_type) + if panel.is_empty(): + raise HTTPException(status_code=400, detail="当前无历史数据, 无法完成保存前试算 (fail-closed)") + physical = set(panel.columns) + to_compute = set(compiled.referenced_factors) | {d for d in compiled.dependencies if d not in physical} + if to_compute: + panel = FactorBacktestService._compute_missing_factors(panel, to_compute) + prepared = compiled.frame_transform(panel) if compiled.frame_transform else None + if prepared is None or prepared[FACTOR_COLUMN].is_not_null().sum() == 0: + raise HTTPException(status_code=400, detail="公式在最近 40 个交易日输出全为空, 拒绝保存") + + +@router.post("/custom") +def create_custom_factor(req: CustomFactorCreateRequest, request: Request) -> dict: + """保存自定义公式因子: 编译通过 + 服务端试算非空 (fail-closed)。""" + data_dir = _data_dir(request) + factor_id = _resolve_id(req.id, req.label, "uf") + definition = { + "id": factor_id, + "kind": "custom", + "version": _next_version(data_dir, factor_id), + "label": req.label, + "group": req.group, + "formula": req.formula, + "description": req.description, + "direction": req.direction, + "status": "draft", + "created_at": store._now(), + "updated_at": store._now(), + } + try: + store.to_spec(definition) # 先做 schema/id/编译校验 + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + _trial_nonempty(request, req.formula) + try: + store.register_definition(definition) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + store.save_one(data_dir, definition) + return {"ok": True, "id": factor_id, "version": definition["version"]} + + +@router.post("/composite") +def create_composite_factor(req: CompositeFactorCreateRequest, request: Request) -> dict: + """保存复合因子: 成员校验 + 循环引用检查 (无需试算, 值由成员物化路径计算)。""" + data_dir = _data_dir(request) + factor_id = _resolve_id(req.id, req.label, "cf") + definition = { + "id": factor_id, + "kind": "composite", + "version": _next_version(data_dir, factor_id), + "label": req.label, + "group": req.group, + "members": req.members, + "description": req.description, + "direction": req.direction, + "status": "draft", + "created_at": store._now(), + "updated_at": store._now(), + } + try: + store.register_definition(definition) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + store.save_one(data_dir, definition) + return {"ok": True, "id": factor_id, "version": definition["version"]} + + +class CustomFactorUpdateRequest(BaseModel): + label: str = Field(..., min_length=1, max_length=32) + group: str = Field(default="自定义", max_length=16) + formula: str = Field(..., min_length=1, max_length=2000) + description: str = Field(default="", max_length=500) + direction: str = Field(default="none", pattern="^(high|low|none)$") + + +@router.post("/custom/{factor_id}/update") +def update_custom_factor(factor_id: str, req: CustomFactorUpdateRequest, request: Request) -> dict: + """编辑已有自定义因子: 编译校验 + 试算非空 (与创建同一门禁) → 版本提升注册。 + + 公式变化时状态回 draft (生命周期语义: 编辑后需重新检验激活); 仅改名称/分组保留状态。 + """ + data_dir = _data_dir(request) + target = None + for definition in store.load_all(data_dir): + if str(definition.get("id")) == factor_id: + target = definition + break + if target is None: + raise HTTPException(status_code=404, detail=f"自定义因子不存在: {factor_id}") + if str(target.get("kind", "custom")) != "custom": + raise HTTPException(status_code=400, detail=f"仅自定义因子支持公式编辑 (kind={target.get('kind')})") + formula_changed = str(target.get("formula")) != req.formula + if formula_changed: + _trial_nonempty(request, req.formula) + target.update({ + "label": req.label, + "group": req.group, + "formula": req.formula, + "description": req.description, + "direction": req.direction, + "version": int(target.get("version", 1)) + 1, # 版本提升 → 注册表允许覆盖 + "status": "draft" if formula_changed else str(target.get("status", "draft")), + "updated_at": store._now(), + }) + try: + store.register_definition(target) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + store.save_one(data_dir, target) + return {"ok": True, "id": factor_id, "version": target["version"], "status": target["status"]} + + +def _find_references(data_dir, factor_id: str) -> list[str]: + """扫描策略与复合因子定义中的引用 (删除前 fail-closed 检查)。""" + references: list[str] = [] + strategies_dir = data_dir / "strategies" + if strategies_dir.is_dir(): + for file in strategies_dir.glob("*.json"): + try: + text = file.read_text(encoding="utf-8") + if factor_id in text: + references.append(f"strategies/{file.name}") + except OSError: + continue + for definition in store.load_all(data_dir): + if str(definition.get("id")) == factor_id: + continue + members = definition.get("members") + if isinstance(members, dict) and factor_id in members: + references.append(f"custom_factors/{definition.get('id')}.json") + return references + + +@router.delete("/custom/{factor_id}") +def delete_custom_factor(factor_id: str, request: Request, force: bool = Query(default=False)) -> dict: + """删除自定义/复合因子; 有引用时列出引用方并拒绝 (需 force)。""" + data_dir = _data_dir(request) + from app.factors.registry import get_factor + + if get_factor(factor_id) is None and not store.delete_one(data_dir, factor_id): + raise HTTPException(status_code=404, detail=f"因子不存在: {factor_id}") + references = _find_references(data_dir, factor_id) + if references and not force: + raise HTTPException( + status_code=409, + detail={"message": "该因子仍有引用, 拒绝删除 (可带 force=true 强制)", "references": references}, + ) + try: + unregister_factor(factor_id) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + store.delete_one(data_dir, factor_id) + return {"ok": True, "id": factor_id, "removed_references": references} + + +class FactorStatusRequest(BaseModel): + status: str = Field(..., pattern="^(draft|active|watch|retired)$") + + +@router.post("/custom/{factor_id}/status") +def update_factor_status(factor_id: str, req: FactorStatusRequest, request: Request) -> dict: + """生命周期状态迁移 (P4): draft->active->watch->retired, 编辑后回 draft。""" + data_dir = _data_dir(request) + target = None + for definition in store.load_all(data_dir): + if str(definition.get("id")) == factor_id: + target = definition + break + if target is None: + raise HTTPException(status_code=404, detail=f"自定义因子不存在: {factor_id}") + target["status"] = req.status + target["updated_at"] = store._now() + try: + # 动态因子先注销再注册: 元数据变更 (status/group) 不提升版本, + # 直接 register 会因"版本未提升"被拒 (启动加载后的真实路径) + unregister_factor(factor_id) + store.register_definition(target) # 状态与 stability 联动 + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + store.save_one(data_dir, target) + return {"ok": True, "id": factor_id, "status": req.status} + + +class FactorGroupRequest(BaseModel): + group: str = Field(..., min_length=1, max_length=24) + + +@router.post("/custom/{factor_id}/group") +def update_factor_group(factor_id: str, req: FactorGroupRequest, request: Request) -> dict: + """修改单个自定义/复合因子的分组 (内置因子分组与快照/预设绑定, 不可改)。""" + data_dir = _data_dir(request) + group = req.group.strip() + if not group: + raise HTTPException(status_code=400, detail="分组名不能为空") + target = None + for definition in store.load_all(data_dir): + if str(definition.get("id")) == factor_id: + target = definition + break + if target is None: + raise HTTPException(status_code=404, detail=f"自定义因子不存在: {factor_id}") + target["group"] = group + target["updated_at"] = store._now() + try: + unregister_factor(factor_id) + store.register_definition(target) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + store.save_one(data_dir, target) + return {"ok": True, "id": factor_id, "group": group} diff --git a/backend/app/api/mining.py b/backend/app/api/mining.py index c80777f..a72b80b 100644 --- a/backend/app/api/mining.py +++ b/backend/app/api/mining.py @@ -13,7 +13,6 @@ from fastapi import APIRouter, Header, HTTPException, Query, Request from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from sse_starlette.sse import EventSourceResponse -from app.backtest.factor import FACTOR_COLUMNS from app.backtest.mining import ( MAX_BEAM_WIDTH, MAX_COMBINATION_SIZE, @@ -21,6 +20,7 @@ from app.backtest.mining import ( evaluate_candidate_gate, ) from app.enriched_generation import EnrichedGenerationUnavailableError +from app.factors.registry import factor_columns_view from app.services import preferences from app.services.mining_jobs import ( RUN_STATUSES, @@ -31,6 +31,7 @@ from app.services.mining_jobs import ( MiningRunValidationError, ) from app.services.mining_preflight import ( + enriched_partition_dates, mining_availability, require_mining_availability, ) @@ -40,7 +41,9 @@ from app.services.mining_schedule import ( ) router = APIRouter(prefix="/api/backtest/mining", tags=["backtest"]) -_FACTOR_IDS = frozenset(str(item["id"]) for item in FACTOR_COLUMNS) +# 校验时动态读取 (含运行期注册的自定义/复合因子) +def _known_factor_ids() -> frozenset[str]: + return frozenset(str(item["id"]) for item in factor_columns_view()) _MAX_ARTIFACT_BYTES = 64 * 1024 * 1024 _SSE_POLL_SECONDS = 0.5 _SSE_HEARTBEAT_SECONDS = 15.0 @@ -87,7 +90,7 @@ class MiningStartRequest(BaseModel): @field_validator("factor_names") @classmethod def _known_factors(cls, values: list[str]) -> list[str]: - unknown = sorted(set(values) - _FACTOR_IDS) + unknown = sorted(set(values) - _known_factor_ids()) if unknown: raise ValueError(f"unknown mining factors: {unknown}") return values @@ -119,6 +122,38 @@ class MiningSchedulePatch(BaseModel): mining_budget_profile: Literal["balanced", "strict"] | None = None +class MiningAutoStartRequest(BaseModel): + """自动挖掘: 因子池由 L1 统计筛选自动生成, 不接受手动指定。""" + + model_config = ConfigDict(extra="forbid", strict=True) + + asset_type: Literal["stock", "etf"] = "stock" + start: date | None = None + end: date | None = None + budget_profile: Literal["exploratory", "balanced", "strict"] = "balanced" + commission_pct: float = Field(0.0002, ge=0.0, le=0.05, allow_inf_nan=False) + stamp_tax_pct: float = Field(0.0005, ge=0.0, le=0.05, allow_inf_nan=False) + slippage_bps: float = Field(5.0, ge=0.0, le=1000.0, allow_inf_nan=False) + correlation_threshold: float = Field(0.75, gt=0.0, le=1.0, allow_inf_nan=False) + force: bool = False + + @field_validator("start", "end", mode="before") + @classmethod + def _iso_dates(cls, value: Any) -> Any: + if isinstance(value, str): + try: + return date.fromisoformat(value) + except ValueError as exc: + raise ValueError("dates must use ISO YYYY-MM-DD format") from exc + return value + + @model_validator(mode="after") + def _date_range(self) -> MiningAutoStartRequest: + if self.start is not None and self.end is not None and self.start > self.end: + raise ValueError("start must not be after end") + return self + + @router.get("/availability") def get_availability( request: Request, @@ -237,6 +272,93 @@ def cancel_run(run_id: str, request: Request) -> dict[str, Any]: raise HTTPException(status_code=400, detail=str(exc)) from exc +@router.post("/auto") +def start_auto_run(payload: MiningAutoStartRequest, request: Request) -> dict[str, Any]: + """自动挖掘: L1 统计筛选全量因子 → 达标池 → 复用挖掘任务管理启动嵌套样本外验证。 + + 筛选结果随请求持久化 (request.auto_screening), 供结果页展示达标因子清单与 + 失败原因分布; 无达标因子时返回 started=false 而不是报错。 + """ + from app.services.auto_mining import screen_all_factors + + manager = _manager(request) + data_dir = request.app.state.repo.store.data_dir + try: + require_mining_availability( + data_dir, + asset_type=payload.asset_type, + budget_profile=payload.budget_profile, + start=payload.start, + end=payload.end, + ) + engine = getattr(request.app.state, "backtest_engine", None) + if engine is None: + from app.backtest.engine import BacktestEngine + + engine = BacktestEngine(request.app.state.repo) + request.app.state.backtest_engine = engine + all_dates = enriched_partition_dates(data_dir, payload.asset_type) + screen_end = payload.end or (all_dates[-1] if all_dates else date.today()) + screening = screen_all_factors( + engine, + asset_type=payload.asset_type, + start=payload.start, + end=screen_end, + profile=payload.budget_profile, + ) + except (MiningRunValidationError, ValueError) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except EnrichedGenerationUnavailableError as exc: + raise HTTPException( + status_code=400, + detail="行情数据正在更新(enriched 发布中), 请等数据更新完成后再开始挖掘", + ) from exc + + if not screening["pool"]: + return {"started": False, "reason": "no_qualified_factors", "screening": screening} + + worker_request = { + "factor_names": screening["pool"], + "strategy_ids": [], + "symbols": None, + "asset_type": payload.asset_type, + "start": payload.start.isoformat() if payload.start else None, + "end": payload.end.isoformat() if payload.end else None, + "budget_profile": payload.budget_profile, + "commission_pct": payload.commission_pct, + "stamp_tax_pct": payload.stamp_tax_pct, + "slippage_bps": payload.slippage_bps, + "correlation_threshold": payload.correlation_threshold, + "max_combination_factors": 4, + "beam_width": 12, + "max_finalists": MAX_FINALISTS, + "auto": True, + "auto_screening": screening, + } + try: + fingerprint = build_data_fingerprint( + request.app.state.repo, + request.app.state, + worker_request, + ) + manifest = manager.start( + worker_request, + fingerprint, + force=payload.force, + source="auto", + ) + except (MiningRunValidationError, ValueError) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except EnrichedGenerationUnavailableError as exc: + raise HTTPException( + status_code=400, + detail="行情数据正在更新(enriched 发布中), 请等数据更新完成后再开始挖掘", + ) from exc + except MiningRunStoreError as exc: + raise HTTPException(status_code=500, detail="failed to persist mining run") from exc + return {"started": True, "run": _project_run(manager.store, manifest), "screening": screening} + + @router.get("/runs/{run_id}/result") def get_result(run_id: str, request: Request) -> dict[str, Any]: store = _manager(request).store diff --git a/backend/app/api/signals.py b/backend/app/api/signals.py index 2da2451..fec23bb 100644 --- a/backend/app/api/signals.py +++ b/backend/app/api/signals.py @@ -87,6 +87,24 @@ def get_options(): groups.append({"key": cat, "label": label, "fields": [{"key": f, "label": ENRICHED_COLUMNS.get(f, f)} for f in cat_fields]}) + # 注册表因子 (虚拟/自定义/复合): 历史路径由 compute_signals 复用评分物化 + # 管线补算; 已是物化列的基础因子 (rsi_14 等) 上面已分组, 此处跳过。 + from app.factors.registry import all_factors + + factor_groups: dict[str, list[dict[str, str]]] = {} + for spec in all_factors(): + if spec.id in allowed: + continue + label = spec.label + if spec.warmup_bars > 1: + label = f"{label} · 预热{spec.warmup_bars}日" + if list(spec.asset_types) == ["stock"]: + label = f"{label} · 仅股票" + factor_groups.setdefault(spec.group or "因子", []).append({"key": spec.id, "label": label}) + for group_label, group_fields in factor_groups.items(): + groups.append({"key": f"factor:{group_label}", "label": f"因子 · {group_label}", "fields": group_fields}) + fields.extend(group_fields) + return { "fields": fields, "groups": groups, diff --git a/backend/app/backtest/factor.py b/backend/app/backtest/factor.py index bc0f915..1219e46 100644 --- a/backend/app/backtest/factor.py +++ b/backend/app/backtest/factor.py @@ -17,12 +17,14 @@ from typing import Any, Literal import numpy as np import polars as pl +from app.backtest import stats_v2 from app.backtest.engine import BacktestEngine from app.backtest.fundamentals import ( FUNDAMENTAL_FACTOR_NAMES, attach_fundamental_factors, load_fundamental_snapshot, ) +from app.factors.registry import factor_columns_view as _factor_columns_view from app.strategy.scoring import ( VIRTUAL_SCORING_DEPENDENCIES as DERIVED_FACTOR_DEPENDENCIES, ) @@ -33,80 +35,8 @@ from app.strategy.scoring import ( logger = logging.getLogger(__name__) # 可研究因子目录。保留历史 ID 兼容已有候选方案; 价格尺度相关指标优先提供归一化版本。 -FACTOR_COLUMNS: list[dict] = [ - {"id": "momentum_5d", "label": "5日动量", "group": "动量", "desc": "5个交易日累计收益率"}, - {"id": "momentum_10d", "label": "10日动量", "group": "动量", "desc": "10个交易日累计收益率"}, - {"id": "momentum_20d", "label": "20日动量", "group": "动量", "desc": "20个交易日累计收益率"}, - {"id": "momentum_30d", "label": "30日动量", "group": "动量", "desc": "30个交易日累计收益率"}, - {"id": "momentum_60d", "label": "60日动量", "group": "动量", "desc": "60个交易日累计收益率"}, - {"id": "change_pct", "label": "日涨跌幅", "group": "动量", "desc": "当日收盘相对前收盘的收益率"}, - - {"id": "ma5_bias", "label": "MA5乖离", "group": "均线偏离", "desc": "收盘价 / MA5 - 1"}, - {"id": "ma10_bias", "label": "MA10乖离", "group": "均线偏离", "desc": "收盘价 / MA10 - 1"}, - {"id": "ma20_bias", "label": "MA20乖离", "group": "均线偏离", "desc": "收盘价 / MA20 - 1"}, - {"id": "ma30_bias", "label": "MA30乖离", "group": "均线偏离", "desc": "收盘价 / MA30 - 1"}, - {"id": "ma60_bias", "label": "MA60乖离", "group": "均线偏离", "desc": "收盘价 / MA60 - 1"}, - {"id": "ema5_bias", "label": "EMA5乖离", "group": "均线偏离", "desc": "收盘价 / EMA5 - 1"}, - {"id": "ema10_bias", "label": "EMA10乖离", "group": "均线偏离", "desc": "收盘价 / EMA10 - 1"}, - {"id": "ema20_bias", "label": "EMA20乖离", "group": "均线偏离", "desc": "收盘价 / EMA20 - 1"}, - {"id": "ema30_bias", "label": "EMA30乖离", "group": "均线偏离", "desc": "收盘价 / EMA30 - 1"}, - {"id": "ema60_bias", "label": "EMA60乖离", "group": "均线偏离", "desc": "收盘价 / EMA60 - 1"}, - - {"id": "rsi_6", "label": "RSI(6)", "group": "超买超卖", "desc": "6日相对强弱指标"}, - {"id": "rsi_14", "label": "RSI(14)", "group": "超买超卖", "desc": "14日相对强弱指标"}, - {"id": "rsi_24", "label": "RSI(24)", "group": "超买超卖", "desc": "24日相对强弱指标"}, - - {"id": "macd_hist", "label": "MACD柱(原值)", "group": "趋势", "desc": "兼容历史研究; 跨股票比较建议优先使用MACD柱强度"}, - {"id": "macd_dif_pct", "label": "MACD DIF强度", "group": "趋势", "desc": "MACD DIF / 收盘价"}, - {"id": "macd_dea_pct", "label": "MACD DEA强度", "group": "趋势", "desc": "MACD DEA / 收盘价"}, - {"id": "macd_hist_pct", "label": "MACD柱强度", "group": "趋势", "desc": "MACD柱 / 收盘价, 消除股价尺度影响"}, - {"id": "kdj_k", "label": "KDJ-K", "group": "趋势", "desc": "KDJ指标K值"}, - {"id": "kdj_d", "label": "KDJ-D", "group": "趋势", "desc": "KDJ指标D值"}, - {"id": "kdj_j", "label": "KDJ-J", "group": "趋势", "desc": "KDJ指标J值"}, - {"id": "boll_position", "label": "布林位置", "group": "趋势", "desc": "收盘价在布林带下轨到上轨之间的位置"}, - - {"id": "annual_vol_20d", "label": "20日波动率", "group": "波动率", "desc": "20日收益率年化标准差"}, - {"id": "atr_14", "label": "ATR(14)原值", "group": "波动率", "desc": "兼容历史研究; 跨股票比较建议优先使用ATR相对波动"}, - {"id": "atr_pct", "label": "ATR相对波动", "group": "波动率", "desc": "ATR(14) / 收盘价"}, - {"id": "amplitude", "label": "日振幅", "group": "波动率", "desc": "当日高低价差 / 前收盘价"}, - {"id": "boll_width", "label": "布林带宽", "group": "波动率", "desc": "布林带上下轨宽度 / MA20"}, - - {"id": "vol_ratio_5d", "label": "5日量比", "group": "量价", "desc": "当日成交量 / 前5日平均成交量"}, - {"id": "vol_ratio_10d", "label": "10日量比", "group": "量价", "desc": "当日成交量 / 前10日平均成交量"}, - {"id": "vol_trend_5_10", "label": "成交量趋势", "group": "量价", "desc": "5日平均成交量 / 10日平均成交量 - 1"}, - {"id": "turnover_rate", "label": "换手率", "group": "量价", "desc": "使用历史时点流通股本计算的当日换手率"}, - {"id": "turnover_ratio_5d", "label": "换手率放大", "group": "量价", "desc": "当日换手率 / 前5日平均换手率 - 1"}, - {"id": "log_amount", "label": "成交额对数", "group": "量价", "desc": "ln(成交额 + 1), 降低极端规模影响"}, - {"id": "amount_ratio_5d", "label": "成交额放大", "group": "量价", "desc": "当日成交额 / 前5日平均成交额 - 1"}, - - {"id": "gap_return", "label": "开盘跳空", "group": "价格位置", "desc": "开盘价 / 前收盘价 - 1"}, - {"id": "intraday_return", "label": "日内收益", "group": "价格位置", "desc": "收盘价 / 开盘价 - 1"}, - {"id": "close_position", "label": "收盘位置", "group": "价格位置", "desc": "收盘价在当日最低价到最高价之间的位置"}, - {"id": "distance_to_high_60d", "label": "距60日高点", "group": "价格位置", "desc": "收盘价 / 60日最高收盘价 - 1"}, - {"id": "distance_from_low_60d", "label": "距60日低点", "group": "价格位置", "desc": "收盘价 / 60日最低收盘价 - 1"}, - {"id": "vwap_bias", "label": "VWAP乖离", "group": "价格位置", "desc": "收盘价 / 当日成交均价 - 1, 成交均价 = 成交额 / (成交量x100)"}, - - {"id": "max_ret_20d", "label": "20日最大单日涨幅", "group": "收益形态", "desc": "近20个交易日单日涨幅最大值(彩票效应, 高值代表博彩型特征强)"}, - {"id": "ret_skew_20d", "label": "20日收益偏度", "group": "收益形态", "desc": "近20个交易日日收益偏度, 高值代表右偏(偶发大涨)"}, - {"id": "up_days_20d", "label": "20日上涨天数", "group": "收益形态", "desc": "近20个交易日中上涨天数(0~20)"}, - - {"id": "amihud_20d", "label": "20日Amihud非流动性", "group": "流动性", "desc": "近20日平均 |日涨跌幅| / 成交额(亿元), 高值代表流动性差"}, - {"id": "turnover_z_60d", "label": "换手率60日z分", "group": "流动性", "desc": "(当日换手率 - 前60日均值) / 前60日标准差, 衡量换手异动"}, - - {"id": "vol_price_corr_20d", "label": "20日量价相关", "group": "量价", "desc": "近20个交易日日涨跌幅与成交量的相关系数, 高值代表量价同向"}, - {"id": "vol_trend_5_60", "label": "量能趋势(5/60)", "group": "量价", "desc": "5日平均成交量 / 60日平均成交量 - 1"}, - - {"id": "limit_up_count_20d", "label": "涨停基因(20日)", "group": "涨停基因", "desc": "近20个交易日涨停次数"}, - {"id": "limit_up_count_60d", "label": "涨停基因(60日)", "group": "涨停基因", "desc": "近60个交易日涨停次数"}, - - {"id": "pb_latest", "label": "市净率(最新公告)", "group": "财务", "desc": "收盘价 / 最新已公告每股净资产; 无财务数据或公告前为空"}, - {"id": "roe_latest", "label": "ROE(最新公告)", "group": "财务", "desc": "最新已公告净资产收益率(%); 无财务数据或公告前为空"}, - {"id": "gross_margin_latest", "label": "毛利率(最新公告)", "group": "财务", "desc": "最新已公告销售毛利率(%)"}, - {"id": "net_margin_latest", "label": "净利率(最新公告)", "group": "财务", "desc": "最新已公告销售净利率(%)"}, - {"id": "revenue_yoy_latest", "label": "营收增速(最新公告)", "group": "财务", "desc": "最新已公告营业收入同比(%)"}, - {"id": "net_income_yoy_latest", "label": "净利增速(最新公告)", "group": "财务", "desc": "最新已公告归母净利润同比(%)"}, - {"id": "debt_ratio_latest", "label": "资产负债率(最新公告)", "group": "财务", "desc": "最新已公告资产负债率(%)"}, -] +# P1 起目录元数据单一权威来源为 app/factors/registry.py, 本常量为兼容别名 (顺序与键不变)。 +FACTOR_COLUMNS: list[dict] = _factor_columns_view() FACTOR_WARMUP_DAYS = 120 FACTOR_METHODOLOGY_VERSION = "factor_v2" @@ -208,6 +138,12 @@ class FactorBatchItem: yearly_ic: list[dict] = field(default_factory=list) ic_decay: list[dict] = field(default_factory=list) regime_stats: list[dict] = field(default_factory=list) + # metrics_v2 (P3): NW HAC t 值 (滞后=1, 日频 1 日前瞻) 与 BH-FDR q 值; 样本不足为 None + t_naive: float | None = None + t_newey_west: float | None = None + nw_lag: int | None = None + p_value: float | None = None + q_value: float | None = None @dataclass @@ -357,6 +293,10 @@ class FactorBacktestService: **evaluate_kwargs, ) long_short = result.long_short_stats + # metrics_v2: 由 IC 序列推导 NW HAC t 值与 p 值 (日频 1 日前瞻 → 滞后 1) + ic_values = [row.get("ic") for row in result.ic_series] + t_naive = stats_v2.naive_t(ic_values) + nw = stats_v2.newey_west_t(ic_values, lag=1) items.append(FactorBatchItem( factor_name=factor_name, label=str(meta.get("label", factor_name)), @@ -377,6 +317,14 @@ class FactorBacktestService: n_dates=result.n_dates, elapsed_ms=result.elapsed_ms, error=result.error, + t_naive=t_naive, + t_newey_west=nw[0] if nw else None, + nw_lag=1 if nw else None, + p_value=( + stats_v2.normal_two_sided_p(nw[0]) if nw + else stats_v2.normal_two_sided_p(t_naive) if t_naive is not None + else None + ), )) except Exception as exc: # 单因子失败不能中止整个筛选批次 logger.exception("factor batch item failed: %s", factor_name) @@ -390,6 +338,10 @@ class FactorBacktestService: n_symbols = max((item.n_symbols for item in items), default=0) n_dates = max((item.n_dates for item in items), default=0) + # metrics_v2: 批内 BH-FDR q 值 (m = 可检验因子数, 计算失败项不计) + q_values = stats_v2.bh_fdr_qvalues([item.p_value for item in items]) + for item, q_value in zip(items, q_values, strict=True): + item.q_value = q_value return FactorBatchResult( run_id=run_id, config=result_config, @@ -701,8 +653,31 @@ class FactorBacktestService: logger.warning("factors %s cannot be computed, missing columns: %s", factor_cols, missing) return panel + from app.factors.registry import get_factor from app.indicators.pipeline import compute_indicators + # custom/composite 因子走注册表→DSL/组合 的同一条物化路径 (P3, 与策略评分共用); + # 其底层依赖 (如 change_pct/ma20) 先经内置补算路径物化, 再做 DSL/组合物化。 + registry_names = { + name for name in factor_cols + if (spec := get_factor(name)) is not None and spec.kind in ("custom", "composite") + } + if registry_names: + base_deps: set[str] = set() + for name in registry_names: + spec = get_factor(name) + if spec is not None: + base_deps.update(spec.dependencies) + base_deps -= set(panel.columns) | registry_names + if base_deps: + panel = FactorBacktestService._compute_missing_factors( + panel, base_deps, assume_sorted=assume_sorted, + ) + panel = materialize_scoring_columns(panel, registry_names) + factor_cols = factor_cols - registry_names + if not factor_cols: + return panel + derived = factor_cols & set(DERIVED_FACTOR_DEPENDENCIES) indicator_columns = factor_cols - derived for factor_name in derived: diff --git a/backend/app/backtest/matrix.py b/backend/app/backtest/matrix.py index 7d7b17a..7337ab6 100644 --- a/backend/app/backtest/matrix.py +++ b/backend/app/backtest/matrix.py @@ -3768,6 +3768,12 @@ _MATRIX_COMPUTED_FEATURES = frozenset({ "amihud_20d", "turnover_z_60d", "vol_price_corr_20d", "vwap_bias", "vol_trend_5_60", "limit_up_count_20d", "limit_up_count_60d", + # --- 扩充批次 (2026-09-05): 与注册表/scoring 口径一致的 16 个新虚拟因子 --- + "log_float_mv", "mom_accel_20_60", "rsi_14_delta_5d", + "overnight_ret_20d", "intraday_ret_20d", "downside_vol_20d", + "vol_regime_5_60", "amplitude_trend_20_60", "obv_trend_20d", + "amount_mean_20d", "turnover_mean_20d", "turnover_std_20d", + "position_240d", "distance_to_high_240d", "kdj_kd_diff", }) @@ -4016,6 +4022,81 @@ def _compute_matrix_feature(market: MarketDataMatrix, name: str) -> np.ndarray: hits = np.where(np.isfinite(consecutive) & (consecutive > 0), np.float32(1.0), np.float32(0.0)) hits = hits.astype(np.float32) return valid_rolling_sum(hits, close_valid, window) + # --- 扩充批次 (2026-09-05): numpy 内核实现, 口径与 strategy/scoring.py 一致 --- + if name == "log_float_mv": + turnover = market.field("turnover_rate") + valid = close_valid & np.isfinite(turnover) & (turnover > 0) & (market.volume > 0) + out = np.full(market.shape, np.nan, dtype=np.float32) + np.multiply(market.close, market.volume, out=out, where=valid) + np.divide(out, turnover, out=out, where=valid) + np.log(out, out=out, where=valid) + return out + if name == "mom_accel_20_60" or name == "kdj_kd_diff": + left, right = ( + (matrix_feature(market, "momentum_20d"), matrix_feature(market, "momentum_60d")) + if name == "mom_accel_20_60" + else (matrix_feature(market, "kdj_k"), matrix_feature(market, "kdj_d")) + ) + out = np.full(market.shape, np.nan, dtype=np.float32) + np.subtract(left, right, out=out, where=np.isfinite(left) & np.isfinite(right)) + return out + if name == "rsi_14_delta_5d": + rsi = matrix_feature(market, "rsi_14") + return rsi - valid_shift(rsi, 5, np.isfinite(rsi)) + if name == "overnight_ret_20d": + overnight = _matrix_relative(market.open, matrix_feature(market, "prev_close")) + return valid_rolling_sum(overnight, np.isfinite(overnight), 20) + if name == "intraday_ret_20d": + intraday = _matrix_relative(market.close, market.open) + return valid_rolling_sum(intraday, np.isfinite(intraday), 20) + if name == "downside_vol_20d": + daily = matrix_feature(market, "change_pct") + downside = np.where( + np.isfinite(daily), np.minimum(daily, np.float32(0.0)), np.nan, + ).astype(np.float32) + mean_sq = valid_rolling_mean(np.square(downside, dtype=np.float32), np.isfinite(downside), 20) + out = np.full(market.shape, np.nan, dtype=np.float32) + np.sqrt(mean_sq, out=out, where=np.isfinite(mean_sq)) + return out + if name == "vol_regime_5_60": + daily = matrix_feature(market, "change_pct") + valid = np.isfinite(daily) + return _matrix_ratio( + valid_rolling_std(daily, valid, 5, ddof=1), + valid_rolling_std(daily, valid, 60, ddof=1), + ) + if name == "amplitude_trend_20_60": + amplitude = matrix_feature(market, "amplitude") + valid = np.isfinite(amplitude) + return _matrix_relative( + valid_rolling_mean(amplitude, valid, 20), + valid_rolling_mean(amplitude, valid, 60), + ) + if name == "obv_trend_20d": + daily = matrix_feature(market, "change_pct") + volume_valid = close_valid & np.isfinite(market.volume) + signed = np.where( + np.isfinite(daily), np.sign(daily) * market.volume, np.nan, + ).astype(np.float32) + total = valid_rolling_sum(signed, volume_valid & np.isfinite(daily), 20) + scale = valid_rolling_mean(market.volume, volume_valid, 20) * np.float32(20.0) + return _matrix_ratio(total, scale) + if name == "amount_mean_20d": + amount = market.field("amount") + return valid_rolling_mean(amount / np.float32(1e8), np.isfinite(amount), 20) + if name == "turnover_mean_20d" or name == "turnover_std_20d": + turnover = market.field("turnover_rate") + valid = np.isfinite(turnover) + mean = valid_rolling_mean(turnover, valid, 20) + if name == "turnover_mean_20d": + return mean + return _matrix_ratio(valid_rolling_std(turnover, valid, 20, ddof=1), mean) + if name == "position_240d": + high = valid_rolling_max(market.close, close_valid, 240) + low = valid_rolling_min(market.close, close_valid, 240) + return _matrix_ratio(market.close - low, high - low) + if name == "distance_to_high_240d": + return _matrix_relative(market.close, valid_rolling_max(market.close, close_valid, 240)) raise ValueError(f"unsupported matrix feature: {name}") diff --git a/backend/app/backtest/mining_runtime.py b/backend/app/backtest/mining_runtime.py index b05dc67..22ac457 100644 --- a/backend/app/backtest/mining_runtime.py +++ b/backend/app/backtest/mining_runtime.py @@ -17,7 +17,6 @@ import numpy as np import polars as pl from app.backtest.factor import ( - FACTOR_COLUMNS, FACTOR_METHODOLOGY_VERSION, FACTOR_WARMUP_DAYS, FactorBacktestService, @@ -57,6 +56,7 @@ from app.enriched_generation import ( EnrichedGenerationUnavailableError, enriched_publication_incomplete, ) +from app.factors.registry import factor_columns_view from app.services.mining_jobs import MiningRunStore from app.services.mining_preflight import enriched_partition_dates from app.services.mining_schedule import MINING_ALGORITHM_VERSION @@ -66,7 +66,6 @@ from app.strategy.engine import StrategyEngine ProgressCallback = Callable[[dict[str, Any]], None] CancelCheck = Callable[[], bool] | Any _PROFILE_NAMES = frozenset({"exploratory", "balanced", "strict"}) -_FACTOR_IDS = frozenset(str(item["id"]) for item in FACTOR_COLUMNS) _MINING_MATRIX_CACHE_BYTES = 32 * 1024 * 1024 _RESULT_POLICY = BacktestResultPolicy( required_stats=frozenset({"total_return", "sharpe", "max_drawdown", "n_trades"}), @@ -807,7 +806,9 @@ def _decode_runtime_request( factor_names = tuple(str(value) for value in request.get("factor_names") or ()) if not factor_names or len(set(factor_names)) != len(factor_names): raise ValueError("factor_names must be non-empty and unique") - unknown_factors = sorted(set(factor_names) - _FACTOR_IDS) + # 注册表动态读取: worker 子进程已在入口加载自定义/复合因子 + known_ids = frozenset(str(item["id"]) for item in factor_columns_view()) + unknown_factors = sorted(set(factor_names) - known_ids) if unknown_factors: raise ValueError(f"unknown mining factors: {unknown_factors}") if len(factor_names) > 48: @@ -1037,7 +1038,7 @@ def _build_artifacts( candidate.factor_names, candidate.directions, strict=True ): direction_by_factor.setdefault(factor_name, int(direction)) - metadata = {str(item["id"]): item for item in FACTOR_COLUMNS} + metadata = {str(item["id"]): item for item in factor_columns_view()} factor_rows = [] for factor_name in request.factor_names: metric = latest_metrics[factor_name] diff --git a/backend/app/backtest/stats_v2.py b/backend/app/backtest/stats_v2.py new file mode 100644 index 0000000..f2a94e5 --- /dev/null +++ b/backend/app/backtest/stats_v2.py @@ -0,0 +1,147 @@ +"""metrics_v2 统计函数 (P3) — Newey-West HAC t 值 / BH-FDR q 值 / DSR。 + +运行时零新增第三方依赖 (后端无 scipy/statsmodels), 全部 numpy 手写; +数值测试用固定黄金参考向量锁定 (tests/test_stats_v2.py)。 + +口径 (设计文档 factor-system-design.md §6): +- IC 序列因 h 日前瞻收益存在 h-1 阶移动平均自相关, 主口径 t 值取 NW HAC, 滞后 L=h。 +- 多因子批量检验按 Benjamini-Hochberg 步进法控制 FDR。 +- DSR (Deflated Sharpe Ratio, Bailey & Lopez de Prado 2014) 用于多重试验校正后的 + 夏普显著性; 期望最大夏普 EM = sqrt(V[SR]) * ((1-gamma)Φ^-1(1-1/N) + gammaΦ^-1(1-1/(Ne))) + 其中 gamma 为欧拉-马歇罗尼常数。 +""" +from __future__ import annotations + +import math + +import numpy as np + +EULER_GAMMA = 0.5772156649015329 + + +def _clean_values(values) -> np.ndarray: + array = np.asarray([value for value in values if value is not None and np.isfinite(value)], dtype=float) + return array + + +def newey_west_t(values, lag: int) -> tuple[float, float, float] | None: + """Newey-West HAC 稳健 t 统计量 (Bartlett 核)。 + + 返回 (t值, 均值, NW标准误); 样本不足 (n <= lag+2) 或方差为零返回 None。 + """ + array = _clean_values(values) + n = array.size + if n <= lag + 2 or n < 3: + return None + mean = float(array.mean()) + centered = array - mean + # 长方差 S = gamma0 + 2 Σ_l w_l gamma_l, w_l = 1 - l/(lag+1) (Bartlett) + gamma = [float(np.dot(centered[: n - lag_i], centered[lag_i:]) / n) for lag_i in range(lag + 1)] + long_variance = gamma[0] + for lag_i in range(1, lag + 1): + weight = 1.0 - lag_i / (lag + 1) + long_variance += 2.0 * weight * gamma[lag_i] + long_variance = max(long_variance, 0.0) + nw_se = math.sqrt(long_variance / n) + if nw_se == 0: + return None + return (mean - 0.0) / nw_se, mean, nw_se + + +def naive_t(values) -> float | None: + array = _clean_values(values) + n = array.size + if n < 3: + return None + std = float(array.std(ddof=1)) + if std == 0: + return None + return float(array.mean()) / (std / math.sqrt(n)) + + +def normal_two_sided_p(t_stat: float) -> float: + """标准正态双侧 p 值: erfc(|t|/sqrt(2))。""" + return math.erfc(abs(t_stat) / math.sqrt(2.0)) + + +def bh_fdr_qvalues(pvalues: list[float | None]) -> list[float | None]: + """Benjamini-Hochberg 步进法 q 值 (与输入等长, None 透传)。 + + m 取可检验假设数 (None 不计入); q_i = min over j>=rank_i { p_j * m / rank_j }, + 从大到小单调回填保证递增约束。 + """ + indexed = [ + (index, p) for index, p in enumerate(pvalues) + if p is not None and np.isfinite(p) + ] + qvalues: list[float | None] = [None] * len(pvalues) + if not indexed: + return qvalues + m = len(indexed) + indexed.sort(key=lambda pair: pair[1]) + running_min = float("inf") + for reverse_rank in range(len(indexed) - 1, -1, -1): + index, p = indexed[reverse_rank] + rank = reverse_rank + 1 + candidate = p * m / rank + running_min = min(running_min, candidate) + qvalues[index] = min(1.0, running_min) + return qvalues + + +def _normal_ppf(probability: float) -> float: + """标准正态分位数 Acklam 逆逼近 (相对误差 < 1.15e-9), 零依赖替代 scipy.stats.norm.ppf。""" + if not (0.0 < probability < 1.0): + raise ValueError("probability 必须在 (0,1) 开区间") + a = (-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, + 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00) + b = (-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, + 6.680131188771972e+01, -1.328068155288572e+01) + c = (-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, + -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00) + d = (7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, + 3.754408661907416e+00) + p_low, p_high = 0.02425, 1 - 0.02425 + if probability < p_low: + q_value = math.sqrt(-2 * math.log(probability)) + return (((((c[0] * q_value + c[1]) * q_value + c[2]) * q_value + c[3]) * q_value + c[4]) * q_value + c[5]) / \ + ((((d[0] * q_value + d[1]) * q_value + d[2]) * q_value + d[3]) * q_value + 1) + if probability <= p_high: + q_value = probability - 0.5 + r = q_value * q_value + return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q_value / \ + (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1) + q_value = math.sqrt(-2 * math.log(1 - probability)) + return -(((((c[0] * q_value + c[1]) * q_value + c[2]) * q_value + c[3]) * q_value + c[4]) * q_value + c[5]) / \ + ((((d[0] * q_value + d[1]) * q_value + d[2]) * q_value + d[3]) * q_value + 1) + + +def expected_max_sharpe(n_trials: int, variance_sharpes: float) -> float: + """N 次独立试验的期望最大夏普 EM (方差>0 时); 单次试验不校正。""" + if n_trials <= 1 or variance_sharpes <= 0: + return 0.0 + z1 = _normal_ppf(1.0 - 1.0 / n_trials) + z2 = _normal_ppf(1.0 - 1.0 / (n_trials * math.e)) + return math.sqrt(variance_sharpes) * ((1.0 - EULER_GAMMA) * z1 + EULER_GAMMA * z2) + + +def deflated_sharpe_psr( + sharpe: float, + n_obs: int, + skewness: float | None = None, + kurtosis: float | None = None, + expected_max_sharpe: float = 0.0, +) -> float | None: + """Deflated Sharpe (PSR 对 EM 校正) 概率; 参数不足或退化返回 None。 + + PSR = Φ( (SR - SR*) * sqrt(n-1) / sqrt(1 - gamma3 SR + (gamma4-1)/4 SR^2) ) + """ + if n_obs < 5 or not np.isfinite(sharpe): + return None + skewness = 0.0 if skewness is None else skewness + kurtosis = 3.0 if kurtosis is None else kurtosis + denominator = 1.0 - skewness * sharpe + (kurtosis - 1.0) / 4.0 * sharpe * sharpe + if denominator <= 0: + return None + statistic = (sharpe - expected_max_sharpe) * math.sqrt(n_obs - 1) / math.sqrt(denominator) + return 0.5 * (1.0 + math.erf(statistic / math.sqrt(2.0))) diff --git a/backend/app/backtest/strategy.py b/backend/app/backtest/strategy.py index 6b1b99a..023c946 100644 --- a/backend/app/backtest/strategy.py +++ b/backend/app/backtest/strategy.py @@ -591,6 +591,7 @@ class StrategyBacktestResult: trades: list[dict] = field(default_factory=list) per_symbol_stats: list[dict] = field(default_factory=list) strategy_info: dict = field(default_factory=dict) + factor_attribution: dict | None = None elapsed_ms: float = 0.0 error: str | None = None @@ -655,6 +656,56 @@ class BacktestResultPolicy: return {key: value for key, value in stats.items() if key in keep} +def _factor_attribution_summary( + snapshot: pl.DataFrame, + trades: list, +) -> dict | None: + """v1 因子归因: 入场信号日因子快照 x 成交盈亏, 对比盈利/亏损单因子均值。 + + snapshot 来自 _apply_score 物化的候选行 (与评分同一条计算管线), 模拟结束后 + 按 (symbol, 信号日) 关联成交。快照缺失、无可关联行或因子列全空时返回 None, + 归因失败不影响回测主结果。 + """ + factor_cols = [c for c in snapshot.columns if c not in ("symbol", "date")] + if not factor_cols or not trades: + return None + normalized = snapshot.with_columns( + pl.col("date").cast(pl.Utf8).str.slice(0, 10).alias("date") + ) + symbols: list[str] = [] + days: list[str] = [] + pnls: list[float] = [] + for trade in trades: + day = trade.entry_signal_date or trade.entry_date + if day is None: + continue + symbols.append(trade.symbol) + days.append(str(day)[:10]) + pnls.append(float(trade.pnl_pct)) + if not symbols: + return None + frame = pl.DataFrame({"symbol": symbols, "date": days, "pnl_pct": pnls}) + joined = frame.join(normalized, on=["symbol", "date"], how="left") + win = joined.filter(pl.col("pnl_pct") > 0) + lose = joined.filter(pl.col("pnl_pct") <= 0) + factors: list[dict] = [] + for col in factor_cols: + win_vals = win.get_column(col).drop_nulls().cast(pl.Float64) + lose_vals = lose.get_column(col).drop_nulls().cast(pl.Float64) + if win_vals.is_empty() and lose_vals.is_empty(): + continue + factors.append({ + "factor": col, + "win_mean": round(float(win_vals.mean()), 6) if not win_vals.is_empty() else None, + "lose_mean": round(float(lose_vals.mean()), 6) if not lose_vals.is_empty() else None, + "win_n": int(win_vals.len()), + "lose_n": int(lose_vals.len()), + }) + if not factors: + return None + return {"factors": factors, "n_win": win.height, "n_lose": lose.height} + + @dataclass(frozen=True) class PreparedMatrixBacktest: """Job-scoped immutable market data reused by every optimizer trial.""" @@ -1012,6 +1063,8 @@ class StrategyBacktestService: t0 = time.perf_counter() run_id = uuid.uuid4().hex[:10] result_policy = result_policy or BacktestResultPolicy() + # 因子归因快照容器: 日线路径在 _apply_score 里填充, 其余路径保持空 + factor_snapshot: dict = {} def _err(msg: str) -> StrategyBacktestResult: return StrategyBacktestResult( @@ -1505,7 +1558,7 @@ class StrategyBacktestService: candidate_filter_mask = self._build_candidate_filter_mask(panel, s, params) candidate_mask = basic_mask & candidate_filter_mask - panel = self._apply_score(panel, s, overrides, universe_mask=candidate_mask) + panel = self._apply_score(panel, s, overrides, universe_mask=candidate_mask, factor_snapshot=factor_snapshot) formal_candidate_mask = candidate_mask & formal_range entry_mask = self._build_entry_mask_from_candidate(panel, candidate_mask, s, entry_signals) entry_mask = entry_mask & formal_range @@ -1662,6 +1715,16 @@ class StrategyBacktestService: selected_stats = result_policy.select_stats(result.stats) + # 因子归因 (fail-open): 快照与成交按信号日关联, 失败只记日志不影响结果 + factor_attribution = None + if factor_snapshot and result.trades and result_policy.include_trades: + try: + factor_attribution = _factor_attribution_summary( + factor_snapshot["frame"], result.trades + ) + except Exception as exc: + logger.warning("factor attribution failed: %s", exc) + elapsed = (time.perf_counter() - t0) * 1000 return StrategyBacktestResult( @@ -1682,6 +1745,7 @@ class StrategyBacktestService: else [] ), strategy_info=strategy_info, + factor_attribution=factor_attribution, elapsed_ms=round(elapsed, 1), ) @@ -2465,6 +2529,7 @@ class StrategyBacktestService: s: StrategyDef, overrides: dict | None, universe_mask: pl.Series | None = None, + factor_snapshot: dict | None = None, ) -> pl.DataFrame: scoring = effective_scoring(s.meta.get("scoring"), overrides) directions = effective_scoring_directions(overrides) @@ -2475,6 +2540,18 @@ class StrategyBacktestService: if has_universe: work = work.with_columns(universe_mask.rename("_score_universe")) + # 因子归因快照: 在临时因子列被 _finish 丢弃前, 截取候选行的 + # (symbol, date, 因子值)。与评分共用同一份物化结果, 无第二次计算。 + if factor_snapshot is not None: + snapshot_cols = ["symbol", "date"] + [ + name for name in scoring if name in work.columns + ] + if len(snapshot_cols) > 2: + frame = work + if has_universe: + frame = frame.filter(pl.col("_score_universe")) + factor_snapshot["frame"] = frame.select(snapshot_cols) + def _value_in_universe(value: pl.Expr) -> pl.Expr: if has_universe: return pl.when(pl.col("_score_universe")).then(value).otherwise(None) diff --git a/backend/app/backtest/worker.py b/backend/app/backtest/worker.py index 6b4ca9d..e9a2225 100644 --- a/backend/app/backtest/worker.py +++ b/backend/app/backtest/worker.py @@ -182,6 +182,12 @@ def _worker_entry(task: dict[str, Any], event_queue, cancel_event) -> None: data_dir = Path(task["data_dir"]) store = DataStore(data_dir) repo = KlineRepository(store) + # 子进程不继承主进程的因子注册表; 自定义/复合因子 (uf_/cf_) 在任何 + # 涉及因子物化的 worker 任务里都依赖注册表, 启动时从存储加载。 + # 单个加载失败只跳过 (fail-open 跳过该因子), 与主进程启动行为一致。 + from app.factors.store import load_into_registry + + load_into_registry(data_dir) strategy_engine = StrategyEngine( strategy_dirs=_strategy_dirs(data_dir), override_loader=lambda sid: strategy_config.load_override(data_dir, sid), diff --git a/backend/app/factors/__init__.py b/backend/app/factors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/factors/dsl.py b/backend/app/factors/dsl.py new file mode 100644 index 0000000..33254b9 --- /dev/null +++ b/backend/app/factors/dsl.py @@ -0,0 +1,749 @@ +"""因子公式 DSL 编译器 (P2)。 + +流水线: text → tokenizer → 递归下降解析(EBNF 见设计文档 §3.4) → AST → 语义检查 +→ 依赖/预热推导 → Polars Expr。编译失败返回结构化错误 (E001-E016), 不抛裸异常。 + +窗口纪律 (Polars 嵌套窗口会静默产出全 null, 必须在编译期杜绝): +- 所有 ts_* 算子只向后看 (负 shift 常量层强制 E005)。 +- 时序子树仅在离开时序上下文时挂一次 over("symbol"); 截面算子挂 over("date")。 +- 截面算子消费含窗口的子树时, 编译为两阶段: 先把该子树物化为临时列 (单层 over), + 再对临时列做截面运算 —— frame_transform 负责按依赖顺序执行全部阶段。 +- 截面算子嵌在时序窗口内 (如 ts_mean(rank(x), n)) v1 不支持, 编译期 E009 拒绝。 +- 引用的注册因子(含 virtual)不内联表达式: 调用方用 materialize_scoring_columns + 物化成列, 编译产物统一以 pl.col(name) 引用; 运行期缺列即 fail-closed。 +""" +from __future__ import annotations + +import copy +import re +from dataclasses import dataclass, field +from functools import lru_cache +from typing import Any + +import polars as pl + +from app.factors.registry import factor_dependencies, get_factor + +FACTOR_COLUMN = "__dsl_factor__" + +# 基准列 (设计文档 §3.1); 指标列 = 注册表 base 因子, 已注册因子 id 经注册表解析。 +BASE_COLUMNS: frozenset[str] = frozenset({ + "open", "high", "low", "close", "volume", "amount", + "turnover_rate", "prev_close", "raw_close", +}) + +MAX_AST_DEPTH = 12 +MAX_TOKENS = 200 +WINDOW_MIN, WINDOW_MAX = 2, 512 +DELAY_MAX = 512 +POWER_ABS_MAX = 4.0 +WINSORIZE_K_RANGE = (1.0, 6.0) + +# 算子表: 名 -> (表达式参数个数, 常量参数名元组); 常量参数必须是数字字面量 (E003)。 +OPERATORS: dict[str, tuple[int, tuple[str, ...]]] = { + "ts_mean": (1, ("n",)), + "ts_std": (1, ("n",)), + "ts_sum": (1, ("n",)), + "ts_max": (1, ("n",)), + "ts_min": (1, ("n",)), + "ts_delay": (1, ("n",)), + "ts_delta": (1, ("n",)), + "ts_rank": (1, ("n",)), + "ts_zscore": (1, ("n",)), + "ts_corr": (2, ("n",)), + "ts_cov": (2, ("n",)), + "ts_quantile": (1, ("n", "q")), + "decay_linear": (1, ("n",)), + "rank": (1, ()), + "zscore": (1, ()), + "winsorize": (1, ("k",)), # k 可省略, 默认 3 + "power": (1, ("c",)), + "clamp": (1, ("lo", "hi")), + "if_else": (3, ()), + "min": (2, ()), + "max": (2, ()), + "log": (1, ()), + "abs": (1, ()), + "sign": (1, ()), + "sqrt": (1, ()), +} +TS_OPERATORS = frozenset({ + "ts_mean", "ts_std", "ts_sum", "ts_max", "ts_min", "ts_delay", "ts_delta", + "ts_rank", "ts_zscore", "ts_corr", "ts_cov", "ts_quantile", "decay_linear", +}) +CROSS_OPERATORS = frozenset({"rank", "zscore", "winsorize"}) + + +@dataclass +class DslError: + code: str + message: str + offset: int = 0 + detail: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict: + return { + "code": self.code, + "message": self.message, + "position": {"offset": self.offset, "line": 1}, + "detail": self.detail, + } + + +@dataclass +class CompiledFormula: + ok: bool + errors: list[DslError] = field(default_factory=list) + frame_transform: Any | None = None # (frame: pl.DataFrame) -> pl.DataFrame | None (缺列 None = E013) + dependencies: frozenset[str] = frozenset() # 展开到 enriched base 列 + referenced_factors: frozenset[str] = frozenset() # 引用的注册因子 id (含 virtual, 需物化) + warmup_bars: int = 1 + cross_sectional: bool = False + formula_text: str = "" + + +# ---------------------------------------------------------------- tokenizer + +_TOKEN_RE = re.compile( + r"\s*(?:(?P\d+(?:\.\d+)?)|(?P[A-Za-z_][A-Za-z0-9_]*)|(?P>=|<=|==|!=|[+\-*/><(),]))" +) +_KEYWORDS = frozenset({"and", "or", "not"}) + + +def _tokenize(text: str) -> tuple[list[tuple[str, Any, int]], DslError | None]: + tokens: list[tuple[str, Any, int]] = [] + pos = 0 + while pos < len(text): + match = _TOKEN_RE.match(text, pos) + if match is None or match.end() == pos: + rest = text[pos:].strip() + if not rest: + break + return [], DslError("E014", f"语法错误: 无法识别的字符 '{rest[0]}'", offset=pos) + if match.group("num") is not None: + tokens.append(("num", float(match.group("num")), match.start("num"))) + elif match.group("ident") is not None: + tokens.append(("ident", match.group("ident"), match.start("ident"))) + else: + tokens.append(("op", match.group("op"), match.start("op"))) + pos = match.end() + return tokens, None + + +# ------------------------------------------------------------------- parser +# AST 节点: dict(kind, value, children, offset[, _constants]) + + +class _Parser: + _CMP = frozenset({">", ">=", "<", "<=", "==", "!="}) + + def __init__(self, tokens: list[tuple[str, Any, int]], text: str) -> None: + self.tokens = tokens + self.text = text + self.index = 0 + + def _peek(self) -> tuple[str, Any, int] | None: + return self.tokens[self.index] if self.index < len(self.tokens) else None + + def _next(self) -> tuple[str, Any, int]: + token = self.tokens[self.index] + self.index += 1 + return token + + def parse(self) -> tuple[dict | None, DslError | None]: + if not self.tokens: + return None, DslError("E014", "语法错误: 表达式为空", offset=0) + node, error = self._or_expr() + if error: + return None, error + if self._peek() is not None: + _, value, offset = self._peek() + return None, DslError("E014", f"语法错误: 多余的记号 '{value}'", offset=offset) + return node, None + + def _or_expr(self): + left, error = self._and_expr() + if error: + return None, error + while (token := self._peek()) and token[0] == "ident" and token[1] == "or": + self._next() + right, error = self._and_expr() + if error: + return None, error + left = {"kind": "bin", "value": "or", "children": [left, right], "offset": token[2]} + return left, None + + def _and_expr(self): + left, error = self._cmp_expr() + if error: + return None, error + while (token := self._peek()) and token[0] == "ident" and token[1] == "and": + self._next() + right, error = self._cmp_expr() + if error: + return None, error + left = {"kind": "bin", "value": "and", "children": [left, right], "offset": token[2]} + return left, None + + def _cmp_expr(self): + left, error = self._add_expr() + if error: + return None, error + while (token := self._peek()) and token[0] == "op" and token[1] in self._CMP: + self._next() + right, error = self._add_expr() + if error: + return None, error + left = {"kind": "bin", "value": token[1], "children": [left, right], "offset": token[2]} + return left, None + + def _add_expr(self): + left, error = self._mul_expr() + if error: + return None, error + while (token := self._peek()) and token[0] == "op" and token[1] in ("+", "-"): + self._next() + right, error = self._mul_expr() + if error: + return None, error + left = {"kind": "bin", "value": token[1], "children": [left, right], "offset": token[2]} + return left, None + + def _mul_expr(self): + left, error = self._unary() + if error: + return None, error + while (token := self._peek()) and token[0] == "op" and token[1] in ("*", "/"): + self._next() + right, error = self._unary() + if error: + return None, error + left = {"kind": "bin", "value": token[1], "children": [left, right], "offset": token[2]} + return left, None + + def _unary(self): + token = self._peek() + if token and token[0] == "op" and token[1] == "-": + self._next() + operand, error = self._unary() + if error: + return None, error + return {"kind": "unary", "value": "-", "children": [operand], "offset": token[2]}, None + return self._primary() + + def _primary(self): + token = self._peek() + if token is None: + return None, DslError("E014", "语法错误: 表达式意外结束", offset=len(self.text)) + kind, value, offset = self._next() + if kind == "num": + return {"kind": "num", "value": value, "children": [], "offset": offset}, None + if kind == "ident": + if value in _KEYWORDS: + return None, DslError("E014", f"语法错误: 关键字 '{value}' 不能作为操作数", offset=offset) + nxt = self._peek() + if nxt and nxt[0] == "op" and nxt[1] == "(": + return self._call(value, offset) + return {"kind": "col", "value": value, "children": [], "offset": offset}, None + if kind == "op" and value == "(": + inner, error = self._or_expr() + if error: + return None, error + closing = self._peek() + if not (closing and closing[0] == "op" and closing[1] == ")"): + return None, DslError("E014", "语法错误: 缺少右括号 ')'", offset=offset) + self._next() + return inner, None + return None, DslError("E014", f"语法错误: 意外的记号 '{value}'", offset=offset) + + def _call(self, name: str, offset: int): + self._next() # consume '(' + args: list[dict] = [] + token = self._peek() + if not (token and token[0] == "op" and token[1] == ")"): + while True: + arg, error = self._or_expr() + if error: + return None, error + args.append(arg) + token = self._peek() + if token and token[0] == "op" and token[1] == ",": + self._next() + continue + break + closing = self._peek() + if not (closing and closing[0] == "op" and closing[1] == ")"): + return None, DslError("E014", f"语法错误: 函数 '{name}' 缺少右括号", offset=offset) + self._next() + return {"kind": "call", "value": name, "children": args, "offset": offset}, None + + +# ---------------------------------------------------------- semantic checks + + +def _ast_depth(node: dict) -> int: + if not node["children"]: + return 1 + return 1 + max(_ast_depth(child) for child in node["children"]) + + +def _collect_identifiers(node: dict, found: set[str]) -> None: + if node["kind"] == "col": + found.add(node["value"]) + for child in node["children"]: + _collect_identifiers(child, found) + + +def _const_value(node: dict) -> float | None: + if node["kind"] == "num": + return float(node["value"]) + if node["kind"] == "unary" and node["value"] == "-" and node["children"][0]["kind"] == "num": + return -float(node["children"][0]["value"]) + return None + + +def _check_call(node: dict, errors: list[DslError]) -> dict[str, float]: + """检查函数签名与常量参数范围; 返回解析出的常量参数表。""" + name = node["value"] + args = node["children"] + if name not in OPERATORS: + errors.append(DslError("E002", f"未知函数: {name}", offset=node["offset"], detail={"name": name})) + return {} + n_expr, const_names = OPERATORS[name] + has_optional_k = name == "winsorize" + total_min, total_max = n_expr + (0 if has_optional_k else len(const_names)), n_expr + len(const_names) + if not (total_min <= len(args) <= total_max): + errors.append(DslError( + "E003", f"函数 {name} 参数数量不符: 期望 {total_min}~{total_max} 个, 实际 {len(args)}", + offset=node["offset"], detail={"name": name, "args": len(args)}, + )) + return {} + constants: dict[str, float] = {} + for index, const_name in enumerate(const_names): + arg = args[n_expr + index] + value = _const_value(arg) + if value is None: + errors.append(DslError( + "E003", f"函数 {name} 的参数 {const_name} 必须是数字常量", + offset=arg["offset"], detail={"name": name, "param": const_name}, + )) + continue + constants[const_name] = value + if "n" in constants: + n_value = constants["n"] + if n_value != int(n_value): + errors.append(DslError("E004", "窗口参数必须是整数", offset=node["offset"], detail={"n": n_value})) + else: + n_int = int(n_value) + if n_int < 0 and name in ("ts_delay", "ts_delta"): + errors.append(DslError( + "E005", f"负 shift: {name} 的 n 必须 ≥ 0 (负数即未来函数)", + offset=node["offset"], detail={"n": n_int}, + )) + elif name == "ts_delay" and not (1 <= n_int <= DELAY_MAX): + errors.append(DslError("E004", f"ts_delay 的 n 必须在 [1,{DELAY_MAX}] 内", offset=node["offset"], detail={"n": n_int})) + elif name == "ts_delta" and not (0 <= n_int <= DELAY_MAX): + errors.append(DslError("E004", f"ts_delta 的 n 必须在 [0,{DELAY_MAX}] 内", offset=node["offset"], detail={"n": n_int})) + elif name not in ("ts_delay", "ts_delta") and not (WINDOW_MIN <= n_int <= WINDOW_MAX): + errors.append(DslError( + "E004", f"窗口 n 必须在 [{WINDOW_MIN},{WINDOW_MAX}] 内", offset=node["offset"], detail={"n": n_int}, + )) + if "q" in constants and not (0.0 < constants["q"] < 1.0): + errors.append(DslError("E004", "ts_quantile 的 q 必须在 (0,1) 开区间内", offset=node["offset"], detail={"q": constants["q"]})) + if "c" in constants and abs(constants["c"]) > POWER_ABS_MAX: + errors.append(DslError("E010", f"power 指数 |c| ≤ {POWER_ABS_MAX}", offset=node["offset"], detail={"c": constants["c"]})) + if "k" in constants and not (WINSORIZE_K_RANGE[0] <= constants["k"] <= WINSORIZE_K_RANGE[1]): + errors.append(DslError("E011", "winsorize 的 k 必须在 [1,6] 内", offset=node["offset"], detail={"k": constants["k"]})) + if "lo" in constants and "hi" in constants and constants["lo"] > constants["hi"]: + errors.append(DslError("E003", "clamp 的 lo 不能大于 hi", offset=node["offset"])) + return constants + + +def _semantic_walk(node: dict, errors: list[DslError], constants_by_call: dict[int, dict]) -> None: + if node["kind"] == "call": + constants_by_call[id(node)] = _check_call(node, errors) + for child in node["children"]: + _semantic_walk(child, errors, constants_by_call) + return + if node["kind"] == "bin" and node["value"] == "/": + right = node["children"][1] + if _const_value(right) == 0: + errors.append(DslError("E008", "静态除零: 分母为常量 0", offset=right["offset"])) + for child in node["children"]: + _semantic_walk(child, errors, constants_by_call) + + +# ------------------------------------------------------------- code generation + +_CMP_METHOD = {">": "gt", ">=": "ge", "<": "lt", "<=": "le", "==": "eq", "!=": "ne"} + + +def _safe_div(numerator: pl.Expr, denominator: pl.Expr) -> pl.Expr: + return ( + pl.when(denominator.is_not_null() & (denominator != 0)) + .then(numerator / denominator) + .otherwise(None) + ) + + +def _rolling_apply(inner: pl.Expr, op: str, n: int, extra: dict[str, float]) -> pl.Expr: + """对无 over 的内层序列应用窗口逻辑; 返回值同样不挂 over。""" + if op == "ts_mean": + return inner.rolling_mean(n, min_samples=n) + if op == "ts_std": + return inner.rolling_std(n, min_samples=n) + if op == "ts_sum": + return inner.rolling_sum(n, min_samples=n) + if op == "ts_max": + return inner.rolling_max(n, min_samples=n) + if op == "ts_min": + return inner.rolling_min(n, min_samples=n) + if op == "ts_delay": + return inner.shift(n) + if op == "ts_delta": + return inner - inner.shift(n) + if op == "ts_rank": + return inner.rolling_rank(n, min_samples=n) + if op == "ts_zscore": + mean = inner.rolling_mean(n, min_samples=n) + std = inner.rolling_std(n, min_samples=n) + return pl.when(std > 0).then((inner - mean) / std).otherwise(None) + if op == "ts_quantile": + return inner.rolling_quantile(extra.get("q", 0.5), window_size=n, min_samples=n) + if op == "decay_linear": + # 近端权重大: 权重 n, n-1, ..., 1, 总权 n(n+1)/2 + weighted = None + for i in range(n): + term = (n - i) * inner.shift(i) + weighted = term if weighted is None else weighted + term + assert weighted is not None + return _safe_div(weighted, pl.lit(float(n * (n + 1) / 2))) + raise AssertionError(op) + + +def _compile_node(node: dict) -> tuple[pl.Expr | None, bool, bool]: + """返回 (expr, needs_symbol_window, is_bool)。 + + needs_symbol_window=True 表示该子树含 ts 窗口逻辑但尚未挂 over; + 由非时序上下文的调用方挂 over("symbol"), 时序上下文继续向内传递。 + """ + kind = node["kind"] + if kind == "num": + return pl.lit(node["value"]), False, False + if kind == "col": + # 基准列/base 因子/虚拟因子统一以列引用; 虚拟因子由调用方物化 (运行期缺列 fail-closed) + return pl.col(node["value"]), False, False + if kind == "unary": + operand, needs_window, _ = _compile_node(node["children"][0]) + if operand is None: + return None, False, False + return -operand, needs_window, False + if kind == "bin": + op = node["value"] + left, left_window, _ = _compile_node(node["children"][0]) + right, right_window, _ = _compile_node(node["children"][1]) + if left is None or right is None: + return None, False, False + if left_window: + left = left.over("symbol") + if right_window: + right = right.over("symbol") + if op == "+": + return left + right, False, False + if op == "-": + return left - right, False, False + if op == "*": + return left * right, False, False + if op == "/": + return _safe_div(left, right), False, False + if op in _CMP_METHOD: + return getattr(left, _CMP_METHOD[op])(right), False, True + if op == "and": + return left & right, False, True + if op == "or": + return left | right, False, True + return None, False, False + if kind == "call": + return _compile_call(node) + return None, False, False + + +def _compile_call(node: dict) -> tuple[pl.Expr | None, bool, bool]: + name = node["value"] + children = node["children"] + constants: dict[str, float] = node.get("_constants", {}) + n_expr, _ = OPERATORS[name] + + if name in TS_OPERATORS: + inner, _, _ = _compile_node(children[0]) + if inner is None: + return None, False, False + if name in ("ts_corr", "ts_cov"): + second, _, _ = _compile_node(children[1]) + if second is None: + return None, False, False + n = int(constants.get("n", 0)) + expr = ( + pl.rolling_corr(inner, second, window_size=n) + if name == "ts_corr" + else pl.rolling_cov(inner, second, window_size=n) + ) + return expr, True, False + expr = _rolling_apply(inner, name, int(constants.get("n", 0)), constants) + return expr, True, False + + if name in CROSS_OPERATORS: + inner, inner_window, _ = _compile_node(children[0]) + if inner is None: + return None, False, False + if inner_window: + inner = inner.over("symbol") + if name == "rank": + count = inner.count().over("date") + return inner.rank(method="average").over("date") / count, False, False + if name == "zscore": + mean = inner.mean().over("date") + std = inner.std().over("date") + return pl.when(std > 0).then((inner - mean) / std).otherwise(None), False, False + k = constants.get("k", 3.0) + mean = inner.mean().over("date") + std = inner.std().over("date") + return inner.clip(mean - k * std, mean + k * std), False, False + + if name == "if_else": + cond, cond_window, _ = _compile_node(children[0]) + then_expr, then_window, _ = _compile_node(children[1]) + else_expr, else_window, _ = _compile_node(children[2]) + if cond is None or then_expr is None or else_expr is None: + return None, False, False + if cond_window: + cond = cond.over("symbol") + if then_window: + then_expr = then_expr.over("symbol") + if else_window: + else_expr = else_expr.over("symbol") + return pl.when(cond).then(then_expr).otherwise(else_expr), False, False + + args: list[pl.Expr | None] = [] + arg_windows: list[bool] = [] + for index in range(n_expr): + arg, arg_window, _ = _compile_node(children[index]) + args.append(arg) + arg_windows.append(arg_window) + if any(arg is None for arg in args): + return None, False, False + resolved: list[pl.Expr] = [] + for arg, arg_window in zip(args, arg_windows, strict=True): + resolved.append(arg.over("symbol") if arg_window else arg) + first = resolved[0] + if name == "log": + return pl.when(first > 0).then(first.log()).otherwise(None), False, False + if name == "abs": + return first.abs(), False, False + if name == "sign": + return first.sign(), False, False + if name == "sqrt": + return pl.when(first >= 0).then(first.sqrt()).otherwise(None), False, False + if name == "power": + return first.pow(constants.get("c", 1.0)), False, False + if name == "clamp": + return first.clip(constants.get("lo"), constants.get("hi")), False, False + if name == "min": + return pl.min_horizontal(*resolved), False, False + if name == "max": + return pl.max_horizontal(*resolved), False, False + return None, False, False + + +def compile_formula(text: str) -> CompiledFormula: + """编译公式文本; 永不抛异常, 失败以 errors 表达 (fail-closed)。""" + if not isinstance(text, str) or not text.strip(): + return CompiledFormula(ok=False, errors=[DslError("E014", "语法错误: 表达式为空")], formula_text=text) + + tokens, tokenize_error = _tokenize(text) + errors: list[DslError] = [tokenize_error] if tokenize_error else [] + if len(tokens) > MAX_TOKENS: + errors.append(DslError("E007", f"规模超限: token 数 {len(tokens)} > {MAX_TOKENS}")) + if errors: + return CompiledFormula(ok=False, errors=errors, formula_text=text) + + ast, parse_error = _Parser(tokens, text).parse() + if parse_error: + return CompiledFormula(ok=False, errors=[parse_error], formula_text=text) + + if _ast_depth(ast) > MAX_AST_DEPTH: + errors.append(DslError("E006", f"嵌套深度超限: AST 深度 {_ast_depth(ast)} > {MAX_AST_DEPTH}")) + + identifiers: set[str] = set() + _collect_identifiers(ast, identifiers) + if not identifiers: + errors.append(DslError("E016", "常量表达式: 公式必须引用至少一个数据列或因子")) + + for name in sorted(identifiers): + if name not in BASE_COLUMNS and get_factor(name) is None: + errors.append(DslError("E001", f"未知标识符: {name}", detail={"name": name})) + + constants_by_call: dict[int, dict] = {} + _semantic_walk(ast, errors, constants_by_call) + + dependencies: set[str] = set() + referenced_factors: set[str] = set() + warmup = 1 + cross_sectional = False + for name in identifiers: + if name in BASE_COLUMNS: + dependencies.add(name) + continue + spec = get_factor(name) + if spec is None: + continue + referenced_factors.add(name) + dependencies.update(factor_dependencies([name])) + warmup = max(warmup, spec.warmup_bars) + + for node_constants in constants_by_call.values(): + n_value = node_constants.get("n") + if n_value is not None and n_value == int(n_value) and int(n_value) > 0: + warmup = max(warmup, int(n_value) + 1) + + def _find_cross(node: dict) -> None: + nonlocal cross_sectional + if node["kind"] == "call" and node["value"] in CROSS_OPERATORS: + cross_sectional = True + for child in node["children"]: + _find_cross(child) + + _find_cross(ast) + + if errors: + return CompiledFormula( + ok=False, errors=errors, dependencies=frozenset(dependencies), + referenced_factors=frozenset(referenced_factors), + warmup_bars=warmup, cross_sectional=cross_sectional, formula_text=text, + ) + + # 挂常量表必须在任何 deepcopy 之前 (deepcopy 携带 _constants; 事后按 id() 重挂会失联) + def _attach(node: dict) -> None: + if node["kind"] == "call": + node["_constants"] = constants_by_call.get(id(node), {}) + for child in node["children"]: + _attach(child) + + _attach(ast) + + # 阶段一: 校验并拒绝"截面算子嵌在时序窗口内" (无法单层 over 表达) + def _contains_cross(node: dict) -> bool: + if node["kind"] == "call" and node["value"] in CROSS_OPERATORS: + return True + return any(_contains_cross(child) for child in node["children"]) + + def _reject_cross_in_ts(node: dict) -> None: + if node["kind"] == "call" and node["value"] in TS_OPERATORS: + for child in node["children"]: + if _contains_cross(child): + errors.append(DslError( + "E009", + f"截面算子不能嵌在时序窗口内: {node['value']}(...) 的参数含 rank/zscore/winsorize", + offset=node["offset"], + )) + return + for child in node["children"]: + _reject_cross_in_ts(child) + + _reject_cross_in_ts(ast) + if errors: + return CompiledFormula( + ok=False, errors=errors, dependencies=frozenset(dependencies), + referenced_factors=frozenset(referenced_factors), + warmup_bars=warmup, cross_sectional=cross_sectional, formula_text=text, + ) + + # 阶段二: 提取截面算子的含窗口子树为临时列 (Polars 嵌套窗口会静默全 null) + # worklist 逐层下钻; temps 后进先出反转即依赖顺序 (深层先算)。 + def _needs_symbol_window(node: dict) -> bool: + kind = node["kind"] + if kind in ("num", "col"): + return False + if kind == "unary": + return _needs_symbol_window(node["children"][0]) + if node["kind"] == "call" and node["value"] in TS_OPERATORS: + return True + return any(_needs_symbol_window(child) for child in node["children"]) + + def _has_any_over(node: dict) -> bool: + # 含时序窗口 或 含截面算子(编译后自带 over("date")) 的子树都不能直接进截面上下文 + return _needs_symbol_window(node) or _contains_cross(node) + + temp_roots: list[dict] = [] + pending: list[dict] = [ast] + while pending: + current = pending.pop(0) + if current.get("kind") == "call" and current.get("value") in CROSS_OPERATORS: + operand = current["children"][0] + if _has_any_over(operand): + alias = f"__tsfx_{len(temp_roots)}__" + current["children"][0] = {"kind": "col", "value": alias, "children": [], "offset": operand["offset"]} + temp_roots.append({"alias": alias, "root": copy.deepcopy(operand)}) + pending.append(temp_roots[-1]["root"]) + continue # 操作数已替换为临时列, 不再下钻原子树 + pending.extend(current.get("children", [])) + + # 阶段三: 编译最终表达式与临时列表达式 (按依赖顺序: 深层在前) + # _constants 已在 deepcopy 前挂载并被复制携带, 不得按 id() 重挂 (复制后 id 失联) + temp_exprs: list[pl.Expr] = [] + for item in reversed(temp_roots): + root = copy.deepcopy(item["root"]) + expr, needs_window, _ = _compile_node(root) + if expr is None: + errors.append(DslError("E009", f"无法编译临时列: {item['alias']}")) + continue + if needs_window: + expr = expr.over("symbol") + temp_exprs.append(expr.alias(item["alias"])) + + final_ast = copy.deepcopy(ast) + compiled, needs_window, is_bool = _compile_node(final_ast) + if compiled is None or errors: + return CompiledFormula( + ok=False, + errors=errors or [DslError("E009", "产出类型非法: 无法编译为数值表达式")], + dependencies=frozenset(dependencies), + referenced_factors=frozenset(referenced_factors), + warmup_bars=warmup, cross_sectional=cross_sectional, formula_text=text, + ) + if needs_window: + compiled = compiled.over("symbol") + if is_bool: + compiled = compiled.cast(pl.Float64) + + # 运行期帧变换: 检查全部引用列 (基准依赖 + 引用因子) 存在, 否则 None (E013 fail-closed) + required_columns = set(dependencies) | set(referenced_factors) + staged_exprs = temp_exprs # 依赖顺序已排 + + def frame_transform(frame: pl.DataFrame) -> pl.DataFrame | None: + if not required_columns.issubset(set(frame.columns)): + return None + result = frame + if staged_exprs: + result = result.with_columns(staged_exprs) + return result.with_columns(compiled.alias(FACTOR_COLUMN)) + + return CompiledFormula( + ok=True, + errors=[], + frame_transform=frame_transform, + dependencies=frozenset(dependencies), + referenced_factors=frozenset(referenced_factors), + warmup_bars=warmup, + cross_sectional=cross_sectional, + formula_text=text, + ) + + +@lru_cache(maxsize=256) +def compile_formula_cached(text: str) -> CompiledFormula: + """带 LRU 缓存的编译入口 (公式文本 → 编译产物, 设计文档 §3.3)。 + + CompiledFormula 为不可变值对象 (frame_transform 闭包只读), 缓存共享安全。 + """ + return compile_formula(text) diff --git a/backend/app/factors/registry.py b/backend/app/factors/registry.py new file mode 100644 index 0000000..468da57 --- /dev/null +++ b/backend/app/factors/registry.py @@ -0,0 +1,383 @@ +"""因子注册表 (L-REG) — 因子元数据的单一权威来源。 + +P1 收口范围: 目录元数据 (id/label/group/公式)、虚拟因子依赖声明、评分预热窗口。 +三处历史清单在此合一: + - backtest/factor.py FACTOR_COLUMNS (由 factor_columns_view() 生成兼容别名) + - strategy/scoring.py VIRTUAL_SCORING_DEPENDENCIES (由 virtual_dependencies() 生成) + - strategy/scoring.py _ROLLING_SCORING_WARMUP (由 scoring_warmups() 生成) + +P1 边界 (诚实声明): + - scoring_value_expr 的表达式分发仍留在 scoring.py, 注册表不含计算逻辑; + 复合/自定义因子 (composite/custom) 与 DSL 在 P2/P3 接入后再收口。 + - unit 字段 P1 统一 "none": 单位口径涉及金融数据契约 (CONTRIBUTING §3), + 未经逐因子核对禁止猜测填充; 前端 P1 也不按 unit 格式化。 +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal + +Kind = Literal["base", "virtual", "composite", "custom"] +Direction = Literal["high", "low", "none"] +Unit = Literal["ratio", "pct", "score", "count", "days", "currency", "none"] +PitSource = Literal["financial_announce", "share_capital_announce", "none"] +Stability = Literal["stable", "experimental", "deprecated"] + +_ALL_ASSETS = frozenset({"stock", "etf"}) +_STOCK_ONLY = frozenset({"stock"}) + + +@dataclass(frozen=True) +class FactorSpec: + id: str + label: str + group: str + formula_text: str + kind: Kind = "base" + version: int = 1 + # base: 空集合 = 已物化列自身; virtual: 展开到 enriched base 列 + dependencies: frozenset[str] = field(default_factory=frozenset) + direction: Direction = "none" # P1 不预填: 方向以最近检验 IC 符号为准 (见平台方案 §3.6) + unit: Unit = "none" + warmup_bars: int = 1 + pit: bool = False + pit_source: PitSource = "none" + asset_types: frozenset[str] = _ALL_ASSETS + incremental_safe: bool = True + scale_free: bool = True + null_policy: Literal["keep", "drop_row"] = "keep" + stability: Stability = "stable" + tags: tuple[str, ...] = () + # composite 专用: ((成员 id, 权重), ...); 其余类型为空 + components: tuple[tuple[str, float], ...] = () + + def column_view(self) -> dict: + """历史 FACTOR_COLUMNS 条目视图 (键与顺序兼容)。""" + return {"id": self.id, "label": self.label, "group": self.group, "desc": self.formula_text} + + +def _base(fid: str, label: str, group: str, desc: str, **overrides) -> FactorSpec: + return FactorSpec(id=fid, label=label, group=group, formula_text=desc, kind="base", **overrides) + + +def _virtual(fid: str, label: str, group: str, desc: str, deps: frozenset[str], **overrides) -> FactorSpec: + return FactorSpec( + id=fid, label=label, group=group, formula_text=desc, + kind="virtual", dependencies=deps, **overrides, + ) + + +def _financial(fid: str, label: str, desc: str) -> FactorSpec: + return FactorSpec( + id=fid, label=label, group="财务", formula_text=desc, + kind="base", pit=True, pit_source="financial_announce", asset_types=_STOCK_ONLY, + ) + + +# 顺序即历史 FACTOR_COLUMNS 顺序 (mining_schedule 取前 48 个, 不得重排)。 +_CATALOG: tuple[FactorSpec, ...] = ( + # --- 动量 --- + _base("momentum_5d", "5日动量", "动量", "5个交易日累计收益率"), + _base("momentum_10d", "10日动量", "动量", "10个交易日累计收益率"), + _base("momentum_20d", "20日动量", "动量", "20个交易日累计收益率"), + _base("momentum_30d", "30日动量", "动量", "30个交易日累计收益率"), + _base("momentum_60d", "60日动量", "动量", "60个交易日累计收益率"), + _base("change_pct", "日涨跌幅", "动量", "当日收盘相对前收盘的收益率"), + # --- 均线偏离 (虚拟) --- + *( + _virtual( + f"ma{period}_bias", f"MA{period}乖离", "均线偏离", f"收盘价 / MA{period} - 1", + deps=frozenset({"close", f"ma{period}"}), + ) + for period in (5, 10, 20, 30, 60) + ), + *( + _virtual( + f"ema{period}_bias", f"EMA{period}乖离", "均线偏离", f"收盘价 / EMA{period} - 1", + deps=frozenset({"close", f"ema{period}"}), + ) + for period in (5, 10, 20, 30, 60) + ), + # --- 超买超卖 --- + _base("rsi_6", "RSI(6)", "超买超卖", "6日相对强弱指标"), + _base("rsi_14", "RSI(14)", "超买超卖", "14日相对强弱指标"), + _base("rsi_24", "RSI(24)", "超买超卖", "24日相对强弱指标"), + # --- 趋势 --- + _base( + "macd_hist", "MACD柱(原值)", "趋势", + "兼容历史研究; 跨股票比较建议优先使用MACD柱强度", + scale_free=False, + ), + _virtual("macd_dif_pct", "MACD DIF强度", "趋势", "MACD DIF / 收盘价", deps=frozenset({"close", "macd_dif"})), + _virtual("macd_dea_pct", "MACD DEA强度", "趋势", "MACD DEA / 收盘价", deps=frozenset({"close", "macd_dea"})), + _virtual("macd_hist_pct", "MACD柱强度", "趋势", "MACD柱 / 收盘价, 消除股价尺度影响", deps=frozenset({"close", "macd_hist"})), + _base("kdj_k", "KDJ-K", "趋势", "KDJ指标K值"), + _base("kdj_d", "KDJ-D", "趋势", "KDJ指标D值"), + _base("kdj_j", "KDJ-J", "趋势", "KDJ指标J值"), + _virtual( + "boll_position", "布林位置", "趋势", "收盘价在布林带下轨到上轨之间的位置", + deps=frozenset({"close", "boll_upper", "boll_lower"}), + ), + # --- 波动率 --- + _base("annual_vol_20d", "20日波动率", "波动率", "20日收益率年化标准差"), + _base("atr_14", "ATR(14)原值", "波动率", "兼容历史研究; 跨股票比较建议优先使用ATR相对波动", scale_free=False), + _virtual("atr_pct", "ATR相对波动", "波动率", "ATR(14) / 收盘价", deps=frozenset({"close", "atr_14"})), + _base("amplitude", "日振幅", "波动率", "当日高低价差 / 前收盘价"), + _virtual( + "boll_width", "布林带宽", "波动率", "布林带上下轨宽度 / MA20", + deps=frozenset({"ma20", "boll_upper", "boll_lower"}), + ), + # --- 量价 --- + _base("vol_ratio_5d", "5日量比", "量价", "当日成交量 / 前5日平均成交量"), + _virtual( + "vol_ratio_10d", "10日量比", "量价", "当日成交量 / 前10日平均成交量", + deps=frozenset({"volume"}), warmup_bars=11, + ), + _virtual( + "vol_trend_5_10", "成交量趋势", "量价", "5日平均成交量 / 10日平均成交量 - 1", + deps=frozenset({"vol_ma5", "vol_ma10"}), + ), + _base("turnover_rate", "换手率", "量价", "使用历史时点流通股本计算的当日换手率"), + _virtual( + "turnover_ratio_5d", "换手率放大", "量价", "当日换手率 / 前5日平均换手率 - 1", + deps=frozenset({"turnover_rate"}), warmup_bars=6, + ), + _virtual( + "log_amount", "成交额对数", "量价", "ln(成交额 + 1), 降低极端规模影响", + deps=frozenset({"amount"}), scale_free=False, + ), + _virtual( + "amount_ratio_5d", "成交额放大", "量价", "当日成交额 / 前5日平均成交额 - 1", + deps=frozenset({"amount"}), warmup_bars=6, + ), + # --- 价格位置 --- + _virtual("gap_return", "开盘跳空", "价格位置", "开盘价 / 前收盘价 - 1", deps=frozenset({"open", "prev_close"})), + _virtual("intraday_return", "日内收益", "价格位置", "收盘价 / 开盘价 - 1", deps=frozenset({"open", "close"})), + _virtual( + "close_position", "收盘位置", "价格位置", "收盘价在当日最低价到最高价之间的位置", + deps=frozenset({"high", "low", "close"}), + ), + _virtual( + "distance_to_high_60d", "距60日高点", "价格位置", "收盘价 / 60日最高收盘价 - 1", + deps=frozenset({"close", "high_60d"}), + ), + _virtual( + "distance_from_low_60d", "距60日低点", "价格位置", "收盘价 / 60日最低收盘价 - 1", + deps=frozenset({"close", "low_60d"}), + ), + _virtual( + "vwap_bias", "VWAP乖离", "价格位置", "收盘价 / 当日成交均价 - 1, 成交均价 = 成交额 / (成交量x100)", + deps=frozenset({"close", "volume", "amount"}), + ), + # --- 收益形态 (虚拟, 滚动窗口) --- + _virtual( + "max_ret_20d", "20日最大单日涨幅", "收益形态", "近20个交易日单日涨幅最大值(彩票效应, 高值代表博彩型特征强)", + deps=frozenset({"close"}), warmup_bars=21, + ), + _virtual( + "ret_skew_20d", "20日收益偏度", "收益形态", "近20个交易日日收益偏度, 高值代表右偏(偶发大涨)", + deps=frozenset({"close"}), warmup_bars=21, + ), + _virtual( + "up_days_20d", "20日上涨天数", "收益形态", "近20个交易日中上涨天数(0~20)", + deps=frozenset({"close"}), warmup_bars=21, + ), + # --- 流动性 (虚拟) --- + _virtual( + "amihud_20d", "20日Amihud非流动性", "流动性", "近20日平均 |日涨跌幅| / 成交额(亿元), 高值代表流动性差", + deps=frozenset({"close", "amount"}), warmup_bars=21, + ), + _virtual( + "turnover_z_60d", "换手率60日z分", "流动性", "(当日换手率 - 前60日均值) / 前60日标准差, 衡量换手异动", + deps=frozenset({"turnover_rate"}), warmup_bars=61, + ), + # --- 量价 (续) --- + _virtual( + "vol_price_corr_20d", "20日量价相关", "量价", "近20个交易日日涨跌幅与成交量的相关系数, 高值代表量价同向", + deps=frozenset({"close", "volume"}), warmup_bars=21, + ), + _virtual( + "vol_trend_5_60", "量能趋势(5/60)", "量价", "5日平均成交量 / 60日平均成交量 - 1", + deps=frozenset({"volume"}), warmup_bars=60, + ), + # --- 涨停基因 (虚拟) --- + _virtual( + "limit_up_count_20d", "涨停基因(20日)", "涨停基因", "近20个交易日涨停次数", + deps=frozenset({"consecutive_limit_ups"}), warmup_bars=21, + ), + _virtual( + "limit_up_count_60d", "涨停基因(60日)", "涨停基因", "近60个交易日涨停次数", + deps=frozenset({"consecutive_limit_ups"}), warmup_bars=61, + ), + # --- 财务 (点时, 仅股票) --- + _financial("pb_latest", "市净率(最新公告)", "收盘价 / 最新已公告每股净资产; 无财务数据或公告前为空"), + _financial("roe_latest", "ROE(最新公告)", "最新已公告净资产收益率(%); 无财务数据或公告前为空"), + _financial("gross_margin_latest", "毛利率(最新公告)", "最新已公告销售毛利率(%)"), + _financial("net_margin_latest", "净利率(最新公告)", "最新已公告销售净利率(%)"), + _financial("revenue_yoy_latest", "营收增速(最新公告)", "最新已公告营业收入同比(%)"), + _financial("net_income_yoy_latest", "净利增速(最新公告)", "最新已公告归母净利润同比(%)"), + _financial("debt_ratio_latest", "资产负债率(最新公告)", "最新已公告资产负债率(%)"), + # --- 扩充批次 (2026-09-05): 规模/收益分解/长窗口/下行风险/量能潮/换手水平 --- + _virtual( + "log_float_mv", "流通市值对数", "规模", + "ln(收盘价 x 当日成交量 / 换手率), 由换手率反推流通股本, 高值代表大盘", + deps=frozenset({"close", "volume", "turnover_rate"}), scale_free=False, + ), + _virtual( + "momentum_120d", "120日动量", "动量", + "120个交易日累计收益率 (中期动量, 与短窗口互补)", + deps=frozenset({"close"}), warmup_bars=121, + ), + _virtual( + "mom_accel_20_60", "动量加速度", "动量", + "20日动量 - 60日动量, 衡量近期动量相对中期是否增强", + deps=frozenset({"momentum_20d", "momentum_60d"}), + ), + _virtual( + "rsi_14_delta_5d", "RSI五日变化", "超买超卖", + "RSI(14) - 5日前的RSI(14), 衡量强弱指标的边际变化", + deps=frozenset({"rsi_14"}), warmup_bars=6, + ), + _virtual( + "overnight_ret_20d", "20日隔夜收益", "收益形态", + "近20日累计隔夜收益(开盘价/前收盘-1求和), A股隔夜与日内收益的定价机制不同", + deps=frozenset({"open", "prev_close"}), warmup_bars=21, + ), + _virtual( + "intraday_ret_20d", "20日日内收益", "收益形态", + "近20日累计日内收益(收盘价/开盘价-1求和), 与隔夜收益构成收益分解", + deps=frozenset({"open", "close"}), warmup_bars=21, + ), + _virtual( + "downside_vol_20d", "20日下行波动", "波动率", + "sqrt(近20日 min(日收益,0)^2 均值), 只度量下跌侧风险", + deps=frozenset({"close"}), warmup_bars=21, + ), + _virtual( + "vol_regime_5_60", "波动率状态(5/60)", "波动率", + "5日收益标准差 / 60日收益标准差, 高值代表波动骤然放大", + deps=frozenset({"close"}), warmup_bars=61, + ), + _virtual( + "amplitude_trend_20_60", "振幅趋势(20/60)", "波动率", + "20日平均振幅 / 60日平均振幅 - 1", + deps=frozenset({"amplitude"}), warmup_bars=61, + ), + _virtual( + "obv_trend_20d", "20日量能潮", "量价", + "近20日 sign(日收益)x成交量 之和 / (20日均量x20), 有界[-1,1], 净买入方向的一致性", + deps=frozenset({"close", "volume"}), warmup_bars=21, + ), + _virtual( + "amount_mean_20d", "20日均成交额(亿)", "量价", + "近20日平均成交额(亿元), 规模/流动性水平量", + deps=frozenset({"amount"}), warmup_bars=21, scale_free=False, + ), + _virtual( + "turnover_mean_20d", "20日均换手", "流动性", + "近20日平均换手率, A股经典低换手溢价因子", + deps=frozenset({"turnover_rate"}), warmup_bars=21, + ), + _virtual( + "turnover_std_20d", "20日换手波动", "流动性", + "近20日换手率标准差 / 均值 (变异系数), 衡量交易活跃的稳定性", + deps=frozenset({"turnover_rate"}), warmup_bars=21, + ), + _virtual( + "position_240d", "一年价格位置", "价格位置", + "收盘价在近240个交易日最低价到最高价之间的位置 (0~1)", + deps=frozenset({"close"}), warmup_bars=241, + ), + _virtual( + "distance_to_high_240d", "距一年高点", "价格位置", + "收盘价 / 240日最高收盘价 - 1, 接近0代表贴近一年新高", + deps=frozenset({"close"}), warmup_bars=241, + ), + _virtual( + "kdj_kd_diff", "KDJ K-D差", "趋势", + "KDJ K值 - D值, 正值代表快线在慢线上方", + deps=frozenset({"kdj_k", "kdj_d"}), + ), +) + +_REGISTRY: dict[str, FactorSpec] = {} + + +def register_factor(spec: FactorSpec) -> None: + """注册因子; 重复 id 且版本未增时拒绝 (fail-closed)。""" + existing = _REGISTRY.get(spec.id) + if existing is not None and existing.version >= spec.version: + raise ValueError(f"factor id 已注册且版本未提升: {spec.id}") + _REGISTRY[spec.id] = spec + + +for _spec in _CATALOG: + register_factor(_spec) + + +def get_factor(fid: str) -> FactorSpec | None: + return _REGISTRY.get(fid) + + +def unregister_factor(fid: str) -> FactorSpec | None: + """注销动态注册的因子 (内置目录因子不可注销, fail-closed)。""" + if any(spec.id == fid for spec in _CATALOG): + raise ValueError(f"内置因子不可注销: {fid}") + return _REGISTRY.pop(fid, None) + + +def _ordered_specs() -> list[FactorSpec]: + """内置目录顺序在前, 动态注册因子 (custom/composite) 按注册顺序追加。""" + ordered: list[FactorSpec] = list(_CATALOG) + known = {spec.id for spec in _CATALOG} + ordered.extend(spec for fid, spec in _REGISTRY.items() if fid not in known) + return ordered + + +def all_factors( + asset_type: str | None = None, + stable_only: bool = False, +) -> list[FactorSpec]: + """按目录顺序返回因子; asset_type 过滤适用资产, stable_only 过滤实验/废弃因子。""" + return [ + spec for spec in _ordered_specs() + if (asset_type is None or asset_type in spec.asset_types) + and (not stable_only or spec.stability == "stable") + ] + + +def factor_dependencies(fids) -> frozenset[str]: + """递归展开依赖到 enriched base 列; 未知 id 原样保留 (与 scoring_dependencies 历史语义一致)。""" + resolved: set[str] = set() + for fid in fids: + spec = _REGISTRY.get(str(fid)) + if spec is None: + resolved.add(str(fid)) + elif spec.dependencies: + resolved.update(spec.dependencies) + else: + resolved.add(spec.id) + return frozenset(resolved) + + +def factor_columns_view() -> list[dict]: + """历史 FACTOR_COLUMNS 兼容视图 (顺序、键一致; 动态注册因子追加在末尾)。""" + return [spec.column_view() for spec in _ordered_specs()] + + +def virtual_dependencies() -> dict[str, frozenset[str]]: + """历史 VIRTUAL_SCORING_DEPENDENCIES 兼容视图。""" + return { + spec.id: spec.dependencies + for spec in _CATALOG + if spec.kind == "virtual" and spec.dependencies + } + + +def scoring_warmups() -> dict[str, int]: + """历史 _ROLLING_SCORING_WARMUP 兼容视图 (仅滚动窗口虚拟因子)。""" + return { + spec.id: spec.warmup_bars + for spec in _CATALOG + if spec.kind == "virtual" and spec.warmup_bars > 1 + } diff --git a/backend/app/factors/store.py b/backend/app/factors/store.py new file mode 100644 index 0000000..1ac9566 --- /dev/null +++ b/backend/app/factors/store.py @@ -0,0 +1,190 @@ +"""自定义/复合因子存储 (P3) — data/user_data/custom_factors/*.json。 + +镜像 custom_signals 的持久化写法; 单文件损坏只禁用该因子并告警, 不影响启动 +(对齐 CONTRIBUTING 第 4 节插件隔离要求)。生命周期状态: draft → active → +watch → retired (P4 状态机, 存储字段就绪, 迁移逻辑见巡检设计)。 +""" +from __future__ import annotations + +import json +import logging +import re +from datetime import datetime +from pathlib import Path + +from app.factors.dsl import compile_formula +from app.factors.registry import FactorSpec, factor_dependencies, get_factor, register_factor + +logger = logging.getLogger(__name__) + +CUSTOM_ID_PATTERN = re.compile(r"^uf_[a-z0-9_]{1,40}$") +COMPOSITE_ID_PATTERN = re.compile(r"^cf_[a-z0-9_]{1,40}$") +MAX_COMPOSITE_MEMBERS = 8 +STATUSES = frozenset({"draft", "active", "watch", "retired"}) + + +def _dir(data_dir: Path) -> Path: + directory = data_dir / "user_data" / "custom_factors" + directory.mkdir(parents=True, exist_ok=True) + return directory + + +def _path(data_dir: Path, factor_id: str) -> Path: + return _dir(data_dir) / f"{factor_id}.json" + + +def load_all(data_dir: Path) -> list[dict]: + """读取全部自定义/复合因子定义; 损坏文件跳过。""" + out: list[dict] = [] + for file in sorted(_dir(data_dir).glob("*.json")): + try: + out.append(json.loads(file.read_text(encoding="utf-8"))) + except Exception as exc: + logger.warning("custom factor load failed %s: %s", file.name, exc) + return out + + +def save_one(data_dir: Path, definition: dict) -> None: + target = _path(data_dir, str(definition["id"])) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(definition, ensure_ascii=False, indent=2), encoding="utf-8") + + +def delete_one(data_dir: Path, factor_id: str) -> bool: + target = _path(data_dir, factor_id) + if target.exists(): + target.unlink() + return True + return False + + +def _now() -> str: + return datetime.now().isoformat(timespec="seconds") + + +def to_spec(definition: dict) -> FactorSpec: + """定义 → FactorSpec; 校验失败抛 ValueError (调用方 fail-closed)。 + + custom: 依赖/预热由 DSL 编译推导 (编译失败即拒绝注册)。 + composite: 依赖 = 成员递归展开; 预热 = 成员最大值; 循环引用拒绝。 + """ + kind = str(definition.get("kind", "custom")) + factor_id = str(definition.get("id", "")) + label = str(definition.get("label", "")).strip() + if not label: + raise ValueError("label 不能为空") + pattern = COMPOSITE_ID_PATTERN if kind == "composite" else CUSTOM_ID_PATTERN + if not pattern.match(factor_id): + raise ValueError(f"id 必须匹配 {pattern.pattern}") + status = str(definition.get("status", "draft")) + if status not in STATUSES: + raise ValueError(f"status 必须是 {sorted(STATUSES)} 之一") + + if kind == "custom": + formula = str(definition.get("formula", "")) + compiled = compile_formula(formula) + if not compiled.ok: + first = compiled.errors[0] + raise ValueError(f"公式无效 [{first.code}]: {first.message}") + return FactorSpec( + id=factor_id, + label=label, + group=str(definition.get("group", "自定义")), + formula_text=formula, + kind="custom", + version=int(definition.get("version", 1)), + dependencies=frozenset(compiled.dependencies), + warmup_bars=compiled.warmup_bars, + direction=str(definition.get("direction", "none")), # type: ignore[arg-type] + stability="stable" if status == "active" else "experimental", + ) + + if kind != "composite": + raise ValueError(f"未知 kind: {kind}") + members_raw = definition.get("members") + if not isinstance(members_raw, dict) or not (2 <= len(members_raw) <= MAX_COMPOSITE_MEMBERS): + raise ValueError(f"composite 成员必须是 {2}~{MAX_COMPOSITE_MEMBERS} 个") + from app.factors.dsl import BASE_COLUMNS + + components: list[tuple[str, float]] = [] + for member_id, weight in members_raw.items(): + member_id = str(member_id) + if member_id == factor_id: + raise ValueError("composite 不能引用自身") + try: + weight = float(weight) + except (TypeError, ValueError) as exc: + raise ValueError(f"成员 {member_id} 权重必须是数字") from exc + if not weight: + raise ValueError(f"成员 {member_id} 权重不能为 0") + # 成员 = 注册表因子 或 enriched 基准列 (已物化, 可直接参与组合) + if get_factor(member_id) is None and member_id not in BASE_COLUMNS: + raise ValueError(f"未知成员因子: {member_id}") + components.append((member_id, weight)) + # 环检测沿 components 链走 (依赖已展开, 看不到链路成员) + seen = {factor_id} + frontier = [member_id for member_id, _ in components] + while frontier: + current = frontier.pop() + if current in seen: + raise ValueError("composite 成员存在循环引用") + seen.add(current) + current_spec = get_factor(current) + if current_spec is not None and current_spec.kind == "composite": + frontier.extend(member_id for member_id, _ in current_spec.components) + dependencies = factor_dependencies([member_id for member_id, _ in components]) + warmup = max( + ((get_factor(member_id).warmup_bars if get_factor(member_id) else 1) for member_id, _ in components), + default=1, + ) + formula_text = " + ".join( + f"{weight:g}*zscore({member_id})" for member_id, weight in components + ) + return FactorSpec( + id=factor_id, + label=label, + group=str(definition.get("group", "组合")), + formula_text=formula_text, + kind="composite", + version=int(definition.get("version", 1)), + dependencies=dependencies, + warmup_bars=warmup, + direction=str(definition.get("direction", "none")), # type: ignore[arg-type] + components=tuple(components), + stability="stable" if status == "active" else "experimental", + ) + + +def register_definition(definition: dict) -> FactorSpec: + """定义 → spec → 注册 (重复 id 版本未升时由注册表拒绝)。""" + spec = to_spec(definition) + register_factor(spec) + return spec + + +def load_into_registry(data_dir: Path) -> list[str]: + """启动期把存储中的因子注册进注册表; 单个失败只跳过并告警。 + + 多轮加载: composite 成员可能引用尚未加载的 custom/其他 composite (文件按 + 字母序加载, cf_* 先于 uf_*), 失败的 composite 延后重试, 覆盖链式引用; + 重试用尽仍失败的只告警不阻塞启动。 + """ + loaded: list[str] = [] + pending = list(load_all(data_dir)) + for round_index in range(3): + deferred: list[dict] = [] + for definition in pending: + try: + register_definition(definition) + loaded.append(str(definition["id"])) + except ValueError as exc: + if round_index < 2 and str(definition.get("kind")) == "composite": + deferred.append(definition) + else: + logger.warning("custom factor 注册失败 %s: %s", definition.get("id"), exc) + except Exception as exc: + logger.warning("custom factor 注册失败 %s: %s", definition.get("id"), exc) + if not deferred: + break + pending = deferred + return loaded diff --git a/backend/app/indicators/pipeline.py b/backend/app/indicators/pipeline.py index 41cabb5..41312a0 100644 --- a/backend/app/indicators/pipeline.py +++ b/backend/app/indicators/pipeline.py @@ -666,9 +666,12 @@ def compute_signals(df: pl.DataFrame, needed: set[str] | None = None) -> pl.Data if want: df = df.with_columns([expressions[name] for name in SIGNAL_DEPENDENCIES if name in want]) - # 自定义信号(用户配置的字段+运算符+值组合,编译为布尔列) + # 自定义信号(用户配置的字段+运算符+值组合,编译为布尔列)。 + # 条件引用的注册表因子列先复用评分物化管线补算 (虚拟/自定义/复合均可)。 from app.strategy import custom_signals - df = custom_signals.inject(df, _get_custom_signal_exprs(), needed=needed) + exprs = _get_custom_signal_exprs() + df = custom_signals.materialize_factor_columns(df, exprs, needed=needed) + df = custom_signals.inject(df, exprs, needed=needed) return df @@ -1053,13 +1056,28 @@ def _select_storage_cols(df: pl.DataFrame) -> pl.DataFrame: DEVIATION_WINDOWS: tuple[int, ...] = (3, 10, 30) -# 各交易所基准指数 (偏离值规则的「对应指数」近似): 优先分类指数, 缺失时回退 +# 各板块基准指数 (偏离值规则的「对应指数」, 按交易所官方口径): 优先首选, 缺失时回退 +# - 沪主板: 上证A指 → 上证指数 (两者差异可忽略) +# - 科创板: 科创50 (上交所《交易规则》2026修订 6.12 指定基准) → 上证A指 +# - 深主板: 深证A指 → 深证成指 (深交所投教口径) +# - 创业板: 创业板综合指数 → 深证A指 (深交所投教口径) +# - 北交所: 北证50 → 上证指数 (北交所《交易规则》5.4.4) _BENCHMARK_PREFERENCE: dict[str, list[str]] = { - "SH": ["000002.SH", "000001.SH"], # 上证A指 → 上证指数 - "SZ": ["399107.SZ", "399001.SZ"], # 深证A指 → 深证成指 - "BJ": ["899050.BJ", "000001.SH"], # 北证50 → 上证指数 + "SH": ["000002.SH", "000001.SH"], + "STAR": ["000688.SH", "000002.SH"], + "SZ": ["399107.SZ", "399001.SZ"], + "GEM": ["399102.SZ", "399107.SZ"], + "BJ": ["899050.BJ", "000001.SH"], } +# 偏离值计算需要的全部基准指数 (quote_service 并入实时显式拉取, 不依赖监控规则) +BENCHMARK_INDEX_SYMBOLS: frozenset[str] = frozenset( + sym for cands in _BENCHMARK_PREFERENCE.values() for sym in cands +) + +# 全部板块基准键 (SH/STAR/SZ/GEM/BJ) +BENCH_KEYS: tuple[str, ...] = tuple(_BENCHMARK_PREFERENCE) + _benchmark_cache: dict[str, tuple[float, pl.DataFrame | None]] = {} _BENCHMARK_CACHE_TTL = 600.0 @@ -1067,7 +1085,8 @@ _BENCHMARK_CACHE_TTL = 600.0 def load_benchmark_momentum(data_dir: Path) -> pl.DataFrame | None: """读取指数日K, 计算各基准指数的滚动 N 日涨跌幅。 - 返回长表: date, bench_exchange, bench_close, bench_mom3d, bench_mom10d, bench_mom30d。 + 返回长表: date, bench_key, bench_close, bench_mom3d, bench_mom10d, bench_mom30d。 + bench_key 为板块基准键 (SH/STAR/SZ/GEM/BJ, 见 _BENCHMARK_PREFERENCE)。 bench_close 供盘中路径外推今日基准动量 (benchmark_momentum_today)。 无可用指数数据时返回 None (偏离列置 null, 不阻塞主流程)。 进程内按 data_dir 缓存 (TTL 10 分钟)。 @@ -1085,11 +1104,11 @@ def load_benchmark_momentum(data_dir: Path) -> pl.DataFrame | None: index_glob = str(Path(data_dir) / "kline_index_daily" / "**" / "*.parquet") wanted: list[str] = [] bench_of: dict[str, str] = {} - for exchange, candidates in _BENCHMARK_PREFERENCE.items(): + for bench_key, candidates in _BENCHMARK_PREFERENCE.items(): for sym in candidates: if sym not in bench_of: wanted.append(sym) - bench_of[sym] = exchange + bench_of[sym] = bench_key lf = scan_daily_parquet( index_glob, cast_options=pl.ScanCastOptions(integer_cast="allow-float") ) @@ -1102,15 +1121,15 @@ def load_benchmark_momentum(data_dir: Path) -> pl.DataFrame | None: if not df_idx.is_empty(): available = set(df_idx["symbol"].to_list()) picked = [s for s in wanted if s in available] - # 每个交易所取优先级最高的可用基准; 全缺时回退到任一可用基准。 - # 同一基准可服务多个交易所 (如北证50 缺失时北交所回退上证指数)。 + # 每个板块取优先级最高的可用基准; 全缺时回退到任一可用基准。 + # 同一基准可服务多个板块 (如科创50 缺失时科创板回退上证A指)。 pairs: list[tuple[str, str]] = [] - for exchange, candidates in _BENCHMARK_PREFERENCE.items(): + for bench_key, candidates in _BENCHMARK_PREFERENCE.items(): hit = next((s for s in candidates if s in available), None) if hit is None and picked: hit = picked[0] if hit is not None: - pairs.append((hit, exchange)) + pairs.append((hit, bench_key)) df_bench = df_idx.filter(pl.col("symbol").is_in([p[0] for p in pairs])) if not df_bench.is_empty(): df_bench = df_bench.with_columns( @@ -1119,16 +1138,16 @@ def load_benchmark_momentum(data_dir: Path) -> pl.DataFrame | None: (pl.col("close") / pl.col("close").shift(n).over("symbol") - 1).alias(f"_bm{n}") for n in DEVIATION_WINDOWS ]).rename({f"_bm{n}": f"bench_mom{n}d" for n in DEVIATION_WINDOWS}) - exchange_map = pl.DataFrame({ + key_map = pl.DataFrame({ "symbol": [p[0] for p in pairs], - "bench_exchange": [p[1] for p in pairs], + "bench_key": [p[1] for p in pairs], }) frame = ( - df_bench.join(exchange_map, on="symbol", how="inner") - .select(["date", "bench_exchange", "close", + df_bench.join(key_map, on="symbol", how="inner") + .select(["date", "bench_key", "close", *[f"bench_mom{n}d" for n in DEVIATION_WINDOWS]]) .rename({"close": "bench_close"}) - .unique(subset=["date", "bench_exchange"]) + .unique(subset=["date", "bench_key"]) ) except Exception as exc: # noqa: BLE001 logger.warning("基准指数偏离数据加载失败: %s", exc) @@ -1138,14 +1157,21 @@ def load_benchmark_momentum(data_dir: Path) -> pl.DataFrame | None: return frame -def _bench_exchange_expr() -> pl.Expr: - """symbol 后缀 → 交易所 (SH/SZ/BJ), 无法识别时 null。""" +def _bench_key_expr() -> pl.Expr: + """symbol → 板块基准键 (SH/STAR/SZ/GEM/BJ), 无法识别时 null。 + + 北交所按后缀; 沪市按 68 前缀区分科创板; 深市按 30 前缀区分创业板。 + 与 abnormal_moves.board_of 的板块判定同口径。 + """ + code = pl.col("symbol").str.slice(0, 6) + suffix = pl.col("symbol").str.slice(-2).str.to_uppercase() return ( - pl.col("symbol").str.slice(-2).str.to_uppercase().replace( - {ex: ex for ex in _BENCHMARK_PREFERENCE}, - default=None, - return_dtype=pl.Utf8, - ) + pl.when(suffix == "BJ").then(pl.lit("BJ")) + .when((suffix == "SH") & code.str.starts_with("68")).then(pl.lit("STAR")) + .when(suffix == "SH").then(pl.lit("SH")) + .when((suffix == "SZ") & code.str.starts_with("30")).then(pl.lit("GEM")) + .when(suffix == "SZ").then(pl.lit("SZ")) + .otherwise(pl.lit(None, dtype=pl.Utf8)) ) @@ -1172,8 +1198,8 @@ def attach_deviation_columns(df: pl.DataFrame, data_dir: Path) -> pl.DataFrame: for n in missing ]) out = ( - df.with_columns(_bench_exchange_expr().alias("_bench_ex")) - .join(bench, left_on=["_bench_ex", "date"], right_on=["bench_exchange", "date"], how="left") + df.with_columns(_bench_key_expr().alias("_bench_ex")) + .join(bench, left_on=["_bench_ex", "date"], right_on=["bench_key", "date"], how="left") .with_columns([ (pl.col(f"momentum_{n}d") - pl.col(f"bench_mom{n}d")).alias(f"deviate_{n}d") for n in DEVIATION_WINDOWS @@ -1210,16 +1236,21 @@ def _bench_rt_pct_of(index_quotes: pl.DataFrame | None, candidates: list[str]) - return 0.0 +def bench_rt_pct_for(index_quotes: pl.DataFrame | None, bench_key: str) -> float: + """板块基准键的指数今日实时涨跌 (小数制), 供异动总览实时叠加等外部消费。""" + return _bench_rt_pct_of(index_quotes, _BENCHMARK_PREFERENCE.get(bench_key, [])) + + def benchmark_momentum_today( data_dir: Path, index_quotes: pl.DataFrame | None = None, ) -> pl.DataFrame | None: - """各交易所基准指数的「今日」N 日动量 (盘中实时外推)。 + """各板块基准指数的「今日」N 日动量 (盘中实时外推)。 基准日K parquet 盘中不含今日, 今日基准收盘 = 昨收 × (1 + 实时涨跌)。 - N 日动量 = 今日基准收盘 / N 个交易日前的收盘 - 1; 交易所与 + N 日动量 = 今日基准收盘 / N 个交易日前的收盘 - 1; 板块与 load_benchmark_momentum 的选基逻辑一致 (同一 TTL 缓存帧)。 - 返回小表: bench_exchange, bench_mom3d, bench_mom10d, bench_mom30d。 + 返回小表: bench_key, bench_mom3d, bench_mom10d, bench_mom30d。 无基准数据时 None。 """ bench = load_benchmark_momentum(data_dir) @@ -1232,15 +1263,15 @@ def benchmark_momentum_today( if bench.is_empty(): return None rows: list[dict[str, float | str]] = [] - for ex in sorted(bench["bench_exchange"].unique().to_list()): - sub = bench.filter(pl.col("bench_exchange") == ex).sort("date") + for k in sorted(bench["bench_key"].unique().to_list()): + sub = bench.filter(pl.col("bench_key") == k).sort("date") closes = sub["bench_close"] if closes.len() == 0: continue yesterday_close = closes[-1] - rt = _bench_rt_pct_of(index_quotes, _BENCHMARK_PREFERENCE.get(ex, [])) + rt = _bench_rt_pct_of(index_quotes, _BENCHMARK_PREFERENCE.get(k, [])) row: dict[str, float | str] = { - "bench_exchange": ex, + "bench_key": k, } for n in DEVIATION_WINDOWS: base = closes[-n] if closes.len() >= n else None # N 个交易日前 (不含今日) @@ -1252,7 +1283,7 @@ def benchmark_momentum_today( rows.append(row) if not rows: return None - schema = {"bench_exchange": pl.Utf8, **{f"bench_mom{n}d": pl.Float64 for n in DEVIATION_WINDOWS}} + schema = {"bench_key": pl.Utf8, **{f"bench_mom{n}d": pl.Float64 for n in DEVIATION_WINDOWS}} return pl.DataFrame(rows, schema=schema) @@ -1283,8 +1314,8 @@ def attach_deviation_columns_today( for n in DEVIATION_WINDOWS ] return ( - df.with_columns(_bench_exchange_expr().alias("_bench_ex")) - .join(bench, left_on="_bench_ex", right_on="bench_exchange", how="left") + df.with_columns(_bench_key_expr().alias("_bench_ex")) + .join(bench, left_on="_bench_ex", right_on="bench_key", how="left") .with_columns(exprs) .drop(["_bench_ex", *[f"bench_mom{n}d" for n in DEVIATION_WINDOWS]]) ) diff --git a/backend/app/jobs/daily_pipeline.py b/backend/app/jobs/daily_pipeline.py index f50b87f..b3f12e0 100644 --- a/backend/app/jobs/daily_pipeline.py +++ b/backend/app/jobs/daily_pipeline.py @@ -66,6 +66,48 @@ def _prune_partial_enriched_partitions(daily_dir: Path, enriched_dir: Path) -> l return pruned +def _prune_stale_price_partitions( + daily_dir: Path, enriched_dir: Path, max_dates: int = 5 +) -> list[str]: + """删除收盘价与官方日线不一致的 enriched 日期分区。 + + 实时 flush 写入的当日分区行数与 daily 相同, 但收盘价可能停留在收盘集合 + 竞价前的快照 (实测: TickFlow 实时端点收盘后仍长期返回旧价, 3392/5554 只 + 股票当日收盘价与官方日线不符), #223 的行数校验识别不到。对最近若干交易日 + 做值级比对: enriched.raw_close 与 daily.close 任一标的差超过半个最小报价 + 单位即删分区, 由后续增量重算按官方日线全市场重建。 + """ + import shutil + + common = sorted( + ( + p.stem.split("=", 1)[1] + for p in enriched_dir.glob("date=*") + if (daily_dir / p.stem).exists() + ), + reverse=True, + )[:max_dates] + pruned: list[str] = [] + for ds in common: + try: + daily = pl.read_parquet( + daily_dir / f"date={ds}" / "*.parquet", columns=["symbol", "close"] + ) + enr = pl.read_parquet( + enriched_dir / f"date={ds}" / "*.parquet", columns=["symbol", "raw_close"] + ) + except Exception: + continue # 列缺失/不可读 → 交给既有完整性检查兜底 + joined = enr.join(daily, on="symbol", how="inner").drop_nulls() + if joined.is_empty(): + continue + bad = joined.filter((pl.col("raw_close") - pl.col("close")).abs() > 0.005) + if not bad.is_empty(): + shutil.rmtree(enriched_dir / f"date={ds}", ignore_errors=True) + pruned.append(ds) + return pruned + + class PipelineStageError(RuntimeError): """管道有阶段软失败(数据可能陈旧)时抛出, 让上层 job_store 把任务标记为 failed。 @@ -388,14 +430,19 @@ def run_now( daily_days = len(list(daily_dir.glob("date=*"))) if daily_dir.exists() else 0 prev_enriched_days = len(list(enriched_dir.glob("date=*"))) if enriched_exists else 0 - # 部分分区修复 (#223): 删除被实时合并提前创建、覆盖不全的 enriched 分区, - # 让下方计数比较与增量计算把它们重新当新日期处理 + # 部分分区修复 (#223) + 收盘价过期分区修复: 删除被实时合并提前创建、覆盖不全 + # 或收盘价停留在竞价前快照的 enriched 分区, 让下方计数比较与增量计算把它们 + # 重新当新日期处理 (值级比对以官方日线为准, 实时源不纠错也能自愈) if enriched_exists: partial_pruned = _prune_partial_enriched_partitions(daily_dir, enriched_dir) - if partial_pruned: + stale_pruned = _prune_stale_price_partitions(daily_dir, enriched_dir) + pruned_dates = sorted(set(partial_pruned) | set(stale_pruned)) + if pruned_dates: logger.warning( - "compute_enriched: 发现 %d 个覆盖不全的 enriched 分区, 已删除待重算: %s", - len(partial_pruned), ", ".join(sorted(partial_pruned)[:10]), + "compute_enriched: 发现 %d 个异常 enriched 分区 (覆盖不全 %d / 收盘价过期 %d), " + "已删除待重算: %s", + len(pruned_dates), len(partial_pruned), len(stale_pruned), + ", ".join(pruned_dates[:10]), ) enriched_exists = enriched_dir.exists() and any(enriched_dir.glob("date=*")) prev_enriched_days = len(list(enriched_dir.glob("date=*"))) if enriched_exists else 0 diff --git a/backend/app/main.py b/backend/app/main.py index 647d0a6..78d1b0d 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -19,6 +19,7 @@ from app.api import ( backtest, data, ext_data, + factors, financials, indices, intraday, @@ -104,6 +105,15 @@ async def _application_lifespan(app: FastAPI): repo = KlineRepository(store) app.state.datastore = store app.state.repo = repo + # 自定义/复合因子载入注册表 (P3); 单个失败只跳过该因子 (fail-隔离) + from app.factors.store import load_into_registry + + try: + loaded_factors = load_into_registry(store.data_dir) + if loaded_factors: + logger.info("custom factors loaded: %s", len(loaded_factors)) + except Exception as exc: # noqa: BLE001 + logger.warning("custom factors load failed: %s", exc) from app.services.mining_manager import MiningJobManager mining_manager = MiningJobManager(store.data_dir) @@ -455,6 +465,7 @@ app.include_router(kline.router) app.include_router(watchlist.router) app.include_router(screener.router) app.include_router(backtest.router) +app.include_router(factors.router) app.include_router(mining.router) app.include_router(intraday.router) app.include_router(indices.router) diff --git a/backend/app/services/abnormal_moves.py b/backend/app/services/abnormal_moves.py index 582dc64..acaa83a 100644 --- a/backend/app/services/abnormal_moves.py +++ b/backend/app/services/abnormal_moves.py @@ -27,7 +27,7 @@ from typing import Any import polars as pl -from app.indicators.pipeline import DEVIATION_WINDOWS +from app.indicators.pipeline import BENCH_KEYS, DEVIATION_WINDOWS, bench_rt_pct_for # ── 规则表 ──────────────────────────────────────────────── @@ -57,6 +57,18 @@ RULES_META: list[dict[str, Any]] = [ _BENCH_RT_CANDIDATES = ["000002.SH", "000001.SH", "399107.SZ", "399001.SZ", "899050.BJ"] +def _bench_key_of(symbol: str) -> str: + """symbol → 板块基准键, 与 pipeline._bench_key_expr 同口径 (SH/STAR/SZ/GEM/BJ)。""" + code = symbol.split(".")[0] + if symbol.endswith(".BJ"): + return "BJ" + if symbol.endswith(".SH"): + return "STAR" if code.startswith("68") else "SH" + if symbol.endswith(".SZ"): + return "GEM" if code.startswith("30") else "SZ" + return "" + + def board_of(symbol: str) -> str: """按代码前缀判定板块。""" code = symbol.split(".")[0] @@ -174,6 +186,15 @@ def build_overview( hist_rows: dict[str, dict[str, Any]] = hist["rows"] bench_rt = _bench_rt_pct(quote_service) if quote_service is not None else 0.0 + # 实时叠加按板块基准: 科创板减科创50、创业板减创业板综指, 不再全市场混均值 + bench_by_key: dict[str, float] = {} + if quote_service is not None: + try: + index_quotes = quote_service.get_index_quotes() + except Exception: + index_quotes = None + for k in BENCH_KEYS: + bench_by_key[k] = bench_rt_pct_for(index_quotes, k) # enriched 已含今日收盘 (盘后已同步) 时, 今日涨跌已计入历史偏离, 不再叠加 includes_today = cache_date is not None and cache_date >= date.today().isoformat() @@ -181,7 +202,9 @@ def build_overview( for symbol, base in hist_rows.items(): rule = rule_for(symbol, base.get("name")) rt_pct = base.get("rt_pct") - rt_delta = 0.0 if includes_today else ((rt_pct or 0.0) - bench_rt) + rt_delta = 0.0 if includes_today else ( + (rt_pct or 0.0) - bench_by_key.get(_bench_key_of(symbol), 0.0) + ) windows: dict[str, dict[str, Any]] = {} max_closeness = 0.0 diff --git a/backend/app/services/auto_mining.py b/backend/app/services/auto_mining.py new file mode 100644 index 0000000..e88dee7 --- /dev/null +++ b/backend/app/services/auto_mining.py @@ -0,0 +1,193 @@ +"""自动挖掘 L1 编排: 全量因子统计筛选 → 达标池。 + +流程定位 (对应方案「四层漏斗」): +- L1 本模块: 注册表全量因子批量检验, 按置信档门槛筛出达标因子 (近期窗口, + 仅作"有信号"的先验过滤; 最终达标由挖掘引擎的逐折训练选择与嵌套样本外 + 验证裁定)。 +- L2/L3/L4 由现有挖掘引擎完成: 相关性剪枝 (prune_correlated_factors)、 + 束搜索组合 (beam_search_factor_combinations)、嵌套样本外验证与达标 + 门槛 (evaluate_candidate_gate), 本模块不重复实现。 + +达标判据与检验页服务端判读同源 (|t_NW| / BH q / |IC| / |IR|), 按档放宽或收紧; +q 值缺失时按"通过"处理 (探索档小样本下 BH 校正保守)。 +""" +from __future__ import annotations + +import math +import time +from dataclasses import dataclass +from datetime import date, timedelta +from typing import Any, Literal + +from app.backtest.factor import FactorBacktestService, FactorBatchConfig +from app.factors.registry import factor_columns_view + +Profile = Literal["exploratory", "balanced", "strict"] + +# 挖掘请求的因子池上限 (与 MiningStartRequest.factor_names max_length 对齐) +MAX_AUTO_POOL = 48 + +# L1 筛选窗口: 近一年 (与挖掘窗口解耦, 只筛"近期有信号", 长窗口验证交给引擎) +SCREEN_WINDOW_DAYS = 365 + + +@dataclass(frozen=True) +class ScreenGate: + min_abs_ic: float + min_abs_ir: float + min_abs_t: float + max_q: float + + def to_dict(self) -> dict[str, float]: + return { + "min_abs_ic": self.min_abs_ic, + "min_abs_ir": self.min_abs_ir, + "min_abs_t": self.min_abs_t, + "max_q": self.max_q, + } + + +SCREEN_GATES: dict[str, ScreenGate] = { + "exploratory": ScreenGate(min_abs_ic=0.02, min_abs_ir=0.15, min_abs_t=1.5, max_q=0.20), + "balanced": ScreenGate(min_abs_ic=0.02, min_abs_ir=0.30, min_abs_t=2.0, max_q=0.10), + "strict": ScreenGate(min_abs_ic=0.03, min_abs_ir=0.50, min_abs_t=2.5, max_q=0.05), +} + + +def classify_factor(item: dict[str, Any], gate: ScreenGate) -> str | None: + """返回 None 表示达标; 否则返回首个未过的门槛, 格式统一为「类别 (细节)」。""" + if item.get("error"): + return f"计算失败 ({str(item['error'])[:40]})" + ic = item.get("ic_mean") + ir = item.get("ir") + t = item.get("t_newey_west") + q = item.get("q_value") + if ic is None or ir is None: + return "样本不足 (无有效 IC/IR)" + if abs(ic) < gate.min_abs_ic: + return f"预测力弱 (|IC|<{gate.min_abs_ic:.2f})" + if abs(ir) < gate.min_abs_ir: + return f"稳定度低 (|IR|<{gate.min_abs_ir:.2f})" + if t is None: + return "样本不足 (无 NW t 值)" + if abs(t) < gate.min_abs_t: + return f"不显著 (|t|<{gate.min_abs_t:.1f})" + if q is not None and q > gate.max_q: + return f"多重检验未过 (q>{gate.max_q:.2f})" + return None + + +def _short_reason(reason: str) -> str: + """失败原因归并到短类别 (「类别 (细节)」的前半段), 供原因分布统计。""" + return reason.split(" (", 1)[0].strip() + + +def _finite_or_none(value: Any) -> float | None: + """NaN/Inf 一律归 None, 避免写入任务存储时产生非法 JSON。""" + if isinstance(value, (int, float)) and math.isfinite(value): + return float(value) + return None + + +def _metric_row(item: dict[str, Any]) -> dict[str, Any]: + return { + "factor_name": item.get("factor_name"), + "label": item.get("label") or item.get("factor_name"), + "group": item.get("group") or "", + "ic": _finite_or_none(item.get("ic_mean")), + "ir": _finite_or_none(item.get("ir")), + "t": _finite_or_none(item.get("t_newey_west")), + "q": _finite_or_none(item.get("q_value")), + "direction": 1 if (item.get("ic_mean") or 0) >= 0 else -1, + } + + +def screen_all_factors( + engine: Any, + *, + asset_type: str, + start: date | None, + end: date, + profile: str, + max_factors: int = MAX_AUTO_POOL, +) -> dict[str, Any]: + """L1 全量筛选: 注册表全部适用因子批量检验 → 达标池 + 失败原因分布。 + + start=None 时取近 SCREEN_WINDOW_DAYS 天; 显式 start 只会收紧 (不放宽) 筛选窗口。 + """ + gate = SCREEN_GATES.get(profile) + if gate is None: + raise ValueError(f"unknown mining profile: {profile}") + + candidates = [ + str(item["id"]) + for item in factor_columns_view() + if asset_type in item.get("asset_types", ["stock"]) + ] + screen_start = max(start or date.min, end - timedelta(days=SCREEN_WINDOW_DAYS)) + began = time.perf_counter() + service = FactorBacktestService(engine) + batch = service.run_batch(FactorBatchConfig( + factor_names=candidates, + symbols=None, + start=screen_start, + end=end, + rebalance="daily", + asset_type=asset_type, + )) + elapsed_ms = round((time.perf_counter() - began) * 1000, 1) + + qualified: list[dict[str, Any]] = [] + failed: list[dict[str, Any]] = [] + by_name = {str(getattr(item, "factor_name", None)): item for item in batch.results} + for name in candidates: + item = by_name.get(name) + if item is None: + failed.append({"factor_name": name, "label": name, "group": "", + "ic": None, "ir": None, "t": None, "q": None, + "reason": "未返回结果"}) + continue + # 非有限值先清洗 (NaN 与任何比较均为 False, 会绕过门槛误判达标) + reason = classify_factor({ + "error": getattr(item, "error", None), + "ic_mean": _finite_or_none(getattr(item, "ic_mean", None)), + "ir": _finite_or_none(getattr(item, "ir", None)), + "t_newey_west": _finite_or_none(getattr(item, "t_newey_west", None)), + "q_value": _finite_or_none(getattr(item, "q_value", None)), + }, gate) + row = _metric_row({ + "factor_name": getattr(item, "factor_name", None), + "label": getattr(item, "label", None), + "group": getattr(item, "group", None), + "ic_mean": getattr(item, "ic_mean", None), + "ir": getattr(item, "ir", None), + "t_newey_west": getattr(item, "t_newey_west", None), + "q_value": getattr(item, "q_value", None), + }) + if reason is None: + qualified.append(row) + else: + failed.append({**row, "reason": reason}) + + # 池按 |IC|*|IR| 降序 (截面信噪比口径), 截断到挖掘上限 + qualified.sort(key=lambda row: abs(row["ic"] or 0.0) * abs(row["ir"] or 0.0), reverse=True) + pool = [row["factor_name"] for row in qualified[:max_factors]] + + reason_counts: dict[str, int] = {} + for row in failed: + category = _short_reason(row["reason"]) + reason_counts[category] = reason_counts.get(category, 0) + 1 + + return { + "profile": profile, + "gate": gate.to_dict(), + "screen_window": {"start": screen_start.isoformat(), "end": end.isoformat()}, + "n_total": len(candidates), + "n_qualified": len(qualified), + "pool": pool, + "pool_truncated": len(qualified) > len(pool), + "qualified": qualified, + "failed": failed, + "reason_counts": dict(sorted(reason_counts.items(), key=lambda kv: -kv[1])), + "elapsed_ms": elapsed_ms, + } diff --git a/backend/app/services/quote_service.py b/backend/app/services/quote_service.py index 16a05db..0fa665a 100644 --- a/backend/app/services/quote_service.py +++ b/backend/app/services/quote_service.py @@ -33,7 +33,7 @@ from datetime import date, datetime, time as dt_time import polars as pl -from app.market_time import cn_now, cn_today +from app.market_time import CN_TZ, cn_now, cn_today from app.parquet import scan_daily_parquet from app.services.index_const import CORE_INDEX_SYMBOLS from app.strategy.intraday_signals import IntradaySignalEvaluator @@ -46,6 +46,15 @@ SOURCE_LABELS = { "volume_delta": "放量", "abnormal": "异动", "date": "日期提醒", } +# final 定版确认容差: 快照时间戳允许早于边界 5s 内 (供应商时间戳精度不一) +_FINAL_CONFIRM_SLACK_MS = 5_000 + +# final 定版边界与重试窗口终点 (北京时间)。收盘窗口终点 15:30, 恰与盘后管道 +# 启动同时: 管道运行期间轮询本就被暂停, 此后未确认的定版不再写盘, 当日分区 +# 由管道按官方日线值级校正 —— 避免定版重试与权威重建互相覆盖。 +_FINAL_BOUNDARY = {"morning_final": dt_time(11, 30), "close_final": dt_time(15, 0)} +_FINAL_DEADLINE = {"morning_final": dt_time(12, 10), "close_final": dt_time(15, 30)} + def _body_with_quote(body: str, ev: dict) -> str: """推送正文尾部补上触发时的现价/涨跌幅 (日期提醒无行情, 自然为空)。 @@ -240,6 +249,8 @@ class QuoteService: # 午休/收盘最终同步状态: 到边界后必须成功拉取一版行情, 再进入休盘态。 self._final_sync_done: set[tuple[date, str]] = set() self._final_sync_failed: dict[tuple[date, str], str] = {} + # 最近一次 final 定版拉取是否取得边界后快照 (None=非 final 拉取) + self._last_final_confirmed: bool | None = None self._holiday_active = False # 交易日探针当前是否判休市 (日志去重) # 轮询放量 (volume_delta 规则): 上一轮全市场股票快照的 (累计成交量[手], 累计成交额[元])。 # 每轮全量快照后更新 (含非连续竞价时段, 保证 13:00 恢复时 prev 是 12:59 @@ -562,8 +573,17 @@ class QuoteService: } def refresh(self) -> dict: - """手动触发一次行情拉取。""" - self._fetch_quotes() + """手动触发一次行情拉取。 + + 午休/收盘定版阶段同样走边界确认: 避免盘后手动刷新把竞价前的陈旧收盘价 + 重新写回当日分区, 覆盖盘后管道按官方日线重建的结果。 + """ + phase = self._market_phase() + is_final = phase in {"morning_final", "close_final"} + self._fetch_quotes( + final=is_final, + final_boundary_ms=self._final_boundary_ms(phase) if is_final else None, + ) return self.status() # ================================================================ @@ -579,16 +599,36 @@ class QuoteService: phase = self._market_phase() if self._should_fetch_for_phase(phase): is_final = phase in {"morning_final", "close_final"} - ok = self._fetch_quotes(final=is_final) + ok = self._fetch_quotes( + final=is_final, + final_boundary_ms=self._final_boundary_ms(phase), + ) if is_final: key = self._final_sync_key(phase) - if key and ok: + label = "午休" if phase == "morning_final" else "收盘" + if key and ok and self._last_final_confirmed: self._final_sync_done.add(key) self._final_sync_failed.pop(key, None) - logger.info("%s 最终行情同步完成, 进入休盘态", "午休" if phase == "morning_final" else "收盘") + logger.info("%s 最终行情同步完成 (快照时间戳已达边界), 进入休盘态", label) + elif key and self._past_final_deadline(phase): + # 重试窗口结束仍未取得边界后快照: 接受现状停止轮询。 + # 实测有实时源收盘后长期返回竞价前旧价 (快照时间戳可信但价格不更新), + # 此时盲目落盘只会固化旧价 —— 交由 15:30 盘后管道按官方日线校正。 + self._final_sync_done.add(key) + self._final_sync_failed[key] = ( + "fetch_failed" if not ok else "unconfirmed_snapshot" + ) + logger.warning( + "%s 定版窗口结束仍未取得边界后快照 (%s), 停止轮询; " + "当日分区由盘后管道按官方日线值级校正", + label, "拉取失败" if not ok else "快照未确认", + ) elif key: - self._final_sync_failed[key] = "fetch_failed" - logger.warning("%s 最终行情同步失败, 将继续重试", "午休" if phase == "morning_final" else "收盘") + self._final_sync_failed[key] = ( + "fetch_failed" if not ok else "unconfirmed_snapshot" + ) + if not ok: + logger.warning("%s 最终行情同步失败, 将继续重试", label) else: logger.debug("非轮询阶段(%s), 跳过行情轮询", phase) except Exception as e: # noqa: BLE001 @@ -599,16 +639,20 @@ class QuoteService: time.sleep(0.5) waited += 0.5 - def _fetch_quotes(self, *, final: bool = False) -> bool: - """拉取行情。加锁串行化 (后台轮询 vs 手动 refresh)。返回本轮是否成功更新。""" + def _fetch_quotes(self, *, final: bool = False, final_boundary_ms: int | None = None) -> bool: + """拉取行情。加锁串行化 (后台轮询 vs 手动 refresh)。返回本轮是否成功更新。 + + final_boundary_ms: final 定版的边界时间戳 (ms)。传入时快照时间戳未达边界 + 的本轮不落盘 (见 _process_full_market_records)。 + """ with self._fetch_lock: before = self._fetched_at if final: logger.info("最终行情同步开始") - self._fetch_full_market_quotes() + self._fetch_full_market_quotes(final_boundary_ms=final_boundary_ms) return self._fetched_at > before - def _fetch_full_market_quotes(self) -> None: + def _fetch_full_market_quotes(self, final_boundary_ms: int | None = None) -> None: """拉取全市场行情 → 写 daily + 计算 enriched + 更新缓存。""" from app.services import preferences @@ -627,7 +671,15 @@ class QuoteService: replace_index_cache = True fetch_indices = getattr(provider, "get_realtime_indices", None) if callable(fetch_indices): - wanted = sorted(set(CORE_INDEX_SYMBOLS) | self._collect_monitor_index_symbols()) + # 偏离值基准指数 (科创50/创业板综指等) 一并拉取, 供盘中 + # attach_deviation_columns_today 实时外推; 展示层仍按核心 + # 四只过滤, 多拉的指数不进侧栏。 + from app.indicators.pipeline import BENCHMARK_INDEX_SYMBOLS + wanted = sorted( + set(CORE_INDEX_SYMBOLS) + | BENCHMARK_INDEX_SYMBOLS + | self._collect_monitor_index_symbols() + ) try: fetched_indices = fetch_indices(wanted) if fetched_indices is None: @@ -645,6 +697,7 @@ class QuoteService: t0=t0, now_ts=now_ts, replace_index_cache=replace_index_cache, + final_boundary_ms=final_boundary_ms, ) return # 自定义源未配置 realtime → 回退 TickFlow @@ -684,8 +737,11 @@ class QuoteService: logger.info("拉取全市场行情 (universes=%s, SDK超时=30s×重试3)", universes) resp.extend(tf.quotes.get_by_universes(universes=universes) or []) logger.info("全市场行情拉取完成: %d 条 (%.2fs)", len(resp), time.perf_counter() - _u0) - # 指数: 固定核心四只 + 监控规则标的, 按码显式拉取 - _core_syms = sorted(core_index_symbols | monitor_index_symbols) + # 指数: 固定核心四只 + 偏离值基准指数 + 监控规则标的, 按码显式拉取 + from app.indicators.pipeline import BENCHMARK_INDEX_SYMBOLS + _core_syms = sorted( + core_index_symbols | BENCHMARK_INDEX_SYMBOLS | monitor_index_symbols + ) if _core_syms: _i0 = time.perf_counter() resp.extend(tf.quotes.get(symbols=_core_syms) or []) @@ -730,7 +786,9 @@ class QuoteService: "session": q.get("session"), }) - self._process_full_market_records(records, t0=t0, now_ts=now_ts) + self._process_full_market_records( + records, t0=t0, now_ts=now_ts, final_boundary_ms=final_boundary_ms + ) def _process_full_market_records( self, @@ -739,8 +797,14 @@ class QuoteService: t0: float, now_ts: float, replace_index_cache: bool = True, + final_boundary_ms: int | None = None, ) -> None: - """把全市场 records 写盘并增量计算 enriched。""" + """把全市场 records 写盘并增量计算 enriched。 + + final_boundary_ms (final 定版边界) 传入时, 快照最大时间戳未达边界的本轮 + 只更新展示缓存, 不写 daily/enriched、不评估监控 —— 防止收盘后数据源仍 + 返回竞价前旧价时把陈旧收盘价固化到当日分区。 + """ from app.services import preferences all_index_symbols = set(self._repo.get_index_symbol_set()) if self._repo else set() core_index_symbols = set(CORE_INDEX_SYMBOLS) @@ -755,6 +819,16 @@ class QuoteService: logger.warning("行情数据为空") return + # ---- final 定版确认: 快照最大时间戳达到边界 (含容差) 才允许落盘 ---- + confirmed_final: bool | None = None + if final_boundary_ms is not None: + ts_vals = [t for t in (r.get("timestamp") for r in records) if t] + max_ts = max(ts_vals) if ts_vals else None + confirmed_final = bool( + max_ts is not None and max_ts >= final_boundary_ms - _FINAL_CONFIRM_SLACK_MS + ) + self._last_final_confirmed = confirmed_final + index_records = [r for r in records if r.get("symbol") in all_index_symbols] etf_records = [r for r in records if r.get("symbol") in all_etf_symbols] stock_records = [ @@ -781,6 +855,16 @@ class QuoteService: _persist_last_fetch(fetched_at) logger.info("行情刷新: %d 只股票, %d 只ETF, %d 只指数, 耗时 %.0fms", len(stock_records), len(etf_records), len(index_records), fetch_ms) + if confirmed_final is False: + # 边界前的陈旧快照: 展示缓存已更新, 落盘与监控评估留待边界后快照。 + # 轮询线程会在定版窗口内持续重试, 窗口结束由 _poll_loop 放弃并告警。 + logger.info( + "final 快照未达定版边界 (max quote_ts=%s, 边界=%s), 本轮跳过落盘", + max_ts, final_boundary_ms, + ) + self._broadcast_quote_updated() + return + # 轮询放量状态更新 (volume_delta 规则的差值来源) self._update_volume_delta(stock_records, fetched_at) @@ -868,6 +952,18 @@ class QuoteService: result = df.select(select_exprs).with_columns( pl.lit(cn_today()).cast(pl.Date).alias("date"), ) + # 停牌股回归: 实时源对停牌标的返回停牌前最后一份快照 — OHLCV 全为旧日 + # 真实值, 仅 timestamp 停在旧日。这类记录不属于当日, 不过滤会把旧日 K 线 + # 原样复制成当日假蜡烛 (如 301266.SZ 2026-09-04)。按 quote_ts 的北京 + # 日期归属过滤; 时间戳缺失/为空的源无法判断, 维持原行为保留。 + if "quote_ts" in result.columns: + day_start_ms = int( + datetime.combine(cn_today(), dt_time(0, 0), tzinfo=CN_TZ).timestamp() * 1000 + ) + result = result.filter( + pl.col("quote_ts").is_null() + | pl.col("quote_ts").is_between(day_start_ms, day_start_ms + 86_400_000, closed="left") + ) # 停牌/尚无集合竞价的记录 open/high 均为 0。必须在下方用 close 填充前 # 过滤, 否则零成交行会被伪装成有效日K, 并在 batch 同步后作为实时残留 # 反复触发历史完整性修复。 @@ -979,6 +1075,20 @@ class QuoteService: return (cn_today(), "close") return None + @classmethod + def _final_boundary_ms(cls, phase: str) -> int | None: + """final 阶段定版边界的 epoch ms (按北京时间当日换算, 不依赖服务器时区)。""" + b = _FINAL_BOUNDARY.get(phase) + if b is None: + return None + return int(datetime.combine(cn_today(), b, tzinfo=CN_TZ).timestamp() * 1000) + + @classmethod + def _past_final_deadline(cls, phase: str) -> bool: + """是否已过 final 重试窗口终点 (用于放弃未确认的定版重试)。""" + dl = _FINAL_DEADLINE.get(phase) + return dl is not None and cn_now().time() >= dl + def _holiday_gate(self) -> bool: """交易日探针门控: 确定休市 → False (停止轮询, 含 final 定版)。 diff --git a/backend/app/strategy/custom_signals.py b/backend/app/strategy/custom_signals.py index e2f70d3..e78995b 100644 --- a/backend/app/strategy/custom_signals.py +++ b/backend/app/strategy/custom_signals.py @@ -66,6 +66,49 @@ _OP_BUILDERS = { } +def allowed_fields() -> frozenset[str]: + """条件可引用字段 = 物化列白名单 并入 注册表因子 (虚拟/自定义/复合)。 + + 因子列在历史路径 (compute_signals) 由 materialize_factor_columns 复用 + 评分物化管线补算; 盘中单日快照无滚动窗口, 依赖因子的信号被 inject 以 + 缺列告警跳过 (与日期偏移条件同样的优雅降级)。 + """ + from app.factors.registry import all_factors + + return frozenset(ALLOWED_FIELDS | {spec.id for spec in all_factors()}) + + +def materialize_factor_columns( + df: pl.DataFrame, + exprs: dict[str, pl.Expr], + needed: set[str] | None = None, +) -> pl.DataFrame: + """把信号表达式引用、且 df 缺失的注册表因子列补算出来。 + + 复用评分物化路径 (materialize_scoring_columns) — 与检验/评分同一条计算 + 逻辑, 不引入第二套实现。非注册表列不在此处理 (缺列仍由 inject 告警跳过)。 + """ + if df.is_empty() or not exprs: + return df + cols = set(df.columns) + missing: set[str] = set() + for name, roots in expression_dependencies(exprs).items(): + if needed is not None and name not in needed: + continue + missing.update(root for root in roots if root not in cols) + if not missing: + return df + from app.factors.registry import all_factors + + factor_ids = {spec.id for spec in all_factors()} + to_compute = missing & factor_ids + if not to_compute: + return df + from app.strategy.scoring import materialize_scoring_columns + + return materialize_scoring_columns(df, sorted(to_compute)) + + # ── 持久化(镜像 strategy/config.py 的写法)────────────── def _dir(data_dir: Path) -> Path: d = data_dir / "user_data" / "custom_signals" @@ -133,9 +176,10 @@ def _parse_right(right: str) -> tuple[str, object]: return ("const", float(right)) if not isinstance(right, str): raise ValueError(f"非法右值: {right!r}") + allowed = allowed_fields() if right.startswith("field:"): col = right[len("field:"):] - if col not in ALLOWED_FIELDS: + if col not in allowed: raise ValueError(f"右值字段不在白名单: {col}") return ("field", col) # 纯数字 @@ -144,7 +188,7 @@ def _parse_right(right: str) -> tuple[str, object]: except ValueError: pass # 裸字段名 — 兜底容错, 仍受白名单约束 - if right in ALLOWED_FIELDS: + if right in allowed: return ("field", right) raise ValueError(f"非法右值(应为 field:xxx 或数字): {right!r}") @@ -167,7 +211,7 @@ def validate(sig: dict) -> None: if not isinstance(c, dict): raise ValueError(f"第 {i+1} 个条件格式错误") left = c.get("left", "") - if left not in ALLOWED_FIELDS: + if left not in allowed_fields(): raise ValueError(f"第 {i+1} 个条件: 字段 {left!r} 不在白名单") if c.get("op") not in OPS: raise ValueError(f"第 {i+1} 个条件: 运算符 {c.get('op')!r} 非法") diff --git a/backend/app/strategy/custom_signals_ai.py b/backend/app/strategy/custom_signals_ai.py index 4b3344f..51a5ee3 100644 --- a/backend/app/strategy/custom_signals_ai.py +++ b/backend/app/strategy/custom_signals_ai.py @@ -32,8 +32,12 @@ _FENCED_JSON_RE = re.compile(r"```(?:json)?\s*\n?(.*?)```", re.DOTALL) def _format_fields() -> str: - """按类别格式化白名单字段(key(中文标签)),供 LLM 参考。""" - allowed = custom_signals.ALLOWED_FIELDS + """按类别格式化白名单字段(key(中文标签)), 供 LLM 参考. + + 行情/指标类物理列之后追加注册表因子, 分组与 /api/custom-signals/options + 的 factor 分组一致: 因子是预计算因子值, 同样可作为条件字段比较. + """ + allowed = custom_signals.allowed_fields() lines: list[str] = [] quote = sorted(f for f in _QUOTE_FIELDS if f in allowed) lines.append( @@ -46,6 +50,18 @@ def _format_fields() -> str: f"{label}: " + ", ".join(f"{f}({ENRICHED_COLUMNS.get(f, f)})" for f in fields) ) + from app.factors.registry import all_factors + + factor_groups: dict[str, list[str]] = {} + for spec in all_factors(): + if spec.id in custom_signals.ALLOWED_FIELDS: + continue # 已作为物理列出现在清单里 + label = spec.label + if spec.asset_types == frozenset({"stock"}): + label += "·仅股票" + factor_groups.setdefault(spec.group, []).append(f"{spec.id}({label})") + for group, items in sorted(factor_groups.items()): + lines.append(f"因子·{group}: " + ", ".join(sorted(items))) return "\n".join(lines) @@ -53,6 +69,7 @@ _SYSTEM_TEMPLATE = """你是A股量化信号设计专家。用户会描述一个 可用字段(白名单,只能使用以下字段,禁止自造或使用白名单之外的字段): {fields} +其中「因子·」开头的行是平台预计算的因子值(动量/波动/量价等衍生特征),可直接比较数值构造条件。 运算符(op):> >= < <= == != @@ -132,8 +149,8 @@ def _normalize_condition(c: object) -> dict: if not isinstance(right, str) or not right.strip(): raise ValueError(f"右值非法: {right!r}") right = right.strip() - # 兜底: AI 偶尔漏写 field: 前缀的裸字段名, 补全为规范形式 - if not right.startswith("field:") and right in custom_signals.ALLOWED_FIELDS: + # 兜底: AI 偶尔漏写 field: 前缀的裸字段名, 补全为规范形式 (含因子字段) + if not right.startswith("field:") and right in custom_signals.allowed_fields(): right = f"field:{right}" return { "left": str(left), diff --git a/backend/app/strategy/scoring.py b/backend/app/strategy/scoring.py index dca28ca..2fc60e3 100644 --- a/backend/app/strategy/scoring.py +++ b/backend/app/strategy/scoring.py @@ -6,61 +6,22 @@ from typing import Any import polars as pl +from app.factors.registry import ( + factor_dependencies as _registry_factor_dependencies, +) +from app.factors.registry import get_factor as _registry_get_factor +from app.factors.registry import scoring_warmups as _registry_scoring_warmups +from app.factors.registry import virtual_dependencies as _registry_virtual_dependencies + SCORING_DIRECTION_HIGH = "high" SCORING_DIRECTION_LOW = "low" SCORING_DIRECTIONS = frozenset({SCORING_DIRECTION_HIGH, SCORING_DIRECTION_LOW}) -VIRTUAL_SCORING_DEPENDENCIES: dict[str, frozenset[str]] = { - **{ - f"ma{period}_bias": frozenset({"close", f"ma{period}"}) - for period in (5, 10, 20, 30, 60) - }, - **{ - f"ema{period}_bias": frozenset({"close", f"ema{period}"}) - for period in (5, 10, 20, 30, 60) - }, - "macd_dif_pct": frozenset({"close", "macd_dif"}), - "macd_dea_pct": frozenset({"close", "macd_dea"}), - "macd_hist_pct": frozenset({"close", "macd_hist"}), - "boll_position": frozenset({"close", "boll_upper", "boll_lower"}), - "atr_pct": frozenset({"close", "atr_14"}), - "boll_width": frozenset({"ma20", "boll_upper", "boll_lower"}), - "vol_ratio_10d": frozenset({"volume"}), - "vol_trend_5_10": frozenset({"vol_ma5", "vol_ma10"}), - "turnover_ratio_5d": frozenset({"turnover_rate"}), - "log_amount": frozenset({"amount"}), - "amount_ratio_5d": frozenset({"amount"}), - "gap_return": frozenset({"open", "prev_close"}), - "intraday_return": frozenset({"open", "close"}), - "close_position": frozenset({"high", "low", "close"}), - "distance_to_high_60d": frozenset({"close", "high_60d"}), - "distance_from_low_60d": frozenset({"close", "low_60d"}), - "max_ret_20d": frozenset({"close"}), - "ret_skew_20d": frozenset({"close"}), - "up_days_20d": frozenset({"close"}), - "amihud_20d": frozenset({"close", "amount"}), - "turnover_z_60d": frozenset({"turnover_rate"}), - "vol_price_corr_20d": frozenset({"close", "volume"}), - "vwap_bias": frozenset({"close", "volume", "amount"}), - "vol_trend_5_60": frozenset({"volume"}), - "limit_up_count_20d": frozenset({"consecutive_limit_ups"}), - "limit_up_count_60d": frozenset({"consecutive_limit_ups"}), -} +# P1 起依赖声明与预热窗口的单一权威来源为 app/factors/registry.py; +# 本常量为兼容别名, 键集合与历史版本逐项一致 (见 tests/test_factor_registry.py 快照测试)。 +VIRTUAL_SCORING_DEPENDENCIES: dict[str, frozenset[str]] = dict(_registry_virtual_dependencies()) -_ROLLING_SCORING_WARMUP: dict[str, int] = { - "vol_ratio_10d": 11, - "turnover_ratio_5d": 6, - "amount_ratio_5d": 6, - "max_ret_20d": 21, - "ret_skew_20d": 21, - "up_days_20d": 21, - "amihud_20d": 21, - "turnover_z_60d": 61, - "vol_price_corr_20d": 21, - "vol_trend_5_60": 60, - "limit_up_count_20d": 21, - "limit_up_count_60d": 61, -} +_ROLLING_SCORING_WARMUP: dict[str, int] = dict(_registry_scoring_warmups()) def effective_scoring( @@ -89,27 +50,61 @@ def effective_scoring_directions(overrides: Mapping[str, Any] | None) -> dict[st def scoring_warmup_bars(scoring: Mapping[str, Any]) -> int: - return max( - (_ROLLING_SCORING_WARMUP.get(str(name), 1) for name, weight in scoring.items() if weight), - default=1, - ) + warmups: list[int] = [ + _ROLLING_SCORING_WARMUP.get(str(name), 1) + for name, weight in scoring.items() + if weight + ] + # composite/custom 因子的预热来自注册表 (P3) + for name, weight in scoring.items(): + if not weight: + continue + spec = _registry_get_factor(str(name)) + if spec is not None and spec.kind in ("custom", "composite"): + warmups.append(spec.warmup_bars) + return max(warmups, default=1) def scoring_dependencies(scoring: Mapping[str, Any]) -> set[str]: - """把受控虚拟评分字段展开为实际数据依赖。""" + """把受控虚拟评分字段展开为实际数据依赖 (含 composite/custom 递归展开)。""" dependencies: set[str] = set() for name, weight in scoring.items(): if not weight: continue - dependencies.update(VIRTUAL_SCORING_DEPENDENCIES.get(str(name), {str(name)})) + dependencies.update(_registry_factor_dependencies([str(name)])) return dependencies +def _composite_value_expr(available: set[str], name: str) -> pl.Expr | None: + """复合因子值 = Σ w_i * 截面 zscore(成员值); 成员可为已物化列或虚拟因子。""" + spec = _registry_get_factor(name) + if spec is None or not spec.components: + return None + total: pl.Expr | None = None + for member_id, weight in spec.components: + member_expr = ( + pl.col(member_id) + if member_id in available + else scoring_value_expr(available, member_id) + ) + if member_expr is None: + return None + mean = member_expr.mean().over("date") + std = member_expr.std().over("date") + piece = pl.when(std > 0).then((member_expr - mean) / std).otherwise(None) * weight + total = piece if total is None else total + piece + return total + + def scoring_value_expr(columns: Collection[str], name: str) -> pl.Expr | None: """返回评分值表达式;依赖不完整时返回 None。""" available = set(columns) if name in available: return pl.col(name) + # composite 在 VIRTUAL 字典门控之前分派 (依赖经注册表递归展开) —— P3 + spec = _registry_get_factor(name) + if spec is not None and spec.kind == "composite": + return _composite_value_expr(available, name) dependencies = VIRTUAL_SCORING_DEPENDENCIES.get(name) if dependencies is None or not dependencies.issubset(available): return None @@ -202,6 +197,66 @@ def scoring_value_expr(columns: Collection[str], name: str) -> pl.Expr | None: window = 20 if name == "limit_up_count_20d" else 60 hit = (pl.col("consecutive_limit_ups").fill_null(0) > 0).cast(pl.Float64) return hit.rolling_sum(window, min_samples=window).over("symbol") + # ── 扩充批次 (2026-09-05): 全部滚动窗口默认 min_samples=窗口长 (fail-closed) ── + if name == "log_float_mv": + # 换手率 = 成交量/流通股本 → 股本 = volume/turnover_rate, 市值 = close x 股本 + return ( + pl.when((pl.col("turnover_rate") > 0) & (pl.col("volume") > 0)) + .then((pl.col("close") * pl.col("volume") / pl.col("turnover_rate")).log()) + .otherwise(None) + ) + if name == "momentum_120d": + return _relative( + pl.col("close"), + pl.col("close").shift(120), + ).over("symbol") + if name == "mom_accel_20_60": + return pl.col("momentum_20d") - pl.col("momentum_60d") + if name == "rsi_14_delta_5d": + return pl.col("rsi_14") - pl.col("rsi_14").shift(5).over("symbol") + if name == "overnight_ret_20d": + overnight = _relative(pl.col("open"), pl.col("prev_close")) + return overnight.rolling_sum(20, min_samples=20).over("symbol") + if name == "intraday_ret_20d": + intraday = _relative(pl.col("close"), pl.col("open")) + return intraday.rolling_sum(20, min_samples=20).over("symbol") + if name == "downside_vol_20d": + downside = (_daily_change_expr().clip(upper_bound=0.0) ** 2) + return downside.rolling_mean(20, min_samples=20).sqrt().over("symbol") + if name == "vol_regime_5_60": + change = _daily_change_expr() + fast = change.rolling_std(5, min_samples=5) + slow = change.rolling_std(60, min_samples=60) + return _ratio(fast, slow).over("symbol") + if name == "amplitude_trend_20_60": + fast = pl.col("amplitude").rolling_mean(20, min_samples=20) + slow = pl.col("amplitude").rolling_mean(60, min_samples=60) + return _relative(fast, slow).over("symbol") + if name == "obv_trend_20d": + change = _daily_change_expr() + signed = change.sign() * pl.col("volume") + total = signed.rolling_sum(20, min_samples=20) + scale = pl.col("volume").rolling_mean(20, min_samples=20) * 20.0 + return _ratio(total, scale).over("symbol") + if name == "amount_mean_20d": + return (pl.col("amount") / 1e8).rolling_mean(20, min_samples=20).over("symbol") + if name == "turnover_mean_20d": + return pl.col("turnover_rate").rolling_mean(20, min_samples=20).over("symbol") + if name == "turnover_std_20d": + mean = pl.col("turnover_rate").rolling_mean(20, min_samples=20) + std = pl.col("turnover_rate").rolling_std(20, min_samples=20) + return _ratio(std, mean).over("symbol") + if name == "position_240d": + high = pl.col("close").rolling_max(240, min_samples=240) + low = pl.col("close").rolling_min(240, min_samples=240) + return _ratio(pl.col("close") - low, high - low).over("symbol") + if name == "distance_to_high_240d": + return _relative( + pl.col("close"), + pl.col("close").rolling_max(240, min_samples=240), + ).over("symbol") + if name == "kdj_kd_diff": + return pl.col("kdj_k") - pl.col("kdj_d") return None @@ -233,6 +288,21 @@ def materialize_scoring_columns( frame: pl.DataFrame, names: Collection[str], ) -> pl.DataFrame: + # custom (DSL) 因子先物化: frame_transform 可能需要多阶段临时列 (嵌套窗口规避), + # 与单表达式路径不同, 必须整体走帧变换 —— 与检验/试算共用同一条计算路径 (P3)。 + from app.factors.dsl import FACTOR_COLUMN, compile_formula_cached + + for name in names: + spec = _registry_get_factor(str(name)) + if spec is None or spec.kind != "custom" or name in frame.columns: + continue + compiled = compile_formula_cached(spec.formula_text) + if compiled.frame_transform is None: + continue + transformed = compiled.frame_transform(frame) + if transformed is None: + continue + frame = transformed.with_columns(pl.col(FACTOR_COLUMN).alias(str(name))).drop(FACTOR_COLUMN) expressions = [ expression.alias(name) for name in names diff --git a/backend/pyproject.toml b/backend/pyproject.toml index a3737bc..5dcd103 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tickflow-stock-panel-backend" -version = "0.2.2" +version = "0.2.3" description = "A 股选股 + 监控 + 回测面板 — TickFlow 适配" requires-python = ">=3.11" license = { text = "MIT" } diff --git a/backend/tests/backtest/test_strategy_backtest_correctness.py b/backend/tests/backtest/test_strategy_backtest_correctness.py index 4dbde70..4919402 100644 --- a/backend/tests/backtest/test_strategy_backtest_correctness.py +++ b/backend/tests/backtest/test_strategy_backtest_correctness.py @@ -636,3 +636,57 @@ def test_matrix_cache_preserves_trades_daily_equity_and_core_stats(): assert cached.stats[name] == uncached.stats[name] assert cached_again.stats[name] == uncached.stats[name] assert cached_again.stats["matrix_compute_cache"]["hits"] > 0 + + +def test_apply_score_captures_factor_snapshot_for_candidates(): + panel = pl.DataFrame({ + "symbol": ["A", "B", "C"], + "date": [date(2024, 1, 1)] * 3, + "factor": [10.0, 20.0, 1000.0], + }) + universe = pl.Series([True, True, False], dtype=pl.Boolean) + strategy = SimpleNamespace(meta={"scoring": {"factor": 1.0}, "order_by": "score", "descending": True}) + + snapshot: dict = {} + StrategyBacktestService._apply_score(panel, strategy, None, universe_mask=universe, factor_snapshot=snapshot) + frame = snapshot["frame"] + assert frame.columns == ["symbol", "date", "factor"] + assert frame["symbol"].to_list() == ["A", "B"] # 非候选行不入快照 + + +def test_full_mode_reports_factor_attribution(): + start = date(2024, 1, 1) + panel = pl.DataFrame([ + {"symbol": "A", "name": "A", "date": start, "open": 10.0, "high": 10.0, "low": 10.0, "close": 10.0, "volume": 1, "amount": 1000.0, "signal_limit_up": False, "signal_limit_down": False, "factor": 5.0}, + {"symbol": "A", "name": "A", "date": start + timedelta(days=1), "open": 11.0, "high": 11.0, "low": 11.0, "close": 11.0, "volume": 1, "amount": 0.0, "signal_limit_up": False, "signal_limit_down": False, "factor": 5.0}, + {"symbol": "A", "name": "A", "date": start + timedelta(days=2), "open": 20.0, "high": 20.0, "low": 20.0, "close": 20.0, "volume": 1, "amount": 1000.0, "signal_limit_up": False, "signal_limit_down": False, "factor": 5.0}, + ]).sort(["symbol", "date"]) + + engine = BacktestEngine(repo=None) # type: ignore[arg-type] + engine.load_panel_for_backtest = lambda symbols, s, e, plan, asset_type="stock": panel # type: ignore[method-assign] + strategy = _strategy( + filter_fn=lambda df, params: pl.col("date") == start, + max_hold_days=1, + ) + strategy.meta["scoring"] = {"factor": 1.0} + service = StrategyBacktestService(engine=engine, strategy_engine=_StrategyEngineStub(strategy)) + + result = service.run(StrategyBacktestConfig( + strategy_id="test", + symbols=None, + start=start, + end=start, + mode="full", + matching="open_t+1", + fees_pct=0, + slippage_bps=0, + holding_days=1, + )) + + assert result.error is None + assert result.stats["n_trades"] == 1 + attribution = result.factor_attribution + assert attribution is not None + assert attribution["n_win"] == 1 and attribution["n_lose"] == 0 + assert attribution["factors"][0]["factor"] == "factor" + assert attribution["factors"][0]["win_mean"] == 5.0 diff --git a/backend/tests/test_abnormal_moves.py b/backend/tests/test_abnormal_moves.py index 6c1d079..77debc9 100644 --- a/backend/tests/test_abnormal_moves.py +++ b/backend/tests/test_abnormal_moves.py @@ -465,3 +465,90 @@ def test_engine_abnormal_down_direction_event_type() -> None: events = engine.evaluate_abnormal([_row("600000.SH", ("3d", -0.16))], now=1006.0) assert len(events) == 1 assert events[0]["type"] == "abnormal_down" + + +# ── 板块基准路由: 科创板→科创50, 创业板→创业板综指 (交易所官方对应指数) ── + +def test_attach_deviation_columns_board_routing(tmp_path) -> None: + """科创板股票减科创50、创业板股票减创业板综指, 不再共用交易所 A 指。 + + 构造 68/30 前缀股票 close 与各自官方基准同序列 → 偏离恰为 0。 + """ + days = [date(2026, 8, 13), date(2026, 8, 14), date(2026, 8, 15), date(2026, 8, 18)] + rows = [("000688.SH", d, 100.0 + i) for i, d in enumerate(days)] # 科创50 + rows += [("399102.SZ", d, 200.0 + 5 * i) for i, d in enumerate(days)] # 创业板综指 + _write_index_daily(tmp_path, rows) + + stock = pl.DataFrame( + { + "symbol": ["688981.SH"] * 4 + ["300750.SZ"] * 4, + "date": days * 2, + "close": [100.0 + i for i in range(4)] + [200.0 + 5 * i for i in range(4)], + } + ) + out = attach_deviation_columns(stock, tmp_path) + star = out.filter(pl.col("symbol") == "688981.SH").sort("date")["deviate_3d"][-1] + gem = out.filter(pl.col("symbol") == "300750.SZ").sort("date")["deviate_3d"][-1] + assert abs(star - 0.0) < 1e-9 + assert abs(gem - 0.0) < 1e-9 + + +def test_attach_deviation_columns_star_fallback(tmp_path) -> None: + """科创50 数据缺失 → 回退上证A指, 偏离列不整体缺失。""" + days = [date(2026, 8, 13), date(2026, 8, 14), date(2026, 8, 15), date(2026, 8, 18)] + _write_index_daily(tmp_path, [("000002.SH", d, 10.0 + i) for i, d in enumerate(days)]) + + stock = pl.DataFrame( + { + "symbol": ["688981.SH"] * 4, + "date": days, + "close": [10.0 + i for i in range(4)], + } + ) + out = attach_deviation_columns(stock, tmp_path) + dev = out.sort("date")["deviate_3d"][-1] + assert dev is not None + assert abs(dev - 0.0) < 1e-9 + + +def test_benchmark_momentum_today_gem_key(tmp_path) -> None: + """盘中外推按板块键产出: 创业板综指行情只喂 GEM 键的实时涨跌。""" + _write_index_daily(tmp_path, [("399102.SZ", d, 10.0 + i) for i, d in enumerate(_BENCH_DAYS)]) + quotes = pl.DataFrame({"symbol": ["399102.SZ"], "change_pct": [10.0]}) # 百分数: +10% + + out = benchmark_momentum_today(tmp_path, quotes) + gem = out.filter(pl.col("bench_key") == "GEM") + assert gem.height == 1 + assert abs(gem["bench_mom3d"][0] - (15.0 * 1.10 / 13 - 1)) < 1e-9 + # 其他键 (如 SH) 候选不在实时缓存 → rt=0, 用昨收外推 + sh = out.filter(pl.col("bench_key") == "SH") + assert sh.height == 1 + assert abs(sh["bench_mom3d"][0] - (15.0 / 13 - 1)) < 1e-9 + + +def test_build_overview_rt_overlay_uses_board_benchmark() -> None: + """实时叠加按板块基准: 创业板股减创业板综指今日涨跌, 不再全市场混均值。""" + with _hist_cache_lock: + _hist_cache.clear() + + class _GemQuotes: + def get_index_quotes(self): + # 百分数口径: 创业板综指 +2%, 其余基准无行情 → 0 + return pl.DataFrame({"symbol": ["399102.SZ"], "change_pct": [2.0]}) + + df = pl.DataFrame( + { + "symbol": ["300001.SZ", "600000.SH"], + "name": ["创业板股", "沪主板股"], + "close": [10.0, 20.0], + "change_pct": [0.10, 0.10], + "deviate_3d": [0.25, 0.25], + "deviate_10d": [None, None], + "deviate_30d": [None, None], + } + ) + result = build_overview(_FakeRepo(df), _GemQuotes(), min_closeness=0.5) + by_symbol = {r["symbol"]: r for r in result["rows"]} + # 创业板: 0.25 + (0.10 - 0.02) = 0.33; 沪主板无对应行情: 0.25 + 0.10 = 0.35 + assert abs(by_symbol["300001.SZ"]["windows"]["3d"]["value"] - 0.33) < 1e-9 + assert abs(by_symbol["600000.SH"]["windows"]["3d"]["value"] - 0.35) < 1e-9 diff --git a/backend/tests/test_auto_mining.py b/backend/tests/test_auto_mining.py new file mode 100644 index 0000000..ab004c9 --- /dev/null +++ b/backend/tests/test_auto_mining.py @@ -0,0 +1,214 @@ +"""自动挖掘 L1 筛选 (auto_mining) 与 /api/backtest/mining/auto 契约测试。""" +from __future__ import annotations + +from datetime import date, timedelta +from types import SimpleNamespace +from typing import Any, ClassVar + +import pytest + +from app.services import auto_mining +from app.services.auto_mining import ( + SCREEN_GATES, + classify_factor, + screen_all_factors, +) + +# ── classify_factor: 门槛分支 ── + + +def _item(**overrides: Any) -> dict[str, Any]: + base = {"error": None, "ic_mean": 0.05, "ir": 0.4, "t_newey_west": 2.5, "q_value": 0.05} + base.update(overrides) + return base + + +def test_classify_balanced_gate_branches() -> None: + gate = SCREEN_GATES["balanced"] + assert classify_factor(_item(), gate) is None + assert "计算失败" in classify_factor(_item(error="boom"), gate) + assert "样本不足" in classify_factor(_item(ic_mean=None), gate) + assert "预测力弱" in classify_factor(_item(ic_mean=0.01), gate) + assert "稳定度低" in classify_factor(_item(ic_mean=0.05, ir=0.1), gate) + assert "NW t" in classify_factor(_item(t_newey_west=None), gate) + assert "不显著" in classify_factor(_item(t_newey_west=1.0), gate) + assert "多重检验" in classify_factor(_item(q_value=0.5), gate) + # q 缺失按通过 (探索档小样本口径) + assert classify_factor(_item(q_value=None), gate) is None + + +def test_classify_profile_gates_tighten() -> None: + item = _item(ic_mean=-0.025, ir=-0.35, t_newey_west=-2.2, q_value=0.08) + # 负值同样达标 (方向反向), 严格档收紧后不达标 + assert classify_factor(item, SCREEN_GATES["balanced"]) is None + assert classify_factor(item, SCREEN_GATES["strict"]) is not None + # 探索档最宽 + weak = _item(ic_mean=0.021, ir=0.16, t_newey_west=1.6, q_value=0.18) + assert classify_factor(weak, SCREEN_GATES["exploratory"]) is None + assert classify_factor(weak, SCREEN_GATES["balanced"]) is not None + + +# ── screen_all_factors: 池构造/排序/截断/清洗 ── + + +class _StubService: + calls: ClassVar[list[Any]] = [] + results: ClassVar[list[Any]] = [] + + def run_batch(self, config: Any) -> Any: + _StubService.calls.append(config) + return SimpleNamespace(results=_StubService.results) + + +def _batch_item(name: str, **overrides: Any) -> SimpleNamespace: + base = { + "factor_name": name, "label": f"label_{name}", "group": "g", + "ic_mean": 0.05, "ir": 0.4, "t_newey_west": 2.5, "q_value": 0.05, + "error": None, + } + base.update(overrides) + return SimpleNamespace(**base) + + +@pytest.fixture() +def _stub_batch(monkeypatch: pytest.MonkeyPatch) -> type[_StubService]: + _StubService.calls = [] + monkeypatch.setattr(auto_mining, "FactorBacktestService", lambda engine: _StubService()) + return _StubService + + +def test_screen_pool_order_truncation_and_reasons(monkeypatch: pytest.MonkeyPatch, _stub_batch: type[_StubService]) -> None: + monkeypatch.setattr(auto_mining, "factor_columns_view", lambda: [ + {"id": name, "asset_types": ["stock"]} for name in ("a", "b", "c", "d", "e", "f") + ]) + items = [ + _batch_item("a", ic_mean=0.10, ir=0.8), # |ic|*|ir|=0.08 → 第 1 + _batch_item("b", ic_mean=0.05, ir=0.4), # 0.02 → 第 2 + _batch_item("c", ic_mean=-0.06, ir=-0.5), # 0.03 → 第 3 (负 IC 反向) + _batch_item("d", ic_mean=0.01), # 预测力弱 + _batch_item("e", ir=0.1), # 稳定度低 + _batch_item("f", t_newey_west=float("nan")), # NaN 清洗 → 样本不足 + ] + _stub_batch.results = items + summary = screen_all_factors( + object(), asset_type="stock", + start=date.today() - timedelta(days=800), + end=date.today(), profile="balanced", + max_factors=2, + ) + assert summary["pool"] == ["a", "c"] # 0.08 > 0.03 > 0.02, 截断到 2 + assert summary["pool_truncated"] is True + assert summary["n_qualified"] == 3 + assert summary["n_total"] == len(summary["qualified"]) + len(summary["failed"]) + assert {row["factor_name"]: row["direction"] for row in summary["qualified"]}["c"] == -1 + # NaN 指标被清洗为 None, 归入样本不足而非写入非法 JSON + row_f = next(row for row in summary["failed"] if row["factor_name"] == "f") + assert row_f["t"] is None and "样本不足" in row_f["reason"] + counts = summary["reason_counts"] + assert counts["预测力弱"] == 1 + assert counts["稳定度低"] == 1 + assert any(key.startswith("样本不足") for key in counts) + assert sum(counts.values()) == 3 + + +def test_screen_window_capped_and_daily_rebalance(monkeypatch: pytest.MonkeyPatch, _stub_batch: type[_StubService]) -> None: + _stub_batch.results = [] + start = date.today() - timedelta(days=1000) + screen_all_factors( + object(), asset_type="stock", start=start, end=date.today(), + profile="exploratory", + ) + config = _stub_batch.calls[0] + assert config.rebalance == "daily" + assert (config.end - config.start).days <= auto_mining.SCREEN_WINDOW_DAYS + # 因子清单来自注册表 (动态视图, 数量与内置一致量级) + assert len(config.factor_names) >= 50 + + +# ── API 契约: /api/backtest/mining/auto ── + + +class _FakeManager: + def __init__(self, store: Any) -> None: + self.store = store + self.start_calls: list[dict[str, Any]] = [] + + def start(self, request, fingerprint, force=False, source="manual", run_id=None): + self.start_calls.append({"request": request, "fingerprint": fingerprint, + "force": force, "source": source}) + return self.store.create(request, fingerprint) + + +@pytest.fixture() +def client(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> Any: + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from app.api import mining as mining_api + from app.services.mining_jobs import MiningRunStore + + store = MiningRunStore(tmp_path / "runs") + manager = _FakeManager(store) + + app = FastAPI() + app.include_router(mining_api.router) + app.state.repo = SimpleNamespace(store=SimpleNamespace(data_dir=tmp_path)) + app.state.mining_manager = manager + + monkeypatch.setattr(mining_api, "require_mining_availability", lambda *a, **k: None) + monkeypatch.setattr(mining_api, "enriched_partition_dates", lambda *a, **k: ["2026-08-31"]) + monkeypatch.setattr(mining_api, "build_data_fingerprint", lambda *a, **k: {"fp": 1}) + return SimpleNamespace(client=TestClient(app), manager=manager, store=store) + + +def _screening(pool: list[str]) -> dict[str, Any]: + return { + "profile": "balanced", "gate": {"min_abs_ic": 0.02}, "n_total": 61, + "n_qualified": len(pool), "pool": pool, "pool_truncated": False, + "qualified": [], "failed": [], "reason_counts": {}, + "screen_window": {"start": "2025-08-31", "end": "2026-08-31"}, + "elapsed_ms": 1.0, + } + + +def test_auto_start_contract_with_pool(client: Any, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + auto_mining, "screen_all_factors", + lambda *a, **k: _screening(["f1", "f2"]), + ) + response = client.client.post("/api/backtest/mining/auto", json={"asset_type": "stock"}) + assert response.status_code == 200 + body = response.json() + assert body["started"] is True + assert body["run"]["status"] == "queued" + call = client.manager.start_calls[0] + assert call["source"] == "auto" + request = call["request"] + assert request["factor_names"] == ["f1", "f2"] + assert request["strategy_ids"] == [] + assert request["auto"] is True + assert request["auto_screening"]["pool"] == ["f1", "f2"] + # 持久化 roundtrip: 任务存储里的 request 保留筛选摘要, 供结果页展示 + manifest = client.store.get(body["run"]["run_id"]) + assert manifest is not None + assert manifest["request"]["auto_screening"]["n_total"] == 61 + + +def test_auto_start_no_qualified_factors(client: Any, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + auto_mining, "screen_all_factors", + lambda *a, **k: _screening([]), + ) + response = client.client.post("/api/backtest/mining/auto", json={}) + assert response.status_code == 200 + body = response.json() + assert body["started"] is False + assert body["reason"] == "no_qualified_factors" + assert client.manager.start_calls == [] + + +def test_auto_start_rejects_bad_date_range(client: Any) -> None: + response = client.client.post("/api/backtest/mining/auto", json={ + "start": "2026-09-01", "end": "2026-08-01", + }) + assert response.status_code == 422 diff --git a/backend/tests/test_custom_provider_indices.py b/backend/tests/test_custom_provider_indices.py index de4603f..1812de0 100644 --- a/backend/tests/test_custom_provider_indices.py +++ b/backend/tests/test_custom_provider_indices.py @@ -14,6 +14,7 @@ from typing import ClassVar import polars as pl +from app.indicators.pipeline import BENCHMARK_INDEX_SYMBOLS from app.services import quote_service as qs from app.services.index_const import CORE_INDEX_SYMBOLS @@ -66,7 +67,7 @@ def _service_with_provider( monkeypatch.setattr(custom_mod, "get_provider", lambda name: provider) monkeypatch.setattr( service, "_process_full_market_records", - lambda records, *, t0, now_ts, replace_index_cache=True: ( + lambda records, *, t0, now_ts, replace_index_cache=True, final_boundary_ms=None: ( captured.append(records), index_cache_replacements.append(replace_index_cache), ), @@ -84,7 +85,7 @@ def test_custom_provider_fetch_appends_index_records(monkeypatch): assert "600519.SH" in symbols and "000001.SH" in symbols and "399001.SZ" in symbols assert replacements == [True] # 请求清单 = 核心四只 (无指数监控规则时) - assert provider.index_calls == [sorted(CORE_INDEX_SYMBOLS)] + assert provider.index_calls == [sorted(set(CORE_INDEX_SYMBOLS) | BENCHMARK_INDEX_SYMBOLS)] def test_custom_provider_monitor_indices_join_fetch(monkeypatch): @@ -102,7 +103,7 @@ def test_custom_provider_monitor_indices_join_fetch(monkeypatch): service._app_state = SimpleNamespace(monitor_engine=_Engine()) service._fetch_full_market_quotes() - assert provider.index_calls == [sorted(set(CORE_INDEX_SYMBOLS) | {"000300.SH"})] + assert provider.index_calls == [sorted(set(CORE_INDEX_SYMBOLS) | BENCHMARK_INDEX_SYMBOLS | {"000300.SH"})] def test_custom_provider_without_indices_protocol_is_silent(monkeypatch): diff --git a/backend/tests/test_custom_signals_ai.py b/backend/tests/test_custom_signals_ai.py index 6757402..4bad19f 100644 --- a/backend/tests/test_custom_signals_ai.py +++ b/backend/tests/test_custom_signals_ai.py @@ -42,16 +42,54 @@ def test_build_messages_contains_whitelist_fields_and_rules(): def test_build_messages_only_contains_whitelisted_fields(): - from app.strategy.custom_signals import ALLOWED_FIELDS + from app.strategy.custom_signals import allowed_fields system = build_messages("x")[0]["content"] # 只检查「字段清单」段落(可用字段 … 运算符),排除 JSON 格式示例里的 "name" field_section = system.split("运算符(op)", 1)[0] for field in ("ma20", "rsi_14", "boll_upper"): assert field in field_section - # 字段清单里出现的每个 key( 都必须在白名单内 + # 字段清单里出现的每个 key( 都必须在白名单内 (物理列 ∪ 因子) keys = set(re.findall(r"([a-z0-9_]+)\(", field_section)) - assert keys and keys <= ALLOWED_FIELDS + assert keys and keys <= allowed_fields() + + +def test_build_messages_contains_factor_fields(): + # 因子注册表字段应进入提示词 (因子·分组), 供 AI 直接构造因子条件 + from app.factors.registry import all_factors + + system = build_messages("动量强的票")[0]["content"] + field_section = system.split("运算符(op)", 1)[0] + factor_ids = {s.id for s in all_factors()} + listed = set(re.findall(r"([a-z0-9_]+)\(", field_section)) + assert factor_ids & listed, "提示词应包含至少一个因子字段" + assert "因子·动量" in field_section + assert "预计算的因子值" in system + + +def test_parse_and_validate_accepts_factor_condition(): + # AI 输出以因子为条件字段: 应通过白名单校验 + raw = json.dumps({ + "name": "强动量", + "conditions": [ + {"left": "momentum_20d", "op": ">=", "right": "0.1", + "leftDays": 0, "rightDays": 0} + ], + }) + result = parse_and_validate(raw) + assert result["conditions"][0]["left"] == "momentum_20d" + + +def test_parse_and_validate_accepts_bare_factor_rhs(): + # 右值裸写因子名: 同样自动补 field: 前缀 + raw = json.dumps({ + "name": "动量走强", + "conditions": [ + {"left": "momentum_5d", "op": ">", "right": "momentum_20d"} + ], + }) + result = parse_and_validate(raw) + assert result["conditions"][0]["right"] == "field:momentum_20d" # ── parse_and_validate ────────────────────────────────────── diff --git a/backend/tests/test_custom_signals_factor_conditions.py b/backend/tests/test_custom_signals_factor_conditions.py new file mode 100644 index 0000000..18b0456 --- /dev/null +++ b/backend/tests/test_custom_signals_factor_conditions.py @@ -0,0 +1,92 @@ +"""自定义信号因子条件 — 字段白名单动态化 + 因子列物化链路。 + +因子接入规则 (P3): 条件可引用注册表因子 (虚拟/自定义/复合), 历史路径由 +materialize_factor_columns 复用评分物化管线补算, 与检验/评分同一条计算逻辑。 +""" +from __future__ import annotations + +from datetime import date, timedelta + +import polars as pl +import pytest + +from app.strategy import custom_signals + + +def _frame(n_days: int = 30) -> pl.DataFrame: + """两标的收盘价缓涨; ma20 手工预置为 0.9 倍滚动均值 (保证乖离恒为正)。""" + rows = [] + end = date(2026, 8, 31) + for symbol_id, base in (("A", 100.0), ("B", 50.0)): + for i in range(n_days): + rows.append({ + "symbol": symbol_id, + "date": end - timedelta(days=n_days - 1 - i), + "close": base * (1.0 + 0.001 * i), + }) + df = pl.DataFrame(rows).sort(["symbol", "date"]) + return df.with_columns( + (pl.col("close").rolling_mean(20).over("symbol") * 0.9).alias("ma20") + ) + + +def test_allowed_fields_union_registry() -> None: + allowed = custom_signals.allowed_fields() + assert "close" in allowed # 物化白名单保留 + assert "ma20_bias" in allowed # 虚拟因子 + assert "turnover_z_60d" in allowed # 虚拟因子 (61 日预热) + assert "momentum_20d" in allowed # 既是白名单列也是基础因子 + assert "nope_col" not in allowed + # 静态白名单不受污染 (供 monitor_rules 等仍按物化列口径使用) + assert "ma20_bias" not in custom_signals.ALLOWED_FIELDS + + +def test_validate_accepts_and_rejects_factor_fields() -> None: + ok = { + "id": "t_bias_low", "name": "乖离超卖", "kind": "entry", + "conditions": [{"left": "ma20_bias", "op": "<", "right": "-0.05", "leftDays": 0, "rightDays": 0}], + } + custom_signals.validate(ok) # 不抛错即通过 + bad = { + "id": "t_bad", "name": "x", "kind": "entry", + "conditions": [{"left": "not_a_field", "op": "<", "right": "1"}], + } + with pytest.raises(ValueError): + custom_signals.validate(bad) + + +def test_materialize_factor_columns_and_inject() -> None: + sig = { + "id": "bias_high", "name": "乖离偏高", "kind": "entry", "enabled": True, + "conditions": [{"left": "ma20_bias", "op": ">", "right": "0", "leftDays": 0, "rightDays": 0}], + } + exprs = custom_signals.build_expressions([sig]) + col = custom_signals.column_name("bias_high") + assert col in exprs + + df = _frame() + assert "ma20_bias" not in df.columns + df2 = custom_signals.materialize_factor_columns(df, exprs) + assert "ma20_bias" in df2.columns # 复用评分物化路径补算 + + # ma20 = 0.9 x 滚动均值 → 窗口内乖离恒 > 0 + warm = df2.filter(pl.col("ma20_bias").is_not_null()) + assert warm.height > 0 + assert (warm["ma20_bias"] > 0).all() + + injected = custom_signals.inject(df2, exprs) + assert col in injected.columns + hit = injected.filter(pl.col("ma20_bias").is_not_null()) + assert hit[col].all() # 条件在窗口内全部成立 + + +def test_materialize_skips_unknown_columns() -> None: + """非注册表缺失列: 物化不处理不报错, 由 inject 缺列告警跳过。""" + sig = { + "id": "t_unknown", "name": "x", "kind": "entry", "enabled": True, + "conditions": [{"left": "not_a_field", "op": "<", "right": "1", "leftDays": 0, "rightDays": 0}], + } + exprs = custom_signals.build_expressions([sig]) # 编译不做白名单校验 (validate 负责) + df = _frame() + df2 = custom_signals.materialize_factor_columns(df, exprs) + assert df2.columns == df.columns diff --git a/backend/tests/test_data_integrity.py b/backend/tests/test_data_integrity.py index 07eebbd..30353d2 100644 --- a/backend/tests/test_data_integrity.py +++ b/backend/tests/test_data_integrity.py @@ -11,7 +11,7 @@ from types import SimpleNamespace import polars as pl import pytest -from app.market_time import CN_TZ +from app.market_time import CN_TZ, cn_today from app.services.data_integrity import ( AUTO_REPAIR_MAX_LAG_DAYS, IntegrityIssue, @@ -189,19 +189,61 @@ def test_realtime_daily_builder_drops_halted_rows_before_zero_fill(): "symbol": "600001.SH", "last_price": 10.0, "open": 9.9, "high": 10.1, "low": 9.8, "volume": 1000.0, "amount": 10000.0, - "timestamp": _ts_ms(TODAY, time(10, 0)), + "timestamp": _ts_ms(cn_today(), time(10, 0)), }, { "symbol": "600002.SH", "last_price": 20.0, "open": 0.0, "high": 0.0, "low": 0.0, "volume": 0.0, "amount": 0.0, - "timestamp": _ts_ms(TODAY, time(9, 15)), + "timestamp": _ts_ms(cn_today(), time(9, 15)), }, ]) assert result["symbol"].to_list() == ["600001.SH"] +def test_realtime_daily_builder_drops_stale_snapshot_rows(): + """回归: 停牌股快照停留旧日, 不得复制成当日假蜡烛 (301266.SZ 2026-09-04)。 + + 实时源对停牌标的返回停牌前最后一份快照 — OHLCV 全为旧日真实值, 仅 + timestamp 停在旧日。这种记录不属于当日, 必须在落盘前按 quote_ts 过滤。 + """ + from app.services.quote_service import QuoteService + + halted_since = cn_today() - timedelta(days=7) + result = QuoteService._build_daily([ + { + "symbol": "600001.SH", "last_price": 10.0, + "open": 9.9, "high": 10.1, "low": 9.8, + "volume": 1000.0, "amount": 10000.0, + "timestamp": _ts_ms(cn_today(), time(15, 0)), + }, + { + "symbol": "301266.SZ", "last_price": 24.97, + "open": 23.10, "high": 24.99, "low": 23.01, + "volume": 65038.0, "amount": 157796300.0, + "timestamp": _ts_ms(halted_since, time(15, 30)), + }, + ]) + + assert result["symbol"].to_list() == ["600001.SH"] + + +def test_realtime_daily_builder_keeps_rows_without_timestamp(): + """无时间戳的源无法判断快照新旧, 维持原行为保留 (不因缺列误删)。""" + from app.services.quote_service import QuoteService + + result = QuoteService._build_daily([ + { + "symbol": "600003.SH", "last_price": 8.0, + "open": 7.9, "high": 8.1, "low": 7.8, + "volume": 500.0, "amount": 4000.0, + }, + ]) + + assert result["symbol"].to_list() == ["600003.SH"] + + def test_halt_filter_drops_legacy_zero_volume_row_after_ohlc_fill(): from app.indicators.pipeline import filter_halt_days diff --git a/backend/tests/test_enriched_stale_price_partition.py b/backend/tests/test_enriched_stale_price_partition.py new file mode 100644 index 0000000..0557809 --- /dev/null +++ b/backend/tests/test_enriched_stale_price_partition.py @@ -0,0 +1,83 @@ +"""收盘价过期分区回归: 盘后管道必须重算"行数完整但价格停留在竞价前"的 enriched 分区。 + +实时 flush 可能在收盘集合竞价结果发布前写入当日分区 (实测 2026-09-04: TickFlow +实时端点收盘后仍返回旧价, 3392/5554 只股票 enriched 收盘价与官方日线不符), +分区行数与 daily 相同, #223 的行数校验识别不到, 增量路径不会重算, 当日偏离值/ +动量等全部 enriched 消费方都会用旧价 (海鸥住工 10 日偏离 98.77% vs 官方 99.61%)。 +_prune_stale_price_partitions 按官方日线做 raw_close vs close 值级比对, 不一致即 +删分区, 让增量重算按官方日线全市场重建。 +""" +from __future__ import annotations + +from pathlib import Path + +import polars as pl + +from app.jobs.daily_pipeline import _prune_stale_price_partitions + + +def _write_partition(base: Path, day: str, symbols: list[str], closes: list[float], col: str) -> None: + part = base / f"date={day}" + part.mkdir(parents=True, exist_ok=True) + pl.DataFrame({"symbol": symbols, col: closes}).write_parquet(part / "part.parquet") + + +def test_stale_close_partition_is_pruned(tmp_path) -> None: + daily = tmp_path / "kline_daily" + enriched = tmp_path / "kline_daily_enriched" + syms = ["002084.SZ", "600519.SH", "000001.SZ"] + # 2026-09-04: 官方日线收盘 7.10, 实时写入的 enriched 停留在竞价前 7.07; + # 行数两侧一致 (覆盖全), #223 行数校验识别不到 (issue 实测形态) + _write_partition(daily, "2026-09-04", syms, [7.10, 1330.10, 11.89], "close") + _write_partition(enriched, "2026-09-04", syms, [7.07, 1330.10, 11.89], "raw_close") + + pruned = _prune_stale_price_partitions(daily, enriched) + + assert pruned == ["2026-09-04"] + assert not (enriched / "date=2026-09-04").exists() + + +def test_matching_close_partitions_untouched(tmp_path) -> None: + daily = tmp_path / "kline_daily" + enriched = tmp_path / "kline_daily_enriched" + syms = ["002084.SZ", "600519.SH"] + _write_partition(daily, "2026-09-04", syms, [7.10, 1330.10], "close") + _write_partition(enriched, "2026-09-04", syms, [7.10, 1330.10], "raw_close") + + assert _prune_stale_price_partitions(daily, enriched) == [] + assert (enriched / "date=2026-09-04" / "part.parquet").exists() + + +def test_multiple_stale_dates_all_pruned(tmp_path) -> None: + # 管道连续数日未触发重建时, 最近多个交易日的过期分区一并修复 + daily = tmp_path / "kline_daily" + enriched = tmp_path / "kline_daily_enriched" + for day, stale in [("2026-09-04", 7.07), ("2026-09-03", 6.88), ("2026-09-02", 6.80)]: + _write_partition(daily, day, ["002084.SZ"], [stale + 0.03], "close") + _write_partition(enriched, day, ["002084.SZ"], [stale], "raw_close") + + pruned = _prune_stale_price_partitions(daily, enriched) + + assert sorted(pruned) == ["2026-09-02", "2026-09-03", "2026-09-04"] + + +def test_missing_raw_close_column_left_alone(tmp_path) -> None: + # 旧 schema 无 raw_close 列 → 读列失败, 交给既有完整性检查, 不误删 + daily = tmp_path / "kline_daily" + enriched = tmp_path / "kline_daily_enriched" + _write_partition(daily, "2026-09-04", ["002084.SZ"], [7.10], "close") + part = enriched / "date=2026-09-04" + part.mkdir(parents=True) + pl.DataFrame({"symbol": ["002084.SZ"], "close": [7.07]}).write_parquet(part / "part.parquet") + + assert _prune_stale_price_partitions(daily, enriched) == [] + assert (part / "part.parquet").exists() + + +def test_daily_partition_missing_left_alone(tmp_path) -> None: + # 官方日线尚未同步的日期不比对 (留给当日正常流程) + enriched = tmp_path / "kline_daily_enriched" + _write_partition(enriched, "2026-09-04", ["002084.SZ"], [7.07], "raw_close") + + assert _prune_stale_price_partitions(tmp_path / "kline_daily", enriched) == [] + assert (enriched / "date=2026-09-04" / "part.parquet").exists() diff --git a/backend/tests/test_factor_api.py b/backend/tests/test_factor_api.py new file mode 100644 index 0000000..b087404 --- /dev/null +++ b/backend/tests/test_factor_api.py @@ -0,0 +1,309 @@ +"""因子 API (validate / trial) 契约测试 (P2)。""" +from __future__ import annotations + +from datetime import date, timedelta + +import polars as pl +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.api.factors import router + + +@pytest.fixture() +def cleanup_registry(): + """测试注册的自定义因子在用例后注销, 不污染全局注册表 (快照测试依赖 77 基线)。""" + created: set[str] = set() + yield created + from app.factors.registry import unregister_factor + + for fid in created: + unregister_factor(fid) + + +class _FakeEngine: + """合成面板: 3 只股票日收益固定 1%/2%/3%, 任何按价格排序的因子 IC 恒为 1。""" + + def __init__(self, n_days: int = 40) -> None: + rows = [] + end = date.today() + for index in range(n_days): + day = end - timedelta(days=n_days - 1 - index) + for symbol_id, daily_return in (("A", 0.01), ("B", 0.02), ("C", 0.03)): + # 正基数且增速同序: C 永远最高价且回报最高 → 按价格排序的因子 IC 恒为 1 + rows.append({ + "symbol": symbol_id, + "date": day, + "close": (1.0 + daily_return) ** index * 10.0 * (ord(symbol_id) - ord("A") + 1), + }) + self.panel = pl.DataFrame(rows).sort(["symbol", "date"]) + + def load_panel(self, symbols, start, end, *, columns=None, asset_type="stock", **_kwargs): + frame = self.panel + if columns is not None: + for column in columns: + if column not in frame.columns: + frame = frame.with_columns(pl.lit(None).cast(pl.Float64).alias(column)) + frame = frame.select(columns) + return frame.filter((pl.col("date") >= start) & (pl.col("date") <= end)) + + +def _client(with_engine: bool = False) -> TestClient: + app = FastAPI() + app.include_router(router) + if with_engine: + app.state.backtest_engine = _FakeEngine() + return TestClient(app) + + +def test_validate_ok_formula() -> None: + response = _client().post("/api/factors/validate", json={"formula": "rank(-ts_sum(change_pct, 5))"}) + assert response.status_code == 200 + payload = response.json() + assert payload["ok"] is True + assert payload["errors"] == [] + assert payload["dependencies"] == ["change_pct"] + assert payload["warmup_bars"] == 6 + assert payload["cross_sectional"] is True + + +def test_validate_future_function_rejected() -> None: + response = _client().post("/api/factors/validate", json={"formula": "rank(ts_delta(close, -5))"}) + assert response.status_code == 200 + payload = response.json() + assert payload["ok"] is False + assert payload["errors"][0]["code"] == "E005" + assert "position" in payload["errors"][0] + + +def test_trial_golden_ic() -> None: + response = _client(with_engine=True).post( + "/api/factors/trial", json={"formula": "close", "days": 30}, + ) + assert response.status_code == 200 + payload = response.json() + assert payload["ok"] is True + assert payload["n_dates"] == 30 # 40 日面板, 首日无前收 → 39 个可算截面, 取最近 30 + assert payload["ic_mean"] == pytest.approx(1.0, abs=1e-9) + assert payload["ic_win_rate"] == pytest.approx(1.0, abs=1e-9) + # 恒定 IC 序列 std=0, IR 无定义 → None (除零保护) + assert payload["ic_std"] in (None, 0.0) + assert payload["ir"] is None + assert len(payload["ic_series"]) == 30 + + +def test_trial_compile_failure_400() -> None: + response = _client(with_engine=True).post( + "/api/factors/trial", json={"formula": "nope_col + 1", "days": 30}, + ) + assert response.status_code == 400 + body = response.json()["detail"] + assert body["errors"][0]["code"] == "E001" + + +def test_trial_computes_virtual_factor_via_shared_path() -> None: + # 引用虚拟因子 ma20_bias: 试算端点复用 _compute_missing_factors 补算路径 + # (compute_indicators 算 ma20 + materialize 物化 bias), 3 列粗面板即可出结果 + response = _client(with_engine=True).post( + "/api/factors/trial", json={"formula": "ma20_bias", "days": 20}, + ) + assert response.status_code == 200 + payload = response.json() + assert payload["ok"] is True + assert payload["n_dates"] == 20 + assert payload["ic_mean"] is not None + + +def test_group_and_status_update_after_registry_load(tmp_path, cleanup_registry) -> None: + """改分组/状态在因子已注册 (启动加载后) 的真实路径下可用。 + + 回归保护: 同版本直接 register 会被注册表拒绝 ("版本未提升"), + 端点必须先注销再按新元数据注册。 + """ + from pathlib import Path + from types import SimpleNamespace + + from app.factors.registry import get_factor + + app = FastAPI() + app.include_router(router) + app.state.backtest_engine = _FakeEngine() + app.state.repo = SimpleNamespace(store=SimpleNamespace(data_dir=Path(tmp_path))) + client = TestClient(app) + + created = client.post("/api/factors/custom", json={ + "id": "uf_group_test", "label": "分组测试", "formula": "rank(-ts_sum(change_pct, 5))", + }) + assert created.status_code == 200 + factor_id = created.json()["id"] + cleanup_registry.add(factor_id) + + from app.factors import store + + store.load_into_registry(Path(tmp_path)) # 模拟重启后的注册状态 + registered = get_factor(factor_id) + assert registered is not None and registered.group == "自定义" + + renamed = client.post(f"/api/factors/custom/{factor_id}/group", json={"group": "我的动量组"}) + assert renamed.status_code == 200 + assert renamed.json()["group"] == "我的动量组" + refreshed = get_factor(factor_id) + assert refreshed is not None and refreshed.group == "我的动量组" # 注册表同步 + on_disk = next(d for d in store.load_all(Path(tmp_path)) if d["id"] == factor_id) + assert on_disk["group"] == "我的动量组" # 磁盘持久化 + + activated = client.post(f"/api/factors/custom/{factor_id}/status", json={"status": "active"}) + assert activated.status_code == 200 # 修复前: 400 "版本未提升" + assert get_factor(factor_id).stability == "stable" + + bad = client.post(f"/api/factors/custom/{factor_id}/group", json={"group": " "}) + assert bad.status_code == 400 # 空白分组名 fail-closed + + +def test_update_custom_factor_bumps_version(tmp_path, cleanup_registry) -> None: + """编辑已有自定义因子: 版本提升注册 + 公式变化回 draft + 试算门禁。""" + from pathlib import Path + from types import SimpleNamespace + + from app.factors import store + from app.factors.registry import get_factor + + cleanup_registry.add("uf_edit_test") + app = FastAPI() + app.include_router(router) + app.state.backtest_engine = _FakeEngine() + app.state.repo = SimpleNamespace(store=SimpleNamespace(data_dir=Path(tmp_path))) + client = TestClient(app) + data_dir = Path(tmp_path) + + created = client.post("/api/factors/custom", json={ + "id": "uf_edit_test", "label": "编辑测试", "formula": "rank(-ts_sum(change_pct, 5))", + }) + assert created.status_code == 200 + assert created.json()["version"] == 1 + + # 激活后再编辑: 公式变化 → 新版本 + 回 draft (生命周期语义) + activated = client.post("/api/factors/custom/uf_edit_test/status", json={"status": "active"}) + assert activated.status_code == 200 + + updated = client.post("/api/factors/custom/uf_edit_test/update", json={ + "label": "编辑测试v2", "group": "新分组", "formula": "rank(-ts_sum(change_pct, 10))", + "description": "窗口从 5 改 10", "direction": "low", + }) + assert updated.status_code == 200 + body = updated.json() + assert body["version"] == 2 and body["status"] == "draft" + + spec = get_factor("uf_edit_test") + assert spec is not None + assert spec.version == 2 and spec.group == "新分组" and spec.label == "编辑测试v2" + assert "change_pct" in spec.dependencies + on_disk = next(d for d in store.load_all(data_dir) if d["id"] == "uf_edit_test") + assert on_disk["version"] == 2 and on_disk["status"] == "draft" + + # 仅改元数据 (公式不变): 版本仍提升, 状态保留 (不回 draft) + client.post("/api/factors/custom/uf_edit_test/status", json={"status": "active"}) + meta = client.post("/api/factors/custom/uf_edit_test/update", json={ + "label": "仅改名字", "group": "新分组", "formula": "rank(-ts_sum(change_pct, 10))", + }) + assert meta.status_code == 200 + assert meta.json() == {"ok": True, "id": "uf_edit_test", "version": 3, "status": "active"} + + missing = client.post("/api/factors/custom/uf_nope/update", json={ + "label": "x", "formula": "close", + }) + assert missing.status_code == 404 + + +def test_trial_response_includes_newey_west_t() -> None: + response = _client(with_engine=True).post( + "/api/factors/trial", json={"formula": "close", "days": 30}, + ) + assert response.status_code == 200 + payload = response.json() + assert payload["n_dates"] == 30 + assert "t_newey_west" in payload # 恒定 IC 序列下可为 None, 但字段必须存在 + + +def test_delete_custom_and_composite_factor(tmp_path, cleanup_registry) -> None: + """删除契约: 复合因子可直接删; 成员被复合引用时 409 列引用方, force 才放行。""" + from pathlib import Path + from types import SimpleNamespace + + from app.factors import store + from app.factors.registry import get_factor + + app = FastAPI() + app.include_router(router) + app.state.backtest_engine = _FakeEngine() # 创建门禁需试算 + app.state.repo = SimpleNamespace(store=SimpleNamespace(data_dir=Path(tmp_path))) + client = TestClient(app) + data_dir = Path(tmp_path) + + client.post("/api/factors/custom", json={ + "id": "uf_del_member", "label": "被引用成员", "formula": "rank(-ts_sum(change_pct, 5))", + }) + cleanup_registry.add("uf_del_member") + created = client.post("/api/factors/composite", json={ + "id": "cf_del_test", "label": "删除测试组合", "members": {"uf_del_member": 1.0, "momentum_10d": -0.5}, + }) + assert created.status_code == 200 + cleanup_registry.add("cf_del_test") + + # 404: 不存在的因子 + assert client.delete("/api/factors/custom/cf_nope").status_code == 404 + + # 409: 成员被复合因子引用 → fail-closed, 返回引用方列表 + blocked = client.delete("/api/factors/custom/uf_del_member") + assert blocked.status_code == 409 + detail = blocked.json()["detail"] + assert "cf_del_test" in str(detail["references"]) + assert get_factor("uf_del_member") is not None # 引用未解除, 因子仍在 + + # 复合因子本身可直接删除 (无人引用它) + removed = client.delete("/api/factors/custom/cf_del_test") + assert removed.status_code == 200 + assert removed.json() == {"ok": True, "id": "cf_del_test", "removed_references": []} + assert get_factor("cf_del_test") is None + assert all(d["id"] != "cf_del_test" for d in store.load_all(data_dir)) # 磁盘已删 + + # 引用解除后成员可正常删除 + freed = client.delete("/api/factors/custom/uf_del_member") + assert freed.status_code == 200 + assert get_factor("uf_del_member") is None + + +def test_delete_with_strategy_reference_requires_force(tmp_path, cleanup_registry) -> None: + """策略文件引用同样拦截: 409 列出 strategies/*.json, force=true 放行。""" + import json + from pathlib import Path + from types import SimpleNamespace + + from app.factors.registry import get_factor + + app = FastAPI() + app.include_router(router) + app.state.backtest_engine = _FakeEngine() # 创建门禁需试算 + app.state.repo = SimpleNamespace(store=SimpleNamespace(data_dir=Path(tmp_path))) + client = TestClient(app) + data_dir = Path(tmp_path) + + client.post("/api/factors/custom", json={ + "id": "uf_strat_ref", "label": "策略引用", "formula": "rank(-ts_sum(change_pct, 5))", + }) + cleanup_registry.add("uf_strat_ref") + strategies_dir = data_dir / "strategies" + strategies_dir.mkdir() + (strategies_dir / "my_strategy.json").write_text( + json.dumps({"name": "my_strategy", "factors": {"uf_strat_ref": 1.0}}), encoding="utf-8", + ) + + blocked = client.delete("/api/factors/custom/uf_strat_ref") + assert blocked.status_code == 409 + assert blocked.json()["detail"]["references"] == ["strategies/my_strategy.json"] + + forced = client.delete("/api/factors/custom/uf_strat_ref?force=true") + assert forced.status_code == 200 + assert forced.json()["removed_references"] == ["strategies/my_strategy.json"] + assert get_factor("uf_strat_ref") is None diff --git a/backend/tests/test_factor_attribution.py b/backend/tests/test_factor_attribution.py new file mode 100644 index 0000000..f8f71fe --- /dev/null +++ b/backend/tests/test_factor_attribution.py @@ -0,0 +1,80 @@ +"""回测因子归因 (v1) — _factor_attribution_summary 单元测试。 + +覆盖: + - 盈利/亏损单因子均值、样本数计算 + - snapshot 日期列 date/str 两种 dtype 均可关联 + - entry_signal_date 缺失时回退 entry_date + - 无可关联行 / 空成交 / 无因子列 → None (fail-open) +""" +from __future__ import annotations + +from datetime import date +from types import SimpleNamespace + +import polars as pl + +from app.backtest.strategy import _factor_attribution_summary + + +def _trade(symbol: str, day: str, pnl: float): + return SimpleNamespace( + symbol=symbol, + entry_signal_date=day, + entry_date=day, + pnl_pct=pnl, + ) + + +def _snapshot(dates_as: str = "str") -> pl.DataFrame: + frame = pl.DataFrame({ + "symbol": ["000001", "000002", "000003", "000004"], + "date": ["2026-01-05"] * 4, + "momentum_20d": [0.10, -0.05, 0.20, 0.00], + "turnover_rate": [5.0, 8.0, 6.0, 7.0], + }) + if dates_as == "date": + frame = frame.with_columns(pl.col("date").str.to_date()) + return frame + + +def test_summary_win_lose_means(): + trades = [ + _trade("000001", "2026-01-05", 0.10), # 盈利: mom 0.10, to 5 + _trade("000003", "2026-01-05", 0.05), # 盈利: mom 0.20, to 6 + _trade("000002", "2026-01-05", -0.03), # 亏损: mom -0.05, to 8 + _trade("000004", "2026-01-05", -0.08), # 亏损: mom 0.00, to 7 + ] + result = _factor_attribution_summary(_snapshot(), trades) + assert result is not None + assert result["n_win"] == 2 and result["n_lose"] == 2 + by_factor = {f["factor"]: f for f in result["factors"]} + assert by_factor["momentum_20d"]["win_mean"] == round((0.10 + 0.20) / 2, 6) + assert by_factor["momentum_20d"]["lose_mean"] == round((-0.05 + 0.00) / 2, 6) + assert by_factor["turnover_rate"]["win_mean"] == 5.5 + assert by_factor["turnover_rate"]["lose_n"] == 2 + + +def test_summary_accepts_date_dtype_snapshot(): + trades = [_trade("000001", "2026-01-05", 0.1), _trade("000002", "2026-01-05", -0.1)] + result = _factor_attribution_summary(_snapshot(dates_as="date"), trades) + assert result is not None + assert result["n_win"] == 1 + + +def test_summary_falls_back_to_entry_date(): + trade = SimpleNamespace(symbol="000001", entry_signal_date=None, + entry_date=date(2026, 1, 5), pnl_pct=0.2) + result = _factor_attribution_summary(_snapshot(), [trade]) + assert result is not None + assert result["factors"][0]["win_n"] == 1 + + +def test_summary_returns_none_when_no_overlap(): + trades = [_trade("600000", "2026-02-10", 0.1)] # 不在快照里 + assert _factor_attribution_summary(_snapshot(), trades) is None + + +def test_summary_returns_none_on_empty_inputs(): + assert _factor_attribution_summary(_snapshot(), []) is None + no_factor = pl.DataFrame({"symbol": ["000001"], "date": ["2026-01-05"]}) + assert _factor_attribution_summary(no_factor, [_trade("000001", "2026-01-05", 0.1)]) is None diff --git a/backend/tests/test_factor_dsl.py b/backend/tests/test_factor_dsl.py new file mode 100644 index 0000000..d2cda03 --- /dev/null +++ b/backend/tests/test_factor_dsl.py @@ -0,0 +1,209 @@ +"""因子公式 DSL 编译器测试 (P2)。 + +覆盖: 全部编译期错误码 E001-E008/E010/E011/E014/E016、窗口纪律 (只向后看)、 +数值正确性 (与手算基准对拍)、依赖/预热推导、运行期 fail-closed。 +""" +from __future__ import annotations + +from datetime import date + +import polars as pl +import pytest + +from app.factors.dsl import BASE_COLUMNS, FACTOR_COLUMN, compile_formula + + +def _panel() -> pl.DataFrame: + # 两个 symbol x 6 日, 便于验证 over("symbol") 不串组 + rows = [] + for symbol, closes in (("A", [10.0, 11.0, 12.0, 13.0, 14.0, 15.0]), + ("B", [100.0, 90.0, 80.0, 70.0, 60.0, 50.0])): + for index, close in enumerate(closes): + rows.append({ + "symbol": symbol, + "date": date(2026, 1, index + 1), + "close": close, + "volume": 1000.0 + index * 100, + "amount": (1000.0 + index * 100) * close, + }) + return pl.DataFrame(rows).sort(["symbol", "date"]) + + +def _eval(formula: str, panel: pl.DataFrame | None = None) -> pl.Series: + compiled = compile_formula(formula) + assert compiled.ok, [error.to_dict() for error in compiled.errors] + assert compiled.frame_transform is not None + base = panel if panel is not None else _panel() + frame = compiled.frame_transform(base) + assert frame is not None + return frame[FACTOR_COLUMN] + + +# ------------------------------------------------------------- 错误码覆盖 + +@pytest.mark.parametrize("formula, code", [ + ("clos + 1", "E001"), # 未知标识符 + ("foo(close)", "E002"), # 未知函数 + ("ts_mean(close)", "E003"), # 缺窗口参数 + ("ts_mean(close, close)", "E003"), # 窗口参数必须是常量 + ("ts_mean(close, 1)", "E004"), # 窗口 < 2 + ("ts_mean(close, 600)", "E004"), # 窗口 > 512 + ("ts_quantile(close, 5, 1.5)", "E004"), # q 不在 (0,1) + ("ts_delay(close, -5)", "E005"), # 负 shift = 未来函数 + ("ts_delta(close, -1)", "E005"), + ("power(close, 5)", "E010"), # 指数越界 + ("winsorize(close, 9)", "E011"), # k 越界 + ("close / 0", "E008"), # 静态除零 + ("1 + 2", "E016"), # 常量表达式 + ("close +", "E014"), # 语法错误 + ("(close", "E014"), + ("close $ 1", "E014"), + ("", "E014"), +]) +def test_error_codes(formula: str, code: str) -> None: + compiled = compile_formula(formula) + assert not compiled.ok + assert any(error.code == code for error in compiled.errors), [e.code for e in compiled.errors] + + +def test_depth_and_token_limits() -> None: + # AST 深度: 嵌套 13 层二元运算 (每层 bin 算 1) 超限 + deep = "close" + for _ in range(13): + deep = f"({deep} + 1)" + compiled = compile_formula(deep) + assert not compiled.ok + assert any(error.code == "E006" for error in compiled.errors) + + wide = " + ".join(["close"] * 120) # 二元链深度为 2, 但 token 超 200 + compiled = compile_formula(wide) + assert any(error.code == "E007" for error in compiled.errors) + + +def test_error_payload_shape() -> None: + compiled = compile_formula("rank(ts_delta(close, -5))") + assert not compiled.ok + payload = compiled.errors[0].to_dict() + assert payload["code"] == "E005" + assert payload["message"] + assert "position" in payload and "offset" in payload["position"] + + +# ------------------------------------------------------------- 依赖与预热 + +def test_dependencies_and_warmup() -> None: + compiled = compile_formula("rank(-ts_sum(change_pct, 5))") + assert compiled.ok + # change_pct 是 base 因子, 依赖为其自身列 + assert compiled.dependencies == frozenset({"change_pct"}) + assert compiled.referenced_factors == frozenset({"change_pct"}) + assert compiled.warmup_bars == 6 # ts 窗口 5 + 1 + assert compiled.cross_sectional + + compiled = compile_formula("close + ma20_bias") + assert compiled.ok + assert compiled.dependencies == frozenset({"close", "ma20"}) + assert compiled.referenced_factors == frozenset({"ma20_bias"}) + + compiled = compile_formula("turnover_z_60d * 2") + assert compiled.ok + assert compiled.warmup_bars == 61 # 引用因子 warmup 传递 + + +def test_base_columns_contract() -> None: + assert "close" in BASE_COLUMNS + assert "clos" not in BASE_COLUMNS + + +# ------------------------------------------------------------- 数值正确性 + +def test_ts_delay_backward_only() -> None: + values = _eval("ts_delay(close, 2)") + # A 组: [null, null, 10, 11, 12, 13]; B 组: [null, null, 100, 90, 80, 70] + assert values[:6].to_list() == [None, None, 10.0, 11.0, 12.0, 13.0] + assert values[6:].to_list() == [None, None, 100.0, 90.0, 80.0, 70.0] + + +def test_ts_mean_no_cross_symbol_leak() -> None: + values = _eval("ts_mean(close, 2)") + # A 组 2 日窗: [null, 10.5, 11.5, 12.5, 13.5, 14.5] + assert values[:6].to_list() == [None, 10.5, 11.5, 12.5, 13.5, 14.5] + # B 组边界: 第一行是 null (窗口不满) 而不是拿到 A 组尾部; 第二行 95.0 + assert values[6] is None + assert values[7] == 95.0 + + +def test_rank_cross_sectional() -> None: + values = _eval("rank(close)") + frame = _panel().with_columns(pl.Series("_f", values)) + per_date = frame.filter(pl.col("date") == date(2026, 1, 1)) + # 第一日 A=10, B=100: rank(A) < rank(B), 且都 ∈ (0,1] + ranks = dict(zip(per_date["symbol"].to_list(), per_date["_f"].to_list(), strict=True)) + assert 0 < ranks["A"] < ranks["B"] <= 1.0 + + +def test_cross_of_timeseries_two_phase() -> None: + # Polars 嵌套窗口会静默全 null; 编译器必须走两阶段 (临时列 + 截面) + values = _eval("rank(ts_sum(close, 2))") + assert sum(1 for value in values.to_list() if value is not None) > 0 + # B 组滚动和数值恒大于 A 组 (百元 vs 十元价位) → 每个 (非首行) 日期 rank(B) > rank(A) + frame = _panel().with_columns(pl.Series("_f", values)) + for day in (date(2026, 1, 3), date(2026, 1, 6)): + day_frame = frame.filter(pl.col("date") == day) + ranks = dict(zip(day_frame["symbol"].to_list(), day_frame["_f"].to_list(), strict=True)) + assert ranks["B"] > ranks["A"] + + +def test_cross_in_timeseries_rejected() -> None: + compiled = compile_formula("ts_mean(rank(close), 5)") + assert not compiled.ok + assert any(error.code == "E009" for error in compiled.errors) + + +def test_safe_division_yields_null() -> None: + # 分母为动态表达式的恒 0: 静态折叠不报 E008, 运行期安全除产出 null + values = _eval("close / (close - close)") + assert all(value is None for value in values.to_list()) + + +def test_if_else_and_comparison() -> None: + values = _eval("if_else(close > 12, 1, 0)") + assert values[:6].to_list() == [0.0, 0.0, 0.0, 1.0, 1.0, 1.0] + + +def test_arithmetic_precedence() -> None: + values = _eval("close + 2 * 3") + assert values[0] == 16.0 # 10 + 6, 而不是 (10+2)*3 + + +def test_decay_linear_weights_recent() -> None: + values = _eval("decay_linear(close, 3)") + # A 组第 3 行: (3*12 + 2*11 + 1*10) / 6 = 68/6 + assert values[2] == pytest.approx((3 * 12 + 2 * 11 + 1 * 10) / 6) + + +def test_log_domain_guard() -> None: + import math + + panel = _panel().with_columns((pl.col("close") - 15.0).alias("neg")) + values = _eval("log(close - 15)", panel) + # A 组全为负 → null; B 组 100-15=85 → log 正常 + assert all(value is None for value in values[:6].to_list()) + assert values[6] == pytest.approx(math.log(85.0)) + + +def test_ts_corr_two_series() -> None: + values = _eval("ts_corr(close, volume, 3)") + # 常数序列或完全单调: 只验证产出为有限值或 null, 无串组异常即可 + assert len(values) == 12 + + +# --------------------------------------------------------- 运行期 fail-closed + +def test_runtime_missing_column_fails_closed() -> None: + compiled = compile_formula("close * volume") + assert compiled.ok + frame_without_volume = _panel().drop("volume") + assert compiled.frame_transform is not None + assert compiled.frame_transform(frame_without_volume) is None # E013 语义 + assert compiled.frame_transform(_panel()) is not None diff --git a/backend/tests/test_factor_expansion.py b/backend/tests/test_factor_expansion.py new file mode 100644 index 0000000..959758b --- /dev/null +++ b/backend/tests/test_factor_expansion.py @@ -0,0 +1,174 @@ +"""扩充批次 (2026-09-05) 16 个新虚拟因子的数值正确性测试。 + +合成单标的日频面板, 黄金值由 numpy 独立重算 (不复制实现), +覆盖: 公式口径 / 无前视 (min_samples) / 除零 fail-closed / 列代数型因子。 +""" +from __future__ import annotations + +from datetime import date, timedelta + +import numpy as np +import polars as pl +import pytest + +from app.strategy.scoring import materialize_scoring_columns + +N_DAYS = 250 +DATES = [date(2025, 1, 1) + timedelta(days=i) for i in range(N_DAYS)] +T = np.arange(N_DAYS, dtype=float) + +# 自然波动收益序列 (趋势 + 正弦): 避免等比价格的常数收益让波动率退化为浮点噪声 +DAILY_RET = 0.002 + 0.01 * np.sin(T / 9.0) +CLOSE = 100.0 * np.cumprod(1.0 + DAILY_RET) +OPEN = np.roll(CLOSE, 1) * 1.005 +OPEN[0] = 99.0 * 1.005 # 隔夜跳空 +0.5% +PREV_CLOSE = np.roll(CLOSE, 1) +PREV_CLOSE[0] = 99.0 +VOLUME = 1_000_000 + 500.0 * T # 量能缓增 +TURNOVER = VOLUME / 200_000_000.0 # 流通股本 2 亿股 +RET = np.concatenate([[np.nan], CLOSE[1:] / CLOSE[:-1] - 1.0]) + +# 列代数型因子的依赖列直接给黄金友好值 +RSI = 50.0 + 10.0 * np.sin(T / 7.0) +MOM20 = np.concatenate([np.full(20, np.nan), CLOSE[20:] / CLOSE[:-20] - 1.0]) +MOM60 = np.concatenate([np.full(60, np.nan), CLOSE[60:] / CLOSE[:-60] - 1.0]) +KDJ_K = 50.0 + 15.0 * np.cos(T / 11.0) +KDJ_D = 50.0 + 5.0 * np.sin(T / 13.0) +AMPLITUDE = 0.02 + 0.0001 * T + + +def _panel() -> pl.DataFrame: + return pl.DataFrame({ + "symbol": ["TEST"] * N_DAYS, + "date": DATES, + "open": OPEN, + "high": np.maximum(OPEN, CLOSE) * 1.01, + "low": np.minimum(OPEN, CLOSE) * 0.99, + "close": CLOSE, + "prev_close": PREV_CLOSE, + "volume": VOLUME, + "amount": VOLUME * CLOSE * 100.0, + "turnover_rate": TURNOVER, + "rsi_14": RSI, + "momentum_20d": MOM20, + "momentum_60d": MOM60, + "kdj_k": KDJ_K, + "kdj_d": KDJ_D, + "amplitude": AMPLITUDE, + }) + + +def _col(frame: pl.DataFrame, name: str) -> np.ndarray: + return frame[name].to_numpy() + + +def _materialize(names: list[str]) -> pl.DataFrame: + return materialize_scoring_columns(_panel(), names) + + +def test_momentum_120d_golden() -> None: + frame = _materialize(["momentum_120d"]) + got = _col(frame, "momentum_120d") + golden = np.full(N_DAYS, np.nan) + golden[120:] = CLOSE[120:] / CLOSE[:-120] - 1.0 + assert np.allclose(got[121:], golden[121:], atol=1e-12) + assert np.isnan(got[:120]).all() # 无前视: 前 120 根必为空 (min_samples) + + +def test_mom_accel_and_kdj_diff_column_algebra() -> None: + frame = _materialize(["mom_accel_20_60", "kdj_kd_diff", "rsi_14_delta_5d"]) + assert np.allclose(_col(frame, "mom_accel_20_60"), MOM20 - MOM60, equal_nan=True) + assert np.allclose(_col(frame, "kdj_kd_diff"), KDJ_K - KDJ_D, atol=1e-12) + delta = np.full(N_DAYS, np.nan) + delta[5:] = RSI[5:] - RSI[:-5] + assert np.allclose(_col(frame, "rsi_14_delta_5d")[6:], delta[6:], atol=1e-12) + + +def test_overnight_and_intraday_decomposition() -> None: + frame = _materialize(["overnight_ret_20d", "intraday_ret_20d"]) + overnight_daily = OPEN / PREV_CLOSE - 1.0 + intraday_daily = CLOSE / OPEN - 1.0 + # 后向滚动窗: got[i] = sum(daily[i-19 .. i]); convolve('valid')[k] = sum(daily[k..k+19]) + # → got[i] 对应 valid[i-19], 从 i=21 起比对 (跳过合成首日 prev_close 特例) + golden_on = np.convolve(overnight_daily, np.ones(20), "valid")[2:] + golden_in = np.convolve(intraday_daily, np.ones(20), "valid")[2:] + got_on = _col(frame, "overnight_ret_20d") + got_in = _col(frame, "intraday_ret_20d") + assert np.allclose(got_on[21:], golden_on, atol=1e-10) + assert np.allclose(got_in[21:], golden_in, atol=1e-10) + # 恒等式: sum(隔夜) + sum(日内) ≈ sum(全天收益); 精确差为每日交叉项 on*in + # (跳空0.5% x 日内~1%, 20日累计 ~1e-3), 故用 5e-3 容差 + total = got_on[21:] + got_in[21:] + golden_total = np.convolve(CLOSE / PREV_CLOSE - 1.0, np.ones(20), "valid")[2:] + assert np.allclose(total, golden_total, atol=5e-3) + + +def test_downside_vol_only_counts_negative_side() -> None: + frame = _materialize(["downside_vol_20d"]) + got = _col(frame, "downside_vol_20d")[21:] + for i, day in enumerate(range(21, N_DAYS)): + window = np.minimum(RET[day - 19: day + 1], 0.0) + golden = np.sqrt(np.mean(window ** 2)) + assert got[i] == pytest.approx(golden, abs=1e-12), f"day index {day}" + + +def test_obv_trend_bounded_and_golden() -> None: + frame = _materialize(["obv_trend_20d"]) + got = _col(frame, "obv_trend_20d") + for day in range(21, N_DAYS, 25): + window_ret = RET[day - 19: day + 1] + window_vol = VOLUME[day - 19: day + 1] + signed = np.sign(window_ret) * window_vol + golden = signed.sum() / (window_vol.mean() * 20.0) + assert got[day] == pytest.approx(golden, abs=1e-9), f"day index {day}" + valid = got[~np.isnan(got)] + assert (np.abs(valid) <= 1.0 + 1e-12).all() # 有界 [-1, 1] + + +def test_log_float_mv_golden_and_fail_closed() -> None: + frame = _materialize(["log_float_mv"]) + got = _col(frame, "log_float_mv") + golden = np.log(CLOSE * VOLUME / TURNOVER) + assert np.allclose(got, golden, atol=1e-10) # = ln(流通市值), 股本=2亿 + # 换手率为 0 → None (fail-closed, 不产生 inf) + broken = _panel().with_columns(pl.lit(0.0).alias("turnover_rate")) + out = materialize_scoring_columns(broken, ["log_float_mv"]) + assert out["log_float_mv"].is_null().all() + + +def test_position_240d_and_distance_to_high() -> None: + frame = _materialize(["position_240d", "distance_to_high_240d"]) + pos = _col(frame, "position_240d") + dist = _col(frame, "distance_to_high_240d") + for day in (241, 245, N_DAYS - 1): + window = CLOSE[day - 239: day + 1] + golden_pos = (CLOSE[day] - window.min()) / (window.max() - window.min()) + assert pos[day] == pytest.approx(golden_pos, abs=1e-12), f"pos day {day}" + assert dist[day] == pytest.approx(CLOSE[day] / window.max() - 1.0, abs=1e-12) + assert np.isnan(pos[:239]).all() # 无前视: 240 日窗在索引 239 才首次有效 + + +def test_vol_regime_amplitude_trend_turnover_stats() -> None: + frame = _materialize(["vol_regime_5_60", "amplitude_trend_20_60", "turnover_mean_20d", "turnover_std_20d"]) + vr = _col(frame, "vol_regime_5_60") + at = _col(frame, "amplitude_trend_20_60") + tm = _col(frame, "turnover_mean_20d") + ts = _col(frame, "turnover_std_20d") + for day in (61, 120, N_DAYS - 1): + fast = np.std(RET[day - 4: day + 1], ddof=1) + slow = np.std(RET[day - 59: day + 1], ddof=1) + assert vr[day] == pytest.approx(fast / slow, rel=1e-9, abs=1e-12), f"vr day {day}" + a_fast = AMPLITUDE[day - 19: day + 1].mean() + a_slow = AMPLITUDE[day - 59: day + 1].mean() + assert at[day] == pytest.approx(a_fast / a_slow - 1.0, rel=1e-9, abs=1e-12) + t_window = TURNOVER[day - 19: day + 1] + assert tm[day] == pytest.approx(t_window.mean(), rel=1e-12) + assert ts[day] == pytest.approx(t_window.std(ddof=1) / t_window.mean(), rel=1e-9) + + +def test_amount_mean_20d_unit_is_yi() -> None: + frame = _materialize(["amount_mean_20d"]) + got = _col(frame, "amount_mean_20d") + day = N_DAYS - 1 + golden = (VOLUME[day - 19: day + 1] * CLOSE[day - 19: day + 1] * 100.0).mean() / 1e8 + assert got[day] == pytest.approx(golden, rel=1e-12) diff --git a/backend/tests/test_factor_registry.py b/backend/tests/test_factor_registry.py new file mode 100644 index 0000000..1203d10 --- /dev/null +++ b/backend/tests/test_factor_registry.py @@ -0,0 +1,315 @@ +"""因子注册表 (L-REG) P1 收口快照测试。 + +黄金数据为收口前 factor.py / scoring.py 的字面量副本。 +任何目录漂移 (id/label/group/desc/顺序/依赖/预热) 都必须在改动前更新这里的黄金数据, +保证历史候选方案引用的因子 id 与挖掘调度顺序 (FACTOR_COLUMNS[:48]) 不受影响。 +""" +from __future__ import annotations + +import pytest + +from app.factors.registry import ( + FactorSpec, + all_factors, + factor_columns_view, + factor_dependencies, + get_factor, + register_factor, + scoring_warmups, + virtual_dependencies, +) + +# --- 黄金数据: 收口前 factor.py FACTOR_COLUMNS 原文 --- +GOLDEN_COLUMNS: list[dict] = [ + {"id": "momentum_5d", "label": "5日动量", "group": "动量", "desc": "5个交易日累计收益率"}, + {"id": "momentum_10d", "label": "10日动量", "group": "动量", "desc": "10个交易日累计收益率"}, + {"id": "momentum_20d", "label": "20日动量", "group": "动量", "desc": "20个交易日累计收益率"}, + {"id": "momentum_30d", "label": "30日动量", "group": "动量", "desc": "30个交易日累计收益率"}, + {"id": "momentum_60d", "label": "60日动量", "group": "动量", "desc": "60个交易日累计收益率"}, + {"id": "change_pct", "label": "日涨跌幅", "group": "动量", "desc": "当日收盘相对前收盘的收益率"}, + {"id": "ma5_bias", "label": "MA5乖离", "group": "均线偏离", "desc": "收盘价 / MA5 - 1"}, + {"id": "ma10_bias", "label": "MA10乖离", "group": "均线偏离", "desc": "收盘价 / MA10 - 1"}, + {"id": "ma20_bias", "label": "MA20乖离", "group": "均线偏离", "desc": "收盘价 / MA20 - 1"}, + {"id": "ma30_bias", "label": "MA30乖离", "group": "均线偏离", "desc": "收盘价 / MA30 - 1"}, + {"id": "ma60_bias", "label": "MA60乖离", "group": "均线偏离", "desc": "收盘价 / MA60 - 1"}, + {"id": "ema5_bias", "label": "EMA5乖离", "group": "均线偏离", "desc": "收盘价 / EMA5 - 1"}, + {"id": "ema10_bias", "label": "EMA10乖离", "group": "均线偏离", "desc": "收盘价 / EMA10 - 1"}, + {"id": "ema20_bias", "label": "EMA20乖离", "group": "均线偏离", "desc": "收盘价 / EMA20 - 1"}, + {"id": "ema30_bias", "label": "EMA30乖离", "group": "均线偏离", "desc": "收盘价 / EMA30 - 1"}, + {"id": "ema60_bias", "label": "EMA60乖离", "group": "均线偏离", "desc": "收盘价 / EMA60 - 1"}, + {"id": "rsi_6", "label": "RSI(6)", "group": "超买超卖", "desc": "6日相对强弱指标"}, + {"id": "rsi_14", "label": "RSI(14)", "group": "超买超卖", "desc": "14日相对强弱指标"}, + {"id": "rsi_24", "label": "RSI(24)", "group": "超买超卖", "desc": "24日相对强弱指标"}, + {"id": "macd_hist", "label": "MACD柱(原值)", "group": "趋势", "desc": "兼容历史研究; 跨股票比较建议优先使用MACD柱强度"}, + {"id": "macd_dif_pct", "label": "MACD DIF强度", "group": "趋势", "desc": "MACD DIF / 收盘价"}, + {"id": "macd_dea_pct", "label": "MACD DEA强度", "group": "趋势", "desc": "MACD DEA / 收盘价"}, + {"id": "macd_hist_pct", "label": "MACD柱强度", "group": "趋势", "desc": "MACD柱 / 收盘价, 消除股价尺度影响"}, + {"id": "kdj_k", "label": "KDJ-K", "group": "趋势", "desc": "KDJ指标K值"}, + {"id": "kdj_d", "label": "KDJ-D", "group": "趋势", "desc": "KDJ指标D值"}, + {"id": "kdj_j", "label": "KDJ-J", "group": "趋势", "desc": "KDJ指标J值"}, + {"id": "boll_position", "label": "布林位置", "group": "趋势", "desc": "收盘价在布林带下轨到上轨之间的位置"}, + {"id": "annual_vol_20d", "label": "20日波动率", "group": "波动率", "desc": "20日收益率年化标准差"}, + {"id": "atr_14", "label": "ATR(14)原值", "group": "波动率", "desc": "兼容历史研究; 跨股票比较建议优先使用ATR相对波动"}, + {"id": "atr_pct", "label": "ATR相对波动", "group": "波动率", "desc": "ATR(14) / 收盘价"}, + {"id": "amplitude", "label": "日振幅", "group": "波动率", "desc": "当日高低价差 / 前收盘价"}, + {"id": "boll_width", "label": "布林带宽", "group": "波动率", "desc": "布林带上下轨宽度 / MA20"}, + {"id": "vol_ratio_5d", "label": "5日量比", "group": "量价", "desc": "当日成交量 / 前5日平均成交量"}, + {"id": "vol_ratio_10d", "label": "10日量比", "group": "量价", "desc": "当日成交量 / 前10日平均成交量"}, + {"id": "vol_trend_5_10", "label": "成交量趋势", "group": "量价", "desc": "5日平均成交量 / 10日平均成交量 - 1"}, + {"id": "turnover_rate", "label": "换手率", "group": "量价", "desc": "使用历史时点流通股本计算的当日换手率"}, + {"id": "turnover_ratio_5d", "label": "换手率放大", "group": "量价", "desc": "当日换手率 / 前5日平均换手率 - 1"}, + {"id": "log_amount", "label": "成交额对数", "group": "量价", "desc": "ln(成交额 + 1), 降低极端规模影响"}, + {"id": "amount_ratio_5d", "label": "成交额放大", "group": "量价", "desc": "当日成交额 / 前5日平均成交额 - 1"}, + {"id": "gap_return", "label": "开盘跳空", "group": "价格位置", "desc": "开盘价 / 前收盘价 - 1"}, + {"id": "intraday_return", "label": "日内收益", "group": "价格位置", "desc": "收盘价 / 开盘价 - 1"}, + {"id": "close_position", "label": "收盘位置", "group": "价格位置", "desc": "收盘价在当日最低价到最高价之间的位置"}, + {"id": "distance_to_high_60d", "label": "距60日高点", "group": "价格位置", "desc": "收盘价 / 60日最高收盘价 - 1"}, + {"id": "distance_from_low_60d", "label": "距60日低点", "group": "价格位置", "desc": "收盘价 / 60日最低收盘价 - 1"}, + {"id": "vwap_bias", "label": "VWAP乖离", "group": "价格位置", "desc": "收盘价 / 当日成交均价 - 1, 成交均价 = 成交额 / (成交量x100)"}, + {"id": "max_ret_20d", "label": "20日最大单日涨幅", "group": "收益形态", "desc": "近20个交易日单日涨幅最大值(彩票效应, 高值代表博彩型特征强)"}, + {"id": "ret_skew_20d", "label": "20日收益偏度", "group": "收益形态", "desc": "近20个交易日日收益偏度, 高值代表右偏(偶发大涨)"}, + {"id": "up_days_20d", "label": "20日上涨天数", "group": "收益形态", "desc": "近20个交易日中上涨天数(0~20)"}, + {"id": "amihud_20d", "label": "20日Amihud非流动性", "group": "流动性", "desc": "近20日平均 |日涨跌幅| / 成交额(亿元), 高值代表流动性差"}, + {"id": "turnover_z_60d", "label": "换手率60日z分", "group": "流动性", "desc": "(当日换手率 - 前60日均值) / 前60日标准差, 衡量换手异动"}, + {"id": "vol_price_corr_20d", "label": "20日量价相关", "group": "量价", "desc": "近20个交易日日涨跌幅与成交量的相关系数, 高值代表量价同向"}, + {"id": "vol_trend_5_60", "label": "量能趋势(5/60)", "group": "量价", "desc": "5日平均成交量 / 60日平均成交量 - 1"}, + {"id": "limit_up_count_20d", "label": "涨停基因(20日)", "group": "涨停基因", "desc": "近20个交易日涨停次数"}, + {"id": "limit_up_count_60d", "label": "涨停基因(60日)", "group": "涨停基因", "desc": "近60个交易日涨停次数"}, + {"id": "pb_latest", "label": "市净率(最新公告)", "group": "财务", "desc": "收盘价 / 最新已公告每股净资产; 无财务数据或公告前为空"}, + {"id": "roe_latest", "label": "ROE(最新公告)", "group": "财务", "desc": "最新已公告净资产收益率(%); 无财务数据或公告前为空"}, + {"id": "gross_margin_latest", "label": "毛利率(最新公告)", "group": "财务", "desc": "最新已公告销售毛利率(%)"}, + {"id": "net_margin_latest", "label": "净利率(最新公告)", "group": "财务", "desc": "最新已公告销售净利率(%)"}, + {"id": "revenue_yoy_latest", "label": "营收增速(最新公告)", "group": "财务", "desc": "最新已公告营业收入同比(%)"}, + {"id": "net_income_yoy_latest", "label": "净利增速(最新公告)", "group": "财务", "desc": "最新已公告归母净利润同比(%)"}, + {"id": "debt_ratio_latest", "label": "资产负债率(最新公告)", "group": "财务", "desc": "最新已公告资产负债率(%)"}, + # --- 扩充批次 (2026-09-05): 追加于目录尾部, 前 48 项挖掘调度顺序不变 --- + {"id": "log_float_mv", "label": "流通市值对数", "group": "规模", "desc": "ln(收盘价 x 当日成交量 / 换手率), 由换手率反推流通股本, 高值代表大盘"}, + {"id": "momentum_120d", "label": "120日动量", "group": "动量", "desc": "120个交易日累计收益率 (中期动量, 与短窗口互补)"}, + {"id": "mom_accel_20_60", "label": "动量加速度", "group": "动量", "desc": "20日动量 - 60日动量, 衡量近期动量相对中期是否增强"}, + {"id": "rsi_14_delta_5d", "label": "RSI五日变化", "group": "超买超卖", "desc": "RSI(14) - 5日前的RSI(14), 衡量强弱指标的边际变化"}, + {"id": "overnight_ret_20d", "label": "20日隔夜收益", "group": "收益形态", "desc": "近20日累计隔夜收益(开盘价/前收盘-1求和), A股隔夜与日内收益的定价机制不同"}, + {"id": "intraday_ret_20d", "label": "20日日内收益", "group": "收益形态", "desc": "近20日累计日内收益(收盘价/开盘价-1求和), 与隔夜收益构成收益分解"}, + {"id": "downside_vol_20d", "label": "20日下行波动", "group": "波动率", "desc": "sqrt(近20日 min(日收益,0)^2 均值), 只度量下跌侧风险"}, + {"id": "vol_regime_5_60", "label": "波动率状态(5/60)", "group": "波动率", "desc": "5日收益标准差 / 60日收益标准差, 高值代表波动骤然放大"}, + {"id": "amplitude_trend_20_60", "label": "振幅趋势(20/60)", "group": "波动率", "desc": "20日平均振幅 / 60日平均振幅 - 1"}, + {"id": "obv_trend_20d", "label": "20日量能潮", "group": "量价", "desc": "近20日 sign(日收益)x成交量 之和 / (20日均量x20), 有界[-1,1], 净买入方向的一致性"}, + {"id": "amount_mean_20d", "label": "20日均成交额(亿)", "group": "量价", "desc": "近20日平均成交额(亿元), 规模/流动性水平量"}, + {"id": "turnover_mean_20d", "label": "20日均换手", "group": "流动性", "desc": "近20日平均换手率, A股经典低换手溢价因子"}, + {"id": "turnover_std_20d", "label": "20日换手波动", "group": "流动性", "desc": "近20日换手率标准差 / 均值 (变异系数), 衡量交易活跃的稳定性"}, + {"id": "position_240d", "label": "一年价格位置", "group": "价格位置", "desc": "收盘价在近240个交易日最低价到最高价之间的位置 (0~1)"}, + {"id": "distance_to_high_240d", "label": "距一年高点", "group": "价格位置", "desc": "收盘价 / 240日最高收盘价 - 1, 接近0代表贴近一年新高"}, + {"id": "kdj_kd_diff", "label": "KDJ K-D差", "group": "趋势", "desc": "KDJ K值 - D值, 正值代表快线在慢线上方"}, +] + +GOLDEN_VIRTUAL_DEPS: dict[str, frozenset[str]] = { + **{ + f"ma{period}_bias": frozenset({"close", f"ma{period}"}) + for period in (5, 10, 20, 30, 60) + }, + **{ + f"ema{period}_bias": frozenset({"close", f"ema{period}"}) + for period in (5, 10, 20, 30, 60) + }, + "macd_dif_pct": frozenset({"close", "macd_dif"}), + "macd_dea_pct": frozenset({"close", "macd_dea"}), + "macd_hist_pct": frozenset({"close", "macd_hist"}), + "boll_position": frozenset({"close", "boll_upper", "boll_lower"}), + "atr_pct": frozenset({"close", "atr_14"}), + "boll_width": frozenset({"ma20", "boll_upper", "boll_lower"}), + "vol_ratio_10d": frozenset({"volume"}), + "vol_trend_5_10": frozenset({"vol_ma5", "vol_ma10"}), + "turnover_ratio_5d": frozenset({"turnover_rate"}), + "log_amount": frozenset({"amount"}), + "amount_ratio_5d": frozenset({"amount"}), + "gap_return": frozenset({"open", "prev_close"}), + "intraday_return": frozenset({"open", "close"}), + "close_position": frozenset({"high", "low", "close"}), + "distance_to_high_60d": frozenset({"close", "high_60d"}), + "distance_from_low_60d": frozenset({"close", "low_60d"}), + "max_ret_20d": frozenset({"close"}), + "ret_skew_20d": frozenset({"close"}), + "up_days_20d": frozenset({"close"}), + "amihud_20d": frozenset({"close", "amount"}), + "turnover_z_60d": frozenset({"turnover_rate"}), + "vol_price_corr_20d": frozenset({"close", "volume"}), + "vwap_bias": frozenset({"close", "volume", "amount"}), + "vol_trend_5_60": frozenset({"volume"}), + "limit_up_count_20d": frozenset({"consecutive_limit_ups"}), + "limit_up_count_60d": frozenset({"consecutive_limit_ups"}), + # --- 扩充批次 (2026-09-05) --- + "log_float_mv": frozenset({"close", "volume", "turnover_rate"}), + "momentum_120d": frozenset({"close"}), + "mom_accel_20_60": frozenset({"momentum_20d", "momentum_60d"}), + "rsi_14_delta_5d": frozenset({"rsi_14"}), + "overnight_ret_20d": frozenset({"open", "prev_close"}), + "intraday_ret_20d": frozenset({"open", "close"}), + "downside_vol_20d": frozenset({"close"}), + "vol_regime_5_60": frozenset({"close"}), + "amplitude_trend_20_60": frozenset({"amplitude"}), + "obv_trend_20d": frozenset({"close", "volume"}), + "amount_mean_20d": frozenset({"amount"}), + "turnover_mean_20d": frozenset({"turnover_rate"}), + "turnover_std_20d": frozenset({"turnover_rate"}), + "position_240d": frozenset({"close"}), + "distance_to_high_240d": frozenset({"close"}), + "kdj_kd_diff": frozenset({"kdj_k", "kdj_d"}), +} + +GOLDEN_WARMUP: dict[str, int] = { + "vol_ratio_10d": 11, + "turnover_ratio_5d": 6, + "amount_ratio_5d": 6, + "max_ret_20d": 21, + "ret_skew_20d": 21, + "up_days_20d": 21, + "amihud_20d": 21, + "turnover_z_60d": 61, + "vol_price_corr_20d": 21, + "vol_trend_5_60": 60, + "limit_up_count_20d": 21, + "limit_up_count_60d": 61, + # --- 扩充批次 (2026-09-05) --- + "momentum_120d": 121, + "rsi_14_delta_5d": 6, + "overnight_ret_20d": 21, + "intraday_ret_20d": 21, + "downside_vol_20d": 21, + "vol_regime_5_60": 61, + "amplitude_trend_20_60": 61, + "obv_trend_20d": 21, + "amount_mean_20d": 21, + "turnover_mean_20d": 21, + "turnover_std_20d": 21, + "position_240d": 241, + "distance_to_high_240d": 241, +} + + +def test_factor_columns_snapshot() -> None: + """注册表生成的 FACTOR_COLUMNS 与收口前字面量逐项一致 (含顺序)。""" + from app.backtest.factor import FACTOR_COLUMNS + + assert FACTOR_COLUMNS == GOLDEN_COLUMNS + assert factor_columns_view() == GOLDEN_COLUMNS + + +def test_virtual_dependencies_snapshot() -> None: + """注册表生成的依赖声明与收口前字面量逐项一致。""" + from app.strategy.scoring import VIRTUAL_SCORING_DEPENDENCIES + + assert VIRTUAL_SCORING_DEPENDENCIES == GOLDEN_VIRTUAL_DEPS + assert virtual_dependencies() == GOLDEN_VIRTUAL_DEPS + + +def test_scoring_warmup_snapshot() -> None: + from app.strategy.scoring import _ROLLING_SCORING_WARMUP + + assert _ROLLING_SCORING_WARMUP == GOLDEN_WARMUP + assert scoring_warmups() == GOLDEN_WARMUP + + +def test_catalog_counts_and_kinds() -> None: + specs = all_factors() + assert len(specs) == 77 + assert len({spec.id for spec in specs}) == 77 # id 唯一 + virtual = [spec for spec in specs if spec.kind == "virtual"] + assert len(virtual) == 52 # ma/ema 10 + 原有 26 + 扩充批次 16 + financial = [spec for spec in specs if spec.pit] + assert len(financial) == 7 + assert all(spec.pit_source == "financial_announce" for spec in financial) + assert all(spec.asset_types == frozenset({"stock"}) for spec in financial) + + +def test_mining_schedule_order_prefix_unchanged() -> None: + """挖掘调度取 FACTOR_COLUMNS[:48], 首元素必须保持 momentum_5d。""" + from app.backtest.factor import FACTOR_COLUMNS + + assert FACTOR_COLUMNS[0]["id"] == "momentum_5d" + assert len(FACTOR_COLUMNS) >= 48 + + +def test_get_factor_and_dependencies() -> None: + spec = get_factor("ma20_bias") + assert spec is not None + assert spec.dependencies == frozenset({"close", "ma20"}) + assert spec.warmup_bars == 1 # 无滚动窗口, 与历史默认一致 + + resolved = factor_dependencies(["ma20_bias", "rsi_14", "unknown_col"]) + assert resolved == frozenset({"close", "ma20", "rsi_14", "unknown_col"}) + + +def test_asset_type_filter() -> None: + stock = all_factors(asset_type="stock") + etf = all_factors(asset_type="etf") + assert len(stock) == 77 + assert len(etf) == 70 # 财务 7 项仅股票 + + +def test_register_factor_rejects_duplicate() -> None: + spec = get_factor("rsi_14") + assert spec is not None + with pytest.raises(ValueError, match="已注册"): + register_factor(spec) + + +def test_register_factor_allows_version_bump() -> None: + from app.factors import registry + + fresh = FactorSpec(id="__test_custom_factor", label="测试因子", group="测试", formula_text="close", kind="custom") + register_factor(fresh) + bumped = FactorSpec( + id="__test_custom_factor", label="测试因子", group="测试", formula_text="close + 1", + kind="custom", version=2, + ) + register_factor(bumped) + try: + current = get_factor("__test_custom_factor") + assert current is not None + assert current.version == 2 + assert current.formula_text == "close + 1" + finally: + # 清理测试注册项; 目录视图 (_CATALOG) 不受 _REGISTRY 动态注册影响 + registry._REGISTRY.pop("__test_custom_factor", None) + + +def _client(): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from app.api.factors import router + + app = FastAPI() + app.include_router(router) + return TestClient(app) + + +def test_factors_api_contract() -> None: + client = _client() + response = client.get("/api/factors") + assert response.status_code == 200 + payload = response.json() + factors = payload["factors"] + assert len(factors) == 77 + first = factors[0] + assert first["id"] == "momentum_5d" + assert first["kind"] == "base" + assert first["formula"] == "5个交易日累计收益率" + assert first["asset_types"] == ["etf", "stock"] + ma20 = next(item for item in factors if item["id"] == "ma20_bias") + assert ma20["kind"] == "virtual" + assert ma20["dependencies"] == ["close", "ma20"] + pb = next(item for item in factors if item["id"] == "pb_latest") + assert pb["pit"] is True + assert pb["asset_types"] == ["stock"] + mv = next(item for item in factors if item["id"] == "log_float_mv") + assert mv["kind"] == "virtual" + assert mv["scale_free"] is False + + +def test_factors_api_asset_filter_and_validation() -> None: + client = _client() + etf = client.get("/api/factors", params={"asset_type": "etf"}).json()["factors"] + assert len(etf) == 70 + assert all("stock" in item["asset_types"] for item in etf) + # 非法资产类型 → 422 (fail-closed, 不静默回退全量) + assert client.get("/api/factors", params={"asset_type": "index"}).status_code == 422 diff --git a/backend/tests/test_factor_store.py b/backend/tests/test_factor_store.py new file mode 100644 index 0000000..5f7578b --- /dev/null +++ b/backend/tests/test_factor_store.py @@ -0,0 +1,148 @@ +"""自定义/复合因子存储与 scoring 桥测试 (P3)。""" +from __future__ import annotations + +from datetime import date + +import polars as pl +import pytest + +from app.factors import store +from app.factors.registry import ( + FactorSpec, + all_factors, + factor_columns_view, + get_factor, + unregister_factor, +) +from app.strategy import scoring + + +@pytest.fixture() +def cleanup_registry(): + """测试注册的自定义因子在用例后清理, 不污染全局注册表。""" + before = set() + yield before + for fid in before: + unregister_factor(fid) + + +def _panel(n_days: int = 30) -> pl.DataFrame: + rows = [] + volumes = {"A": 1000.0, "B": 3000.0, "C": 2000.0} + for index in range(n_days): + for symbol, close in (("A", 10.0 + index), ("B", 50.0 - index), ("C", 20.0 + index * 2)): + rows.append({ + "symbol": symbol, "date": date(2026, 1, index + 1), + "close": close, "volume": volumes[symbol] + index, "amount": (1000.0 + index) * close, + }) + return pl.DataFrame(rows).sort(["symbol", "date"]) + + +def test_custom_factor_definition_roundtrip(tmp_path, cleanup_registry) -> None: + definition = { + "id": "uf_test_rev", "kind": "custom", "version": 1, "label": "测试反转", + "group": "自定义", "formula": "rank(-ts_sum(close / ts_delay(close, 1) - 1, 5))", + "description": "", "direction": "low", "status": "draft", + } + spec = store.register_definition(definition) + cleanup_registry.add("uf_test_rev") + assert spec.kind == "custom" + assert "close" in spec.dependencies + assert spec.warmup_bars >= 6 + + store.save_one(tmp_path, definition) + loaded = store.load_all(tmp_path) + assert len(loaded) == 1 and loaded[0]["id"] == "uf_test_rev" + + # 目录视图与 all_factors 追加动态因子 + ids = [item["id"] for item in factor_columns_view()] + assert ids[:77] == [item["id"] for item in factor_columns_view()[:77]] + assert "uf_test_rev" in ids and ids.index("uf_test_rev") >= 77 + assert any(s.id == "uf_test_rev" for s in all_factors()) + + # 快照约束不受影响: 未注册动态因子时目录 = 77 内置 + unregister_factor("uf_test_rev") + assert len(factor_columns_view()) == 77 + + +def test_custom_factor_invalid_rejected(cleanup_registry) -> None: + with pytest.raises(ValueError, match="E005"): + store.register_definition({ + "id": "uf_bad", "kind": "custom", "label": "坏因子", + "formula": "ts_delay(close, -3)", "status": "draft", + }) + with pytest.raises(ValueError, match="uf_"): + store.register_definition({ + "id": "wrong_prefix", "kind": "custom", "label": "坏前缀", + "formula": "close", "status": "draft", + }) + + +def test_composite_definition_and_cycle_guard(cleanup_registry) -> None: + definition = { + "id": "cf_test_combo", "kind": "composite", "version": 1, "label": "测试组合", + "members": {"momentum_20d": 0.6, "turnover_rate": 0.4}, "status": "draft", + } + spec = store.register_definition(definition) + cleanup_registry.add("cf_test_combo") + assert spec.kind == "composite" + assert spec.components == (("momentum_20d", 0.6), ("turnover_rate", 0.4)) + assert spec.dependencies == frozenset({"momentum_20d", "turnover_rate"}) + + # 自引用拒绝 + with pytest.raises(ValueError, match="自身"): + store.to_spec({**definition, "id": "cf_self", "members": {"cf_self": 1.0, "close": 1.0}}) + + +def test_scoring_bridge_composite(cleanup_registry) -> None: + """复合因子经 scoring 物化: 截面加权 z 分可计算且依赖展开正确。""" + store.register_definition({ + "id": "cf_ztest", "kind": "composite", "version": 1, "label": "桥接测试", + "members": {"close": 0.5, "volume": 0.5}, "status": "active", + }) + cleanup_registry.add("cf_ztest") + + deps = scoring.scoring_dependencies({"cf_ztest": 1.0}) + assert deps == {"close", "volume"} + assert scoring.scoring_warmup_bars({"cf_ztest": 1.0}) >= 1 + + frame = scoring.materialize_scoring_columns(_panel(), {"cf_ztest"}) + assert "cf_ztest" in frame.columns + values = frame.filter(pl.col("date") == date(2026, 1, 10))["cf_ztest"] + assert values.is_not_null().all() + # 截面 z 之和的均值近似为 0 (等权两成员) + assert abs(values.mean()) < 1e-9 + + +def test_scoring_bridge_custom_materializes(cleanup_registry) -> None: + """自定义 DSL 因子经 materialize_scoring_columns 物化 (与检验共用路径)。""" + store.register_definition({ + "id": "uf_rank_close", "kind": "custom", "version": 1, "label": "价格排名", + "formula": "rank(close)", "status": "draft", + }) + cleanup_registry.add("uf_rank_close") + frame = scoring.materialize_scoring_columns(_panel(), {"uf_rank_close"}) + assert "uf_rank_close" in frame.columns + day = frame.filter(pl.col("date") == date(2026, 1, 1)) + assert day["uf_rank_close"].is_not_null().all() + + +def test_load_into_registry_isolated_failure(tmp_path, cleanup_registry) -> None: + good = { + "id": "uf_good", "kind": "custom", "version": 1, "label": "好因子", + "formula": "close + 1", "status": "draft", + } + store.save_one(tmp_path, good) + (tmp_path / "user_data" / "custom_factors" / "uf_broken.json").write_text( + "{ not json", encoding="utf-8" + ) + loaded = store.load_into_registry(tmp_path) + assert loaded == ["uf_good"] + cleanup_registry.add("uf_good") + + +def test_unregister_builtin_rejected() -> None: + with pytest.raises(ValueError, match="内置"): + unregister_factor("rsi_14") + spec = get_factor("rsi_14") + assert isinstance(spec, FactorSpec) diff --git a/backend/tests/test_final_sync_confirmation.py b/backend/tests/test_final_sync_confirmation.py new file mode 100644 index 0000000..d5b513f --- /dev/null +++ b/backend/tests/test_final_sync_confirmation.py @@ -0,0 +1,149 @@ +"""final 定版确认回归: 午休/收盘定版必须校验快照时间戳, 未达边界不落盘。 + +实测 (2026-09-04): 收盘定版在 15:00:02 仅凭"拉取成功"即标记完成并落盘, 但 +实时源当时仍返回 14:59:5x 的竞价前快照 (海鸥住工 7.07 而非官方收盘 7.10), +旧价被永久固化到当日分区; 且重启后的盘后手动刷新会再次写回旧价。修复后: + +- _process_full_market_records 收到 final_boundary_ms 时, 快照最大时间戳 + 达到边界 (含容差) 才落盘/评估监控, 否则只更新展示缓存; +- _final_boundary_ms/_past_final_deadline 提供边界与重试窗口 (收盘 15:00/15:30)。 +""" +from __future__ import annotations + +from datetime import datetime, time as dt_time + +import pytest + +import app.services.quote_service as qs_module +from app.market_time import CN_TZ, cn_today +from app.services.quote_service import QuoteService + + +def _beijing_ms(h: int, m: int, s: int = 0) -> int: + return int(datetime.combine(cn_today(), dt_time(h, m, s), tzinfo=CN_TZ).timestamp() * 1000) + + +def _record(ts_ms: int) -> dict: + return { + "symbol": "002084.SZ", + "last_price": 7.07, + "open": 7.00, "high": 7.10, "low": 6.95, + "volume": 100_000, "amount": 707_000.0, + "timestamp": ts_ms, + } + + +class _StubRepo: + """记录写盘调用的最小仓库桩。""" + + def __init__(self) -> None: + self.calls: list[str] = [] + + def get_index_symbol_set(self) -> set: + return set() + + def get_etf_instruments(self): + import polars as pl + return pl.DataFrame() + + def flush_live_daily(self, df) -> None: + self.calls.append("daily") + + def flush_live_daily_asset(self, asset_type: str, df) -> None: + self.calls.append(f"daily:{asset_type}") + + +@pytest.fixture +def service(monkeypatch) -> tuple[QuoteService, _StubRepo, dict]: + qs = QuoteService() + repo = _StubRepo() + qs._repo = repo + events: dict[str, int] = {"broadcast": 0, "enriched": 0} + monkeypatch.setattr(qs_module, "_persist_last_fetch", lambda ms: None) + monkeypatch.setattr(qs, "_update_volume_delta", lambda *a, **k: None) + monkeypatch.setattr(qs, "_evaluate_monitors", lambda *a, **k: None) + monkeypatch.setattr(qs, "_broadcast_quote_updated", lambda: events.__setitem__("broadcast", events["broadcast"] + 1)) + monkeypatch.setattr(qs, "_flush_live_enriched", lambda *a, **k: events.__setitem__("enriched", events["enriched"] + 1)) + return qs, repo, events + + +def test_final_snapshot_before_boundary_skips_disk(service) -> None: + """竞价前快照 (时间戳 < 15:00): 只更新展示缓存, 不写 daily/enriched。""" + qs, repo, events = service + boundary = _beijing_ms(15, 0) + + qs._process_full_market_records( + [_record(boundary - 60_000)], t0=0.0, now_ts=0.0, + final_boundary_ms=boundary, + ) + + assert qs._last_final_confirmed is False + assert repo.calls == [] # 未写 kline_daily + assert events["enriched"] == 0 # 未写 enriched + assert events["broadcast"] == 1 # 展示缓存路径仍走通 + + +def test_final_snapshot_after_boundary_writes(service) -> None: + """边界后快照 (时间戳 ≥ 15:00): 定版落盘。""" + qs, repo, events = service + boundary = _beijing_ms(15, 0) + + qs._process_full_market_records( + [_record(boundary + 30_000)], t0=0.0, now_ts=0.0, + final_boundary_ms=boundary, + ) + + assert qs._last_final_confirmed is True + assert repo.calls == ["daily"] + assert events["enriched"] == 1 + assert events["broadcast"] == 1 + + +def test_snapshot_without_timestamp_never_confirmed(service) -> None: + """无时间戳的快照无法确认定版 → 不落盘 (交由盘后管道兜底)。""" + qs, repo, events = service + rec = _record(0) + rec.pop("timestamp") + + qs._process_full_market_records( + [rec], t0=0.0, now_ts=0.0, final_boundary_ms=_beijing_ms(15, 0), + ) + + assert qs._last_final_confirmed is False + assert repo.calls == [] + assert events["enriched"] == 0 + + +def test_normal_poll_ignores_boundary(service) -> None: + """普通轮询 (无 final_boundary_ms): 时间戳在边界前也照常落盘。""" + qs, repo, events = service + + qs._process_full_market_records( + [_record(_beijing_ms(14, 59))], t0=0.0, now_ts=0.0, + ) + + assert qs._last_final_confirmed is None + assert repo.calls == ["daily"] + assert events["enriched"] == 1 + + +def test_final_boundary_ms_matches_beijing_close(monkeypatch) -> None: + fake_now = datetime.combine(cn_today(), dt_time(15, 10), tzinfo=CN_TZ) + monkeypatch.setattr(qs_module, "cn_now", lambda: fake_now) + assert QuoteService._final_boundary_ms("close_final") == _beijing_ms(15, 0) + assert QuoteService._final_boundary_ms("morning_final") == _beijing_ms(11, 30) + assert QuoteService._final_boundary_ms("afternoon") is None + + +def test_past_final_deadline(monkeypatch) -> None: + def _at(h: int, m: int): + return datetime.combine(cn_today(), dt_time(h, m), tzinfo=CN_TZ) + + monkeypatch.setattr(qs_module, "cn_now", lambda: _at(15, 29)) + assert QuoteService._past_final_deadline("close_final") is False + monkeypatch.setattr(qs_module, "cn_now", lambda: _at(15, 30)) + assert QuoteService._past_final_deadline("close_final") is True + monkeypatch.setattr(qs_module, "cn_now", lambda: _at(12, 9)) + assert QuoteService._past_final_deadline("morning_final") is False + monkeypatch.setattr(qs_module, "cn_now", lambda: _at(12, 11)) + assert QuoteService._past_final_deadline("morning_final") is True diff --git a/backend/tests/test_stats_v2.py b/backend/tests/test_stats_v2.py new file mode 100644 index 0000000..04b877c --- /dev/null +++ b/backend/tests/test_stats_v2.py @@ -0,0 +1,110 @@ +"""metrics_v2 统计函数测试 (P3) — 黄金参考向量 + 性质断言。 + +NW t 的黄金值由测试内的独立第二实现 (显式循环求 Bartlett 加权长方差) 推导, +与 stats_v2 向量化实现互为对拍。 +""" +from __future__ import annotations + +import math + +import numpy as np +import pytest + +from app.backtest.stats_v2 import ( + _normal_ppf, + bh_fdr_qvalues, + deflated_sharpe_psr, + expected_max_sharpe, + naive_t, + newey_west_t, + normal_two_sided_p, +) + + +def _nw_t_reference(values: list[float], lag: int) -> float: + """独立第二实现: 显式循环按定义计算 Bartlett 核 HAC t 值。""" + n = len(values) + mean = sum(values) / n + centered = [value - mean for value in values] + gamma = [ + sum(centered[i] * centered[i + lag_i] for i in range(n - lag_i)) / n + for lag_i in range(lag + 1) + ] + long_var = gamma[0] + for lag_i in range(1, lag + 1): + long_var += 2.0 * (1.0 - lag_i / (lag + 1)) * gamma[lag_i] + se = math.sqrt(long_var / n) + return mean / se + + +def test_newey_west_matches_reference() -> None: + rng = np.random.default_rng(42) + values = np.cumsum(rng.normal(0, 0.01, 60)).tolist() # 高自相关 + for lag in (1, 3, 5): + result = newey_west_t(values, lag) + assert result is not None + t_stat, mean, se = result + assert t_stat == pytest.approx(_nw_t_reference(values, lag), rel=1e-9) + assert mean == pytest.approx(float(np.mean(values))) + assert se > 0 + + +def test_newey_west_deflates_autocorrelated_t() -> None: + # 强正自相关序列: NW t 的绝对值必须小于朴素 t (自相关被正确惩罚) + rng = np.random.default_rng(7) + phi = 0.9 + values, last = [], 0.0 + for shock in rng.normal(0, 0.01, 500): + last = phi * last + shock + values.append(last) + t_naive = naive_t(values) + result = newey_west_t(values, lag=5) + assert t_naive is not None and result is not None + assert abs(result[0]) < abs(t_naive) + + +def test_newey_west_insufficient_samples() -> None: + assert newey_west_t([0.1, 0.2], lag=1) is None + assert newey_west_t([], lag=1) is None + assert newey_west_t([1.0] * 20, lag=1) is None # 零方差 + + +def test_bh_fdr_golden() -> None: + # 经典 BH 示例 (Wikipedia): q = [.005, .02, .042, .042, .042] + pvalues = [0.001, 0.008, 0.039, 0.041, 0.042] + assert bh_fdr_qvalues(pvalues) == pytest.approx([0.005, 0.02, 0.042, 0.042, 0.042]) + # 乱序输入: q 值跟随原位置 (m=3: .042→r3 raw .042; .001→.003; .039→min(.0585,.042)=.042) + assert bh_fdr_qvalues([0.042, 0.001, 0.039]) == pytest.approx([0.042, 0.003, 0.042]) + # None 透传 + assert bh_fdr_qvalues([None, 0.05]) == [None, 0.05] + + +def test_normal_p_and_ppf_inverse() -> None: + assert normal_two_sided_p(1.959964) == pytest.approx(0.05, abs=1e-6) + assert normal_two_sided_p(0.0) == pytest.approx(1.0) + assert _normal_ppf(0.975) == pytest.approx(1.959964, abs=1e-6) + assert _normal_ppf(0.5) == pytest.approx(0.0, abs=1e-9) + with pytest.raises(ValueError): + _normal_ppf(0.0) + + +def test_expected_max_sharpe_monotone() -> None: + assert expected_max_sharpe(1, 0.04) == 0.0 # 单试验不校正 + assert expected_max_sharpe(10, 0.0) == 0.0 + # 试验数越多期望最大夏普越高 (越难超越) + em_10 = expected_max_sharpe(10, 0.04) + em_100 = expected_max_sharpe(100, 0.04) + assert 0 < em_10 < em_100 + + +def test_deflated_sharpe_psr() -> None: + # 无偏斜无超额峰度时退化为 Φ(SR * sqrt(n-1)) + probability = deflated_sharpe_psr(sharpe=0.1, n_obs=2500, skewness=0.0, kurtosis=3.0, expected_max_sharpe=0.0) + assert probability == pytest.approx(0.5 * (1 + math.erf(0.1 * math.sqrt(2499) / math.sqrt(2)))) + # 校正项抬高分母会降低 PSR + penalized = deflated_sharpe_psr(0.1, 2500, skewness=0.0, kurtosis=10.0) + assert penalized < probability + # EM 校正降低显著性 + deflated = deflated_sharpe_psr(0.1, 2500, 0.0, 3.0, expected_max_sharpe=0.08) + assert deflated < probability + assert deflated_sharpe_psr(0.1, 3) is None # 样本不足 diff --git a/docs/factor-platform-plan.md b/docs/factor-platform-plan.md new file mode 100644 index 0000000..fe6fc78 --- /dev/null +++ b/docs/factor-platform-plan.md @@ -0,0 +1,403 @@ +# 因子平台总体方案(研究线产品与架构设计 v1) + +> 状态标记约定与 `factor-system-design.md` 一致: +> 【现状】= 当前仓库已存在、已核对代码;【设计】= 本方案提出、尚未实现。 +> 所有阶段代号 P0–P4 见 §10 路线图。 + +## 执行状态(2026-09-04) + +P0–P3 已全部实现并通过验证(后端 98 项因子相关测试 + 全量套件 + 前端 build + 真实数据 E2E): + +- **P0 已实现**:`/factors` 一级页(检验/因子库/编辑器/组合 四 tab)、检验页 UX 六项、回测页 tab 移除与跳转、候选方案注入策略页。 +- **跟进整合(2026-09-04)**:侧边栏「挖掘」菜单移除,AI 挖掘并入 `/factors?tab=mining`(第 5 个 tab);`/mining` 与 `/backtest?tab=mining` 旧链接保留参数重定向。 +- **自动挖掘(2026-09-05,P3 后续)**:检验页新增「自动挖掘」入口(`POST /api/backtest/mining/auto`)——L1 全量统计筛选(`app/services/auto_mining.py`,门槛按探索/均衡/严格三档)自动生成达标因子池,随请求持久化筛选摘要(达标清单 + 未达标原因分布),L2 相关性剪枝 / L3 束搜索组合 / L4 嵌套样本外验证与达标门槛全部复用现有挖掘引擎;worker 子进程启动时加载因子注册表(自定义/复合因子可入池)。组合搜索本身不依赖 LLM;AI 的位置留给后续"从失败模式生成新 DSL 因子假设"。 +- **因子 ↔ 策略结合(2026-09-05,四条桥全部已实现)**:① 策略触发器编辑内 Zap 快捷建因子条件信号(`SignalTriggerActions` + `AddFactorSignalDialog`,阈值给建议值,`csg_f_*` 自动挂接当前策略入场/卖出触发器);② 因子库「生成策略」一键产出单因子排名策略(`GenerateFactorStrategyDialog` 渲染 `FactorRankResearchMatrixStrategy` 模板,matrix_native,保存为 `custom_factor_*`);③ 自定义信号 AI 提示词注入全部因子分组(`custom_signals_ai.py::_format_fields`,行前缀「因子·」,白名单 `allowed_fields()` 收口);④ 策略回测因子归因(`strategy.py` 入场候选因子快照 → 胜/败单入场信号日因子均值,结果区第 4 个 tab,`meta.scoring` 非空即产出;矩阵单因子排名类策略因 META.scoring 设计为空,暂不产出归因,为已知边界)。 +- **编辑器交互重构(2026-09-05)**:算子区合并为单一可点面板(全部 25 个 DSL 算子按「时序/截面/工具」分组,点击插入光标处,悬停显示签名与参数约束,替代"16 高频 chips + 静态速查表"双列表);可用字段 chip 双语(英文 id 等宽 + 中文标签浅色);「从模板开始」下拉新增「我的因子」分组(导入已保存自定义因子公式),非初始公式替换前弹确认防误覆盖。 +- **因子库扩充(2026-09-05)**:目录 61 → 77(新增 16 个虚拟因子,追加于目录尾部,挖掘调度前 48 项顺序不变):规模(流通市值对数,由换手率反推流通股本)、收益分解(隔夜/日内 20 日累计)、120 日动量与动量加速度、RSI 边际变化、下行波动、波动率状态(5/60)、振幅趋势、量能潮 OBV、20 日均成交额、均换手/换手波动(低换手溢价)、一年价格位置与距一年高点、KDJ K-D 差。双内核同步实现(`strategy/scoring.py` 评分路径 + `backtest/matrix.py` 矩阵内核,ddof 口径对齐存量约定);批量检验上限 64 → 96。 +- **P1 已实现**:`backend/app/factors/registry.py` 注册表(61 内置因子单一事实源)、快照/契约测试、`GET /api/factors`。 +- **P2 已实现**:`backend/app/factors/dsl.py` 公式编译器(词法→EBNF 语法→语义校验 E001–E016→Polars 表达式,两阶段求值解决跨截面套时序窗口)、`POST /api/factors/validate|trial`、编辑器 tab。 +- **P3 已实现**:`stats_v2.py`(Newey-West t / BH-FDR q / DSR,纯 numpy)、批量检验接线与前端服务端判读;`store.py` 自定义/复合因子持久化 + 评分桥(与策略共用同一条物化路径)+ CRUD API + 启动多轮加载;组合 tab;生命周期状态(草稿/激活/观察/退役)与状态流转 API。 +- **P4 部分实现**:生命周期状态已落地;**巡检 tab 与后台衰减定时任务延后**(见 §10),复合因子权重 ICIR 自动配置模式延后。 +- 验证环境备注:因子批量检验对 custom/composite 的底层依赖先经内置补算路径物化(`factor.py::_compute_missing_factors`),已按 fail-closed 语义处理缺失列。 + +## 0. 文档定位与阅读顺序 + +| 文档 | 角色 | 生效裁定 | +| --- | --- | --- | +| 本文(factor-platform-plan.md) | **产品与架构总体方案**:两线架构、导航与页面、交互设计、功能全集、路线图 | 本文 §2–§8 生效后,`factor-system-design.md` §11 的"不新增顶层页面"结论**由本文取代** | +| `factor-system-design.md` | **后端技术 spec**:FactorSpec schema、DSL 文法与算子表、统计口径(NW/BH-FDR/DSR)、API 契约、缓存键、测试矩阵 | 除 §11 外全部继续有效;本文引用不复制 | + +两文冲突时:产品/前端/导航以本文为准;后端 schema、统计公式、API 字段以旧文为准。 + +## 1. 架构总览:两线模型 + +### 1.1 模型 + +```text +研究线(发现问题) 交易线(执行战法) +┌─────────────────────────┐ ┌─────────────────────────┐ +│ 因子库(注册表) │ │ 策略引擎(战法) │ +│ ↑ 检验(IC/IR/多空) │ │ 时机/买卖规则/仓位/风控 │ +│ ↑ 公式DSL(自定义因子) │ │ ↑ 评分选股 │ +│ ↑ 组合因子(加权合成) │ │ ↑ 验证(鲁棒性/步进) │ +│ 挖掘(自动搜索因子组合) │ │ 回测(策略绩效) │ +└───────────┬─────────────┘ └───────────↑─────────────┘ + │ 唯一桥梁:策略 scoring 配置 │ + └────────── 因子(组合)权重 ──────────────────┘ +``` + +- **因子层**回答"此刻全市场谁排前面"(截面排序);**战法层**回答"何时进、买几只、多重仓、何时出、怎么止损"。 +- 两线只通过**评分配置**结合:`{"scoring": {"cf_hotmom": 0.6, ...}}`。因子不会"变成"策略,因子给策略供货。 +- 【现状】桥已存在:`backend/app/strategy/scoring.py:13` `VIRTUAL_SCORING_DEPENDENCIES`(35 个虚拟因子依赖声明)、`scoring_value_expr`(按名取值表达式)、`mining.py:342` 挖掘候选产物本身就是 `scoring = dict(zip(factor_names, weights))`。 +- 【现状】研究线已分裂在两处:因子检验是 `/backtest?tab=factor`,挖掘是独立页 `/mining`(`frontend/src/router.tsx:126-127`)。本方案将研究线收拢。 + +### 1.2 各能力现状对照 + +| 能力 | 现状 | 本方案落点 | +| --- | --- | --- | +| 因子检验(批量/单因子) | 【现状】`backtest/factor.py` 61 因子、Rank IC/分年/衰减/分环境;前端 FactorDiscovery | §3(搬迁 + UX 重设计) | +| 因子注册 | 【现状】硬编码 `FACTOR_COLUMNS`(`factor.py:36`),加因子=改代码 | §4 + 旧文 §2 L-REG 注册表 | +| 公式配置 | 【现状】无(公式即代码) | §5 + 旧文 §3 DSL | +| 自动挖掘 | 【现状】`mining.py` beam search + 嵌套验证 + 晋升门槛,`/mining` 页 | §7(协同设计,不搬家) | +| 因子→策略 | 【现状】scoring 机制可用;复合因子需改代码 | §6 + 旧文 §8 FactorCompositeSpec | +| 生命周期管理 | 【现状】无 | §8(P4) | + +## 2. 信息架构与导航【设计】 + +### 2.1 导航变化 + +```text +现状: 回测(/backtest: 因子|策略|验证) 挖掘(/mining) … +目标: 回测(/backtest: 策略|验证) 因子(/factors) 挖掘(/mining) … + ↑ 导航顺序:回测 → 因子 → 挖掘 相邻 +``` + +- 新增一级路由 `/factors`,导航标签「因子」,插入在回测与挖掘之间(`router.tsx` + `Layout.tsx`,两者为二开文档 §5.2 高冲突热点,diff 必须最小)。 +- `/backtest` 瘦身为「策略」「验证」两个 tab。 +- **旧链接兼容**:`/backtest?tab=factor` → ``,沿用现有先例——`Backtest.tsx:38-43` 已用同一模式把 `tab=mining` 重定向到 `/mining` 并保留 query 参数。`tab=factor` 的重定向须保留 `run/candidate` 类参数的传递习惯(对齐 mining 先例)。 + +### 2.2 候选方案抽屉的归属 + +【现状】候选方案抽屉内联在 `Backtest.tsx:114`,头部门按钮打开;因子候选(FactorDiscovery「保存候选」)与策略候选共用该存储,且已有"载入复测"流转(`Backtest.tsx:35`)。 + +【设计】P0 将抽屉提取为共享组件 `CandidateDrawer`,**回测页与因子页都放入口**。裁定理由:候选是研究线的产物、交易线的原料,两边都要能开;提取是搬移不是复制(CONTRIBUTING §7 共享组件原则)。 + +### 2.3 因子页骨架 + +```text +/factors +├─ tab=inspect(检验) P0:现有 FactorDiscovery 整体迁入 + UX 六项改造(§3) +├─ tab=library(因子库) P1:注册表浏览器(§4) +├─ tab=editor(编辑器) P2:自定义公式因子(§5) +├─ tab=composite(组合) P3:组合因子构建器(§6) +└─ tab=lifecycle(巡检) P4:衰减监控看板(§8) +``` + +- tab 状态走 URL query(`?tab=`),与回测页习惯一致,保证可收藏、可跳转。 +- P0 只落地 `inspect`,其余 tab 以"规划中"占位不渲染入口(YAGNI:不提前放空壳 tab)。 + +## 3. 检验 tab 详细设计(P0,纯前端) + +### 3.1 页面线框 + +```text +┌──────────────────────────────────────────────────────────────────────┐ +│ 因子检验 [候选方案] [AI 挖掘→] │ +│ ┌ 三句话说清楚(可折叠,默认展开,关后记忆到 localStorage)─────────┐ │ +│ │ ① 因子 = 用来给股票打分排序的特征(如 20日涨幅、换手率变化)。 │ │ +│ │ ② 这里检验:过去一段日子里,哪些特征真的能预测次日涨跌。 │ │ +│ │ ③ IC = 预测准确度(绝对值越大越准),IR = 稳定度(越大越稳)。 │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ ┌ 筛选配置 ────────────────────────────────────────────────────────┐ │ +│ │ 想好要测什么了吗?没有的话从预设开始: │ │ +│ │ [全面体检 61] [快速体检 10] [趋势动量 14] [超跌反转 9] │ │ +│ │ [量价资金 11] [财务价值 7] 当前: 全面体检 ✓(点预设=替换选择) │ │ +│ │ 已选 61/61 全选 清空 [🔍 搜索因子 (名称/公式)] │ │ +│ │ ┌─ 限高 45vh 内部滚动列表(现状保留)────────────────────────┐ │ │ +│ │ │ ▾ 动量 (6) 本组测:过去 N 日涨了多少 … │ │ │ +│ │ │ ☑ 5日动量 5个交易日累计收益率 │ │ │ +│ │ └────────────────────────────────────────────────────────────┘ │ │ +│ │ 资产与范围 / 日期 / 调仓 / 分组 / 佣金(现状保留) │ │ +│ │ [ 筛选 61 个因子 ] │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ ┌ 筛选结果 ─────────────────────────────────────────────────────────┐ │ +│ │ 结论句:61 个因子中 7 个有效*。最强:5日反转(IC −0.041)。 │ │ +│ │ 表格: 排名 | 因子(含方向箭头) | 预测力IC | 稳定度IR | IC胜率 | │ │ +│ │ 多空收益 | 最大回撤 | [结论徽章] | 操作 │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────┘ +``` + +### 3.2 预设场景入口 + +**设计动机**:现状默认全选 61 个,用户第一个动作是"做选择题";预设把第一个动作变成"按一下"。 + +预设清单(数量已按 `factor.py` FACTOR_COLUMNS 实际分组核对:动量 6、趋势 8、均线偏离 10、量价 9、价格位置 6、超买超卖 3、收益形态 3、波动率 5、流动性 2、涨停基因 2、财务 7,共 61): + +| 预设 | 数量 | 构成规则 | 适合谁 | +| --- | --- | --- | --- | +| 全面体检(默认) | 61 | 全部因子 | 不知道从哪开始的人;标注"耗时最长" | +| 快速体检 | 10 | 除财务组外每组取窗口中位代表因子(如动量组取 20日动量) | 想快速看一圈的人 | +| 趋势动量 | 14 | 动量组 + 趋势组 | 关心趋势行情下什么有效 | +| 超跌反转 | 9 | 超买超卖组 + 价格位置组 | 关心抄底反转逻辑 | +| 量价资金 | 11 | 量价组 + 流动性组 | 关心资金与量能信号 | +| 财务价值 | 7 | 财务组 | 关心基本面(需财务数据源能力) | + +交互规则: + +1. 点预设 = **替换**当前选择集(不是叠加),按钮立即显示"当前预设 ✓"。 +2. 用户手动改勾选后,状态变为「自定义」,所有预设按钮取消高亮。 +3. 预设是**纯前端选择集**,不新增后端概念;构成规则在代码里以"组名 + 组内规则/显式 id 列表"声明,实现时从 `columns.data` 按组过滤生成(不硬编码 61 个 id 的第二份清单)。 +4. 「快速体检」的代表因子实现时从 FACTOR_COLUMNS 按"窗口中位"解析;解析失败的组整组纳入(fail-open 但数量标注会变)。 + +### 3.3 三句话引导条 + +- 位置:页面标题下方,结果区上方;`
` 式可折叠,默认展开,关闭状态记 `localStorage`(key 纳入现有 preferences 机制则更好,实现时择一,不新增后端字段)。 +- 文案即 §3.1 线框中三句,不许加第四句——解释超过三句就没人读。 +- 每句中的术语(IC/IR)与表头 tooltip(§3.4)文案保持一致。 + +### 3.4 表头判读 tooltip(文案定稿) + +悬停表头即出,内容一句话 + 判读标准。列名同步通俗化(括号内保留原术语供专业用户): + +| 列 | 通俗列名 | tooltip 文案 | +| --- | --- | --- | +| IC 均值 | 预测力 IC | 每天用因子给股票打分、与次日真实涨跌算相关性(Rank IC)的均值。\|IC\|≥0.02 且稳定,即有预测力;负值同样有效(反向使用)。 | +| IR | 稳定度 IR | IC 均值 ÷ IC 波动。≥0.3 值得关注,≥0.5 相当稳定。 | +| IC 胜率 | 预测日占比 | IC 与预测方向一致的天数占比。50% 是抛硬币,55%+ 不错。 | +| 多空收益 | 多空收益 | 每期买因子最高组、(模拟)卖最低组的累计收益差。注意 A 股做空受限,此列为理论口径。 | +| 最大回撤 | 最大回撤 | 上述多空组合最痛的一段亏损幅度,衡量拿得住不住。 | +| 排名 | 排名 | 按当前排序键排序的名次,默认按 \|IC\|。 | + +### 3.5 结果判读(徽章 + 结论句) + +- 【现状】结果行字段:`ic_mean / ir / ic_win_rate / long_short_return / long_short_max_drawdown / error`(`FactorDiscovery.tsx:372-376`);**当前 API 无 t 值**。 +- 【设计·P0】客户端经验阈值判读(须在 UI 上明示"经验规则"): + - `有效`:`|ic_mean| ≥ 0.02 且 |ir| ≥ 0.3` + - `边缘`:`|ic_mean| ≥ 0.02 或 |ir| ≥ 0.3`(其一成立) + - `无效`:其余 + - `计算失败`:`error` 非空(现状已有 error 显示) +- 【设计·P3 起】后端 metrics_v2(旧文 §6,`t_newey_west`)落地后,判读切换为**服务端显著性与阈值双条件**,前端仅渲染服务端 verdict;P0 的客户端阈值作为降级路径保留。 +- 结论句模板:`{N} 个因子中 {M} 个有效。最强:{label}(IC {值})。`N=参与计算的因子数(排除 error),M=有效徽章数。结论句挂在表头右侧,字号小、颜色 muted。 + +### 3.6 因子方向标识 + +- 位置:**结果表**因子名旁,非复选框列表(检验前方向未知,不预测)。 +- 推导:`ic_mean > 0` → `↑ 值大看多`;`ic_mean < 0` → `↓ 值小看多`;null/error → 不显示。 +- 这是从数据推导的事实(样本内 IC 符号),非先验声明,因此**不进因子库元数据**(P1 因子库的方向列同样取最近一次检验的 IC 符号,并标注样本区间)。 + +### 3.7 路径闭环 + +1. **保存候选**后 toast:`已保存到候选方案(右上角查看)`;因子页头部有「候选方案」按钮(§2.2 共享抽屉)。 +2. **单因子检验**页(FactorDiscovery 内部 mode=single)顶部加返回链:`← 返回批量结果`,保留现有列表参数不丢。 +3. **因子库 → 检验**联动(P1):`/factors?tab=inspect&focus={factor_id}` 打开检验页并只勾选该因子。 +4. **AI 挖掘 →**:因子页头部的 `/mining` 链接(研究线互链,§7)。 + +### 3.8 状态设计(CONTRIBUTING §7 强制项) + +| 状态 | 触发 | 展示 | +| --- | --- | --- | +| 加载(因子清单) | columns.isLoading | 列表区骨架/「因子加载中…」(现状已有) | +| 加载(运行中) | run.isPending | 按钮禁用 + 进度文案(现状已有),期间**保留上次结果**不闪空(现状行为核对) | +| 空结果 | results=[] | EmptyState + 引导语「试试更长时间范围或减少因子」 | +| 错误 | run.isError | 错误条 + 重试按钮,保留左侧配置 | +| 无权限/无数据 | 财务预设但无财务数据源能力 | 预设按钮禁用 + title 说明「需要财务数据能力」(对齐能力矩阵门控习惯) | + +### 3.9 迁移清单(P0 范围内的代码动作,供实施 PR 参考) + +| # | 动作 | 文件 | 性质 | +| --- | --- | --- | --- | +| 1 | 新建 `/factors` 路由 + `Factors.tsx` 页壳(tab=inspect) | `router.tsx`、`pages/Factors.tsx` | 新增 | +| 2 | FactorDiscovery 连同其 queryKeys 原样迁入 | `pages/backtest/FactorDiscovery.tsx` → 组件复用 | 搬移 | +| 3 | `tab=factor` 重定向 `/factors`(保参,对齐 mining 先例) | `pages/Backtest.tsx` | 修改 | +| 4 | Backtest 移除因子 tab,MODES 剩 策略/验证 | `pages/Backtest.tsx` | 修改 | +| 5 | 候选抽屉提取共享组件,两页挂入口 | `Backtest.tsx` → `components/CandidateDrawer.tsx` | 提取 | +| 6 | UX 六项(§3.2–§3.7) | `FactorDiscovery.tsx`(或其子组件拆分) | 新增 | +| 7 | 导航项「因子」 | `Layout.tsx` | 修改 | + +不做的:不改 queryKeys.ts 键结构、不改任何 API、不动 factor.py。 + +## 4. 因子库 tab(P1,依赖 L-REG 注册表) + +### 4.1 列表 + +- 数据源:`GET /api/factors`(旧文 §10 已定义),P1 前无此接口时**本 tab 不上线**(不放假数据)。 +- 列:`id / 中文名 / 分组 / 类型(base|virtual|composite|custom)/ 公式(desc 或 DSL)/ 最近 IC(含样本区间)/ 状态徽章 / 资产类型 / warmup 天数`。 +- 筛选:搜索框(id/名称/公式,复用检验 tab 同款逻辑)+ 类型、分组、状态下拉。 +- 行点击 → 详情抽屉。 + +### 4.2 详情抽屉 + +- FactorSpec 全字段(旧文 §2.2 的 17 字段)按人话展示:公式、方向(最近检验 IC 符号 + 样本区间)、warmup("需要 X 日历史预热")、依赖列、适用资产。 +- 操作:「去检验」(§3.7 联动)、「克隆到编辑器」(P2 后可用)、custom/composite 显示「编辑」「停用」。 +- 底部:最近一次检验报告摘要(IC/IR/多空/回撤)+「查看完整报告」跳检验 tab。 + +### 4.3 状态徽章 + +与 §8 状态机一致:`active / watch / retired / draft`。P1 只有 `active`(全部内置因子)与 `draft`(编辑器产物),watch/retired 在 P4 激活。 + +## 5. 因子编辑器 tab(P2,依赖 DSL 编译器) + +### 5.1 布局 + +```text +┌────────────────────────┬──────────────────────────┐ +│ 公式编辑区 │ 校验与预览区 │ +│ textarea (等宽字体) │ ✓/✗ 语法校验(错误码+定位) │ +│ 算子面板 (25 个, 三组) │ 依赖列集合 │ +│ 时序/截面/工具, 点击插入│ warmup = max(窗口) 提示 │ +│ 可用字段 ▾ (双语 chip) │ [试算最近 40 日 IC] │ +│ 从模板开始/我的因子 ▾ │ 试算结果: IC 走势 sparkline │ +│ [校验] [试算 40 日] │ 注册/更新表单 │ +└────────────────────────┴──────────────────────────┘ +``` + +### 5.2 交互流 + +1. 输入公式 → 调 `POST /api/factors/validate`(旧文 §10;错误码 E001–E016 见旧文 §3.5,含负 shift 即未来函数的 E005)。 +2. 校验通过 → 展示编译产物:依赖列集合、warmup_bars(旧文 §3 规则 4)。 +3. 试算:选近 40 交易日 + 当前资产,跑一次 Rank IC 快照(复用检验引擎计算路径,**不另写计算逻辑**);展示 IC 序列 sparkline 与均值。 +4. 注册:状态 `draft`,写入注册表;后续在检验 tab 跑完整检验后转 `active`(§8 状态机)。编辑已有因子时公式变化保存为新版本并回草稿态。 +5. 模板起点:「从模板开始 / 我的因子」下拉——经典模板(5 个范式)+ 我的因子(已保存自定义因子公式导入);替换非初始公式前弹确认。 +6. 点选插入:全部 25 个 DSL 算子按「时序(13)/截面(3)/工具(9)」分组,点击插入光标处(与后端 `dsl.OPERATORS` 同步维护),悬停显示签名与参数约束;79 个可用字段 chip 双语显示(英文 id 等宽 + 中文标签),点击插入。 + +### 5.3 约束(红线) + +- 编辑器产物只经 DSL 编译到 Polars 表达式这一条计算路径(旧文 §3),**禁止**出现第二套求值器(CONTRIBUTING 禁止平行实现)。 +- 注册因子 id 冲突拒绝,不自增后缀静默改名。 +- 每一步的服务端校验 fail-closed:校验不过,注册按钮禁用且展示原因。 + +## 6. 组合 tab(P3,依赖 FactorCompositeSpec) + +### 6.1 组合构建器 + +- 成分:从因子库多选(上限 8 个,对齐挖掘 max_factors 习惯),每行显示该因子最近 IC/IR。 +- 权重三模式: + 1. **手动**:逐项权重滑杆,实时归一显示; + 2. **ICIR 自动**:`w_i ∝ max(ICIR_i, 0)`,窗口 504 交易日,**只用 (t−window, t−1] 数据**(旧文 §8 点时规则,严禁 t 日数据参与 t 日权重); + 3. **均等**:1/N。 +- 预览:选定回看区间内,组合 IC/IR 曲线 vs 各成分——证明"组合优于单因子"或及时劝退。 + +### 6.2 注册与桥接 + +- 「注册为复合因子」→ FactorCompositeSpec 入注册表,类型 `composite`。 +- 桥接机制**零引擎改动**(旧文 §8 已论证):注册时自动写入 `VIRTUAL_SCORING_DEPENDENCIES` 对应条目 → 立即出现在策略评分可选列表。 +- 「一键创建策略」:打开策略新建表单并预填 `scoring = {组合id: 1.0}`,用户补战法参数——完成研究线→交易线的最后一步。 + +## 7. 与挖掘页协同【设计】 + +裁定:**挖掘保持独立页**(已有用户习惯 + 任务型界面本就该独立),做研究线互链而非合并。 + +1. 因子页头部「AI 挖掘 →」链到 `/mining`。 +2. 挖掘候选详情加「查看因子检验」:跳 `/factors?tab=inspect&focus=...` 预选该候选的成分因子。 +3. P4 可选:挖掘晋升的候选自动以 `watch` 状态写入因子库(人工确认后转 active)——默认不自动转正,晋升门槛(`mining.py` GATE_*)只证明样本外可行,不过不代表可持续。 + +## 8. 因子生命周期(P4) + +### 8.1 状态机 + +```text +draft ──(完整检验通过)──▶ active ──(衰减告警/人工)──▶ watch ──(持续衰减/人工)──▶ retired + ▲ │ + └──────────────(重新编辑公式, 重新走检验)──────────────────┘ +``` + +| 迁移 | 触发条件 | 动作 | +| --- | --- | --- | +| draft→active | 检验报告满足:\|IC\|≥0.02 且 \|IR\|≥0.3(或 metrics_v2 后 t 显著) | 进策略可选列表 | +| active→watch | 滚动 63 日 IR < 0,或滚动 63 日 \|IC\| 较全样本减半 | 因子库标黄,不影响策略运行 | +| watch→retired | 连续两个观察窗(126 日)仍衰减,或人工 | 从策略可选列表隐藏(已引用它的策略配置**不破坏**,仅新建不可选) | +| retired→draft | 人工重新编辑 | 重走检验 | + +### 8.2 衰减监控 + +- 计算:后台低频任务(日线收盘后),对 active/watch 因子重算滚动 63 日 Rank IC/IR——复用检验引擎同一条 IC 计算路径。 +- 展示:因子库 tab 状态列 + P4 的 `lifecycle` 看板(衰减榜:谁最近不行了)。 +- 通知:首版只做页面内徽章;接通知服务(`NotificationFormatter` 继承点)为可选后续,防打扰。 + +## 9. 后端支撑映射 + +| 本方案功能 | 后端依赖 | 旧文章节 | 代码落点 | +| --- | --- | --- | --- | +| 检验 tab(§3) | 无新依赖(P0 纯前端) | — | — | +| 因子库(§4) | L-REG 注册表 + `GET /api/factors` | §2、§10 | 【设计】`app/factors/registry.py`(三处合一:factor.py:36 + scoring.py 虚拟因子 + pipeline ENRICHED_COLUMNS) | +| 编辑器(§5) | DSL 编译器 + `POST /api/factors/validate` | §3、§10 | 【设计】`app/factors/dsl.py` | +| 判读升级(§3.5) | metrics_v2(t_newey_west 等) | §6 | factor.py `_calc_ic` 扩展(现状 `factor.py:738-750`) | +| 组合(§6) | FactorCompositeSpec + scoring 桥 | §8 | scoring.py `VIRTUAL_SCORING_DEPENDENCIES`(现状 `scoring.py:13`,写路径自动化) | +| 生命周期(§8) | 衰减重算任务 + 状态字段 | §6(IC 口径复用) | 【设计】注册表状态字段 + 低频任务 | + +## 10. 路线图(PR 切分) + +> 与旧文 §14 的十个 PR 合并视图:数据线四个 PR(st_history / delisted_kline / industry_pit / index_ext)独立并行,不受本路线图阻塞。 + +### P0 — 因子页拆分 + 检验 UX(1 个 PR,纯前端) + +- 范围:§2 导航/路由/重定向 + §3.9 迁移清单 7 项 + §3.2–§3.7 UX 六项。 +- 依赖:无。 +- 验收:`/factors` 可用;旧链接 301 语义跳转;候选抽屉两页可用;六项 UX 逐项截图;窄屏检查;`pnpm build` 过;queryKeys 无变更。 +- 回滚:revert 单 PR 即回到现状,无数据迁移。 + +### P1 — 注册表 + 因子库(2 个 PR:后端 / 前端) + +- 后端:L-REG 注册表(旧文 §2,FACTOR_COLUMNS 由注册表生成、对外常量保留兼容别名;scoring.py 改读注册表、函数签名不变)+ `GET /api/factors`。 +- 前端:因子库 tab 上线(§4)。 +- 验收:注册表生成结果与现 61 因子逐项一致(快照测试);旧候选方案引用的历史 id 全部可读。 + +### P2 — DSL + 编辑器(2 个 PR) + +- 后端:DSL 编译器 + validate/试算端点(旧文 §3,含 E001–E016)。 +- 前端:编辑器 tab(§5)。 +- 验收:错误因子公式全路径测试;试算与检验引擎同源计算对拍一致。 + +### P3 — metrics_v2 + 组合 + 判读升级(2 个 PR) + +- 后端 a:metrics_v2(NW t 值等,旧文 §6;测试=黄金参考向量 + 可选 dev 组 statsmodels 对拍)。 +- 后端 b+前端:FactorCompositeSpec + scoring 桥自动化 + 组合 tab(§6)+ 结果判读切服务端(§3.5)。 +- 验收:组合因子在策略评分中可选且数值路径唯一;DSR/BH-FDR 数值测试过。 + +### P4 — 生命周期(1 个 PR,可延后) + +- §8 状态机 + 衰减任务 + 巡检看板。 +- 验收:衰减告警可复现(构造样本);retired 因子不破坏存量策略配置(兼容测试)。 + +## 11. 验证矩阵(对齐 CONTRIBUTING §9) + +| 改动 | 最低验证 | +| --- | --- | +| P0 前端 | `pnpm build`;手工检查:加载/空/错/禁用/无权限五态、预设切换与自定义态、搜索不丢选择(回归本轮已验证的行为)、旧链接重定向、候选抽屉双页、窄屏 | +| P1 注册表 | 后端 pytest:注册表快照=现 61 因子、历史 id 兼容、scoring 行为不变(对拍测试);API 契约测试 | +| P2 DSL | 编译器单测(合法/非法公式全错误码)、防未来函数(E005 用例)、试算与检验同源对拍 | +| P3 统计 | NW/BH-FDR/DSR 数值断言(黄金向量);组合因子端到端:注册→策略评分可选→回测出数 | +| P4 生命周期 | 衰减触发用例、retired 兼容用例、低频任务不阻塞启动 | +| 全阶段 | `git diff --check`;每 PR 附界面证据(截图覆盖改动前后) | + +## 12. 兼容性与风险 + +| 项 | 影响 | 对策 | +| --- | --- | --- | +| 旧链接 `/backtest?tab=factor` | 收藏/外链失效 | Navigate 重定向保参(mining 先例) | +| 候选方案存储 | 无 schema 变化 | 仅抽屉组件提取,存储与"载入复测"链路不动 | +| 因子 id | 不变 | P0 不碰 factor.py;P1 注册表生成需逐项快照对齐 | +| `router.tsx`/`Layout.tsx` 高冲突热点 | 上游合并冲突面 | diff 最小化:只加一条路由/一个导航项,不动既有结构 | +| P0 判读阈值是经验值 | 可能误判"有效" | UI 明示"经验规则";P3 换服务端显著性后降级为兜底 | +| 方向箭头取自样本内 IC 符号 | 过拟合误读 | tooltip 注明"样本内方向,历史不代表未来" | +| 编辑器产生大量低质因子 | 注册表膨胀 | draft 不进策略可选列表;必须过完整检验才 active | + +## 13. 不做什么(YAGNI,续旧文 §16) + +1. 因子市场/分享/导入第三方因子包——无真实需求前不做。 +2. 因子自动再训练/自动调参流水线——自动权重窗口已是点时滚动,更细即过拟合。 +3. 实时因子流(盘中逐 tick 因子值)——研究线是日频截面,实时因子属于监控线的事。 +4. 跨市场(港美股)因子——数据源与交易规则均不支持。 +5. 因子血缘图谱可视化——注册表依赖列字段已可追溯,图形化为炫技。 +6. 把挖掘页合并进因子页——任务型界面独立存在(§7 裁定)。 + +## 14. 完成标准 + +本方案"完成"= 路线图 P0–P3 全部合入且: + +1. `/factors` 四 tab(检验/库/编辑器/组合)可用,旧链接可达; +2. 一个不懂量化的用户按 §3.1 引导能独立跑出一次检验并读懂结论句(可用性自测标准); +3. 一个自定义公式因子走完 编辑→校验→试算→注册→检验→进策略评分 全链路无断点; +4. 全程未产生第二套计算逻辑(注册/编辑器/组合均编译到同一条 Polars/检验路径); +5. P0–P3 各 PR 验收标准逐项通过,界面证据与测试结果真实可查。 diff --git a/docs/factor-system-design.md b/docs/factor-system-design.md new file mode 100644 index 0000000..2c7e4f2 --- /dev/null +++ b/docs/factor-system-design.md @@ -0,0 +1,842 @@ +# 因子体系专业化设计(提案) + +> **状态声明**:本文是设计提案,**尚未实现**。凡标注【现状】的条目引用当前仓库真实代码(基于 main@2ce8b4b1),可直接核对;凡标注【设计】的条目是目标契约,**不得当作已存在的 API 导入或调用**(遵循 `docs/secondary-development.md` 第 1 节的状态区分要求)。 +> +> 全部【现状】引用已于 2026-09-04 逐条核对,Polars API 与依赖可行性已实测(polars 1.40.1),验证记录见 §17;一处初稿引用错误(pipeline.py:1738→1795)已修正。 +> +> 涉及改动均按二次开发分级标注(L1 配置 / L2 扩展点 / L3 核心源码修改,见 `docs/secondary-development.md` 第 2 节)。 + +--- + +## 0. 设计目标 + +1. **因子定义单一事实源**:公式、元数据、计算、测试同处一地,可审计、可版本化。 +2. **研究结论可辩护**:宇宙可解释、风险调整显式、统计检验完备、指标口径唯一。 +3. **策略接入零摩擦**【用户核心诉求】:因子研究成果(含用户自定义因子、复合因子、挖掘产物)以统一形态被策略评分、选股、回测、监控四端消费,一处定义、处处生效。 +4. **不推倒重来**:挖掘框架(purge/embargo/嵌套样本外)、enriched 列体系、虚拟评分机制全部保留,只做补层和收口。 + +--- + +## 1. 分层总览与现状映射 + +| 层 | 目标模块 | 现状代码 | 动作 | 分级 | +| --- | --- | --- | --- | --- | +| L-REG 因子注册表 | `app/factors/registry.py`【设计】 | `backtest/factor.py:36` FACTOR_COLUMNS + `strategy/scoring.py` 虚拟因子 + `indicators/pipeline.py` ENRICHED_COLUMNS | 三处合一收口 | L3(重构) | +| L-DSL 表达式层 | `app/factors/dsl/`【设计】 | 无(`strategy/custom_signals.py` 白名单模式可借鉴) | 新增 | L2 | +| L-UNI 宇宙构建 | `app/factors/universe.py`【设计】 | 无(tradable/limit_up_locked/listing_date 素材已存在) | 新增 | L2 | +| L-NEU 风险调整 | `app/factors/neutralize.py`【设计】 | 无(`get_index_daily`、行业 preset、`share_capital.py` 素材已存在) | 新增 | L2 | +| L-INF 统计检验 | `app/factors/stats.py`【设计】 | 无 | 新增 | L2 | +| L-MET 指标统一 | `app/factors/metrics.py`【设计】 | `backtest/engine.py` 三种 Sharpe(:2899/:2993/:3116) | 收敛 + 版本化 | L3(热点) | +| L-CMP 复合因子→策略 | `app/factors/composite.py`【设计】 | `strategy/scoring.py` 虚拟评分字段机制【现状·已可用】 | 扩展既有机制 | L2→L3 接线 | +| 数据契约 | provider dataset 声明 | 无 ST 历史/退市股/点时行业 | 新增 dataset | L1(YAML)+provider 实现 | + +模块落点说明:新建 `app/factors/` 包而不是塞进 `backtest/`,因为因子目录、宇宙、中性化被选股(`strategy/`)、回测(`backtest/`)、挖掘(`backtest/mining.py`)三方消费,放任一方都会造成反向依赖(违反 CONTRIBUTING 2.3 模块边界)。 + +--- + +## 2. 因子注册表(L-REG) + +### 2.1 现状问题 + +因子元数据目前分散在四处,互相漂移无感知: + +| 位置 | 内容 | 缺陷 | +| --- | --- | --- | +| `backtest/factor.py:36-109` | 62+ 因子目录(id/label/group/desc) | desc 是自然语言,与计算无绑定 | +| `strategy/scoring.py:13-51` | VIRTUAL_SCORING_DEPENDENCIES | 虚拟因子的依赖声明,但与因子目录是两套清单 | +| `strategy/scoring.py:80+` | `scoring_value_expr` | 虚拟因子的 Polars 表达式,硬编码 if-else 分发 | +| `strategy/scoring.py:53-66` | `_ROLLING_SCORING_WARMUP` | 预热窗口第三套清单 | + +### 2.2 FactorSpec 完整 schema【设计】 + +```python +@dataclass(frozen=True) +class FactorSpec: + id: str # 全局唯一,^f_[a-z0-9_]{1,40}$;内置因子保持现有列名不变(如 momentum_20d) + version: int # 因子语义版本;公式变更必须 +1,进缓存键 + label: str # 中文显示名 + group: str # 展示分组(沿用现有:动量/均线偏离/超买超卖/趋势/波动率/量价/…) + kind: Literal["base", "virtual", "composite", "custom"] + # base: 已物化在 enriched parquet + # virtual: 按需由 base 列编译计算(如 ma5_bias) + # composite: 复合因子(见 §8) + # custom: 用户 DSL 因子(见 §3) + expr_factory: Callable[[frozenset[str]], pl.Expr | None] | None + # virtual/custom 的计算:输入可用列集合,依赖不完整返回 None(fail-closed) + formula_text: str # 人类可读公式;virtual 由表达式自动生成,base 手写并配特征化测试锁定 + dependencies: frozenset[str] # 展开到 enriched base 列(自递归展开 composite/custom 依赖) + direction: Literal["high", "low", "none"] + # 预期信号方向;进复合因子默认权重与 UI 排序展示 + unit: Literal["ratio", "pct", "score", "count", "days", "currency", "none"] + # 单位口径,UI 格式化与 sanity check 用(禁止"数值<1 乘 100"启发式) + warmup_bars: int # 历史窗口需求(交易日数);对齐 _ROLLING_SCORING_WARMUP 语义 + pit: bool # 是否点时数据依赖(财务因子 = True) + pit_source: Literal["financial_announce", "share_capital_announce", "none"] + asset_types: frozenset[Literal["stock", "etf"]] + incremental_safe: bool # 盘中增量路径(pipeline.py:1795)能否复算;False 则盘中不含该列 + scale_free: bool # 跨标的可比(可直接截面排序);如 atr_14 原值 = False,atr_pct = True + null_policy: Literal["keep", "drop_row"] + # 研究路径默认 keep(不填零,沿用 fundamentals.py 纪律) + stability: Literal["stable", "experimental", "deprecated"] + tags: tuple[str, ...] # 风格标签:"momentum"/"value"/"size"/"lottery"/"liquidity"/… +``` + +注册表 API(仅内部 Python 接口,不新增 HTTP): + +```python +register_factor(spec) # 启动期注册;重复 id 且 version 未增 → 拒绝启动(fail-closed) +get_factor(fid) -> FactorSpec +all_factors(asset_type=None, stable_only=False) -> list[FactorSpec] +factor_dependencies(fids) -> frozenset[str] # 递归展开 +factor_value_exprs(available_cols) -> dict[str, pl.Expr | None] +``` + +### 2.3 迁移策略【设计】 + +1. **特征化测试先行**:固定样本(≥50 只股票 × 含除权日、停牌日、涨跌停日的窗口)快照当前全部 62+ 因子在 enriched 与 `scoring_value_expr` 两条路径的输出,重构后断言逐位一致。扩展 `backend/tests/backtest/test_factor_library_v2.py`。 +2. `VIRTUAL_SCORING_DEPENDENCIES`、`scoring_value_expr` 的 if-else 分发、`_ROLLING_SCORING_WARMUP` 逐一改读注册表,**函数签名不变**(`scoring.py` 对外契约保持)。 +3. `FACTOR_COLUMNS` 改由注册表生成,`factor.py` 对外常量保留为兼容别名。 +4. desc 公式与 `formula_text` 不一致处,以特征化测试输出的实际计算为准修正文档。 + +### 2.4 因子分类学与补全清单【设计】 + +现有 11 组保留;补全以下专业常用因子(标注数据依赖,缺数据不注册、不静默): + +| 族 | 建议新增 | 公式要点 | 依赖 | +| --- | --- | --- | --- | +| 动量 | 特质动量 `f_idio_mom_20d` | 个股日收益对基准收益回归残差的 20 日累计 | 指数日K(已有) | +| 动量 | 52 周新高接近度 `f_near_high_52w` | close / 250 日最高 close − 1 | 已有 | +| 反转 | 短期反转 `f_rev_5d` | −momentum_5d(direction=low 的语义化封装) | 已有 | +| 波动 | 已实现波动偏度差、高低频波动分解 | 简化:`f_vol_ratio_short_long` = vol_5d/vol_60d | 已有 | +| 波动(条件) | 条件波动率 `f_ewma_vol` | RiskMetrics EWMA(λ=0.94) 条性日波动年化;比等权 rolling_std 对近端冲击响应更快,低成本低争议 | 已有(`ewm_std` 向量化) | +| 波动(条件) | 波动的波动 `f_vol_of_vol_60d` / 波动区制 `f_vol_regime` | 波动率的滚动 std / EWMA 波动 ÷ 长期波动;区分"高波市场"与"波动突变",A 股风格切换敏感因子 | 已有 | +| 波动(条件) | GARCH(1,1) 条件波动 `f_garch_vol` | **标记 experimental、按需实现**:逐 symbol 递归拟合与全向量化管线冲突,若引入必须走 `numba_runtime.py`【现状】路径或 numba/arch 依赖,先以 EWMA 交付(日频下 EWMA ≈ GARCH 的 90% 价值) | 已有 + numba | +| 量价 | 量价背离 `f_pv_divergence_20d` | −vol_price_corr_20d 语义化 | 已有 | +| 流动性 | 非流动性变化 `f_amihud_chg` | amihud_20d / amihud_60d − 1 | 已有 | +| 规模 | 流通市值对数 `f_log_float_mv` | ln(历史流通股本 × raw_close)【点时股本,share_capital.py 已有】 | 已有 | +| 价值 | `f_ep_latest`、`f_ep_ttm` | 1/PE 口径(E/P 比 PE 统计性质更好);ttm 需财务四表滚动 | 财务(已有);ttm 需扩展 | +| 质量 | 应收/存货增速差、商誉/净资产 | 财务表字段 | 财务(需字段核对) | +| 涨停 | 首板/连板区分、炸板后回封率 | 基于 consecutive_limit_ups、炸板列 | 已有 | +| 财务 | SUE(盈余惊喜) | (E_t − E_{t-4}) / σ(ΔE, 4期),公告日口径 | 财务历史(已有 `_merge_report_history`) | + +--- + +### 2.5 全量因子目录映射表(代码推导,PR-6 注册蓝本) + +下表覆盖现有全部 61 个因子【现状:`factor.py:36-109`】,kind 与依赖由 `scoring.py:13-51` VIRTUAL_SCORING_DEPENDENCIES 逐字推导(virtual = 表中出现的键,base = 未出现即已物化列),运行时预热取自 `scoring.py:53-66` `_ROLLING_SCORING_WARMUP` 代码值。direction 列为**建议初值**(high=因子值大预期收益高;"待标定"= 振荡类/方向依市场状态,PR-6 注册时依 IC 实证方向标定并允许研究层覆盖)。 + +| 因子 | 组 | kind | 依赖(virtual 展开) | 预热 | direction | +| --- | --- | --- | --- | --- | --- | +| momentum_5d/10d/20d/30d/60d | 动量 | base | — | 全局120日 | high | +| change_pct | 动量 | base | — | 全局 | high | +| ma5..60_bias(5个) | 均线偏离 | virtual | {close, maN} | 全局 | high | +| ema5..60_bias(5个) | 均线偏离 | virtual | {close, emaN} | 全局 | high | +| rsi_6 / rsi_14 / rsi_24 | 超买超卖 | base | — | 全局 | 待标定 | +| macd_hist | 趋势 | base | — | 全局 | 待标定 | +| macd_dif_pct / macd_dea_pct / macd_hist_pct | 趋势 | virtual | {close, macd_dif/dea/hist} | 全局 | high | +| kdj_k / kdj_d / kdj_j | 趋势 | base | — | 全局 | 待标定 | +| boll_position | 趋势 | virtual | {close, boll_upper, boll_lower} | 全局 | high | +| annual_vol_20d | 波动率 | base | — | 全局 | low | +| atr_14 | 波动率 | base | — | 全局 | 待标定 | +| atr_pct | 波动率 | virtual | {close, atr_14} | 全局 | low | +| amplitude | 波动率 | base | — | 全局 | low | +| boll_width | 波动率 | virtual | {ma20, boll_upper, boll_lower} | 全局 | low | +| vol_ratio_5d | 量价 | base | — | 全局 | 待标定 | +| vol_ratio_10d | 量价 | virtual | {volume} | 11 | 待标定 | +| vol_trend_5_10 | 量价 | virtual | {vol_ma5, vol_ma10} | 全局 | high | +| turnover_rate | 量价 | base | — | 全局 | low | +| turnover_ratio_5d | 量价 | virtual | {turnover_rate} | 6 | high | +| log_amount | 量价 | virtual | {amount} | 全局 | 待标定 | +| amount_ratio_5d | 量价 | virtual | {amount} | 6 | high | +| gap_return | 价格位置 | virtual | {open, prev_close} | 全局 | 待标定 | +| intraday_return | 价格位置 | virtual | {open, close} | 全局 | 待标定 | +| close_position | 价格位置 | virtual | {high, low, close} | 全局 | 待标定 | +| distance_to_high_60d | 价格位置 | virtual | {close, high_60d} | 全局 | high | +| distance_from_low_60d | 价格位置 | virtual | {close, low_60d} | 全局 | high | +| vwap_bias | 价格位置 | virtual | {close, volume, amount} | 全局 | 待标定 | +| max_ret_20d | 收益形态 | virtual | {close} | 21 | low | +| ret_skew_20d | 收益形态 | virtual | {close} | 21 | low | +| up_days_20d | 收益形态 | virtual | {close} | 21 | 待标定 | +| amihud_20d | 流动性 | virtual | {close, amount} | 21 | low | +| turnover_z_60d | 流动性 | virtual | {turnover_rate} | 61 | 待标定 | +| vol_price_corr_20d | 量价 | virtual | {close, volume} | 21 | 待标定 | +| vol_trend_5_60 | 量价 | virtual | {volume} | 60 | high | +| limit_up_count_20d | 涨停基因 | virtual | {consecutive_limit_ups} | 21 | high | +| limit_up_count_60d | 涨停基因 | virtual | {consecutive_limit_ups} | 61 | high | +| pb_latest | 财务 | base(点时联表) | — | 公告日机制 | low | +| roe_latest | 财务 | base(点时联表) | — | 公告日机制 | high | +| gross_margin_latest | 财务 | base(点时联表) | — | 公告日机制 | high | +| net_margin_latest | 财务 | base(点时联表) | — | 公告日机制 | high | +| revenue_yoy_latest | 财务 | base(点时联表) | — | 公告日机制 | high | +| net_income_yoy_latest | 财务 | base(点时联表) | — | 公告日机制 | high | +| debt_ratio_latest | 财务 | base(点时联表) | — | 公告日机制 | low | + +计数核对:virtual 35 + base 非财务 19 + 财务 7 = 61,与 FACTOR_COLUMNS 一致。base 因子的研究预热由 `FACTOR_WARMUP_DAYS=120`(`factor.py:111`【现状】)统一承担;财务因子 pit=true、pit_source=financial_announce。 + +## 3. 因子表达式层(L-DSL) + +### 3.1 语法与算子表【设计】 + +表达式 = `expr ::= operand | expr op expr | func(expr[, expr[, const]])`;中缀 + 函数调用,无变量赋值、无循环。 + +**操作数**:基准列(open/high/low/close/volume/amount/turnover_rate/prev_close/raw_close)、白名单指标列(注册表中 base 因子)、已注册因子 id(virtual/composite/custom,递归内联展开)、数值常量。 + +**时序算子**(`over("symbol")`,窗口 n ∈ [2, 512],全部只向后看): + +| 算子 | 语义 | Polars 编译 | +| --- | --- | --- | +| `ts_mean(x,n)` / `ts_std(x,n)` / `ts_sum(x,n)` | 滚动均值/样本标准差/求和 | `rolling_mean/std/sum(n)` | +| `ts_max(x,n)` / `ts_min(x,n)` | 滚动极值 | `rolling_max(n)` / `rolling_min(n)` | +| `ts_delta(x,n)` | x − ts_delay(x,n) | `x - x.shift(n)` | +| `ts_delay(x,n)` | n 期前的值(n ∈ [1, 512],**禁止负数**——负数即未来函数,编译期报错) | `x.shift(n)` | +| `ts_rank(x,n)` | 当期值在滚动窗口内的分位 | `rolling_rank(n)`【已验证:polars 1.40.1 存在且行为正确,§17】 | +| `ts_zscore(x,n)` | (x − ts_mean)/ts_std | 组合表达式 | +| `ts_corr(x,y,n)` / `ts_cov(x,y,n)` | 滚动相关/协方差 | 顶层函数 `pl.rolling_corr(x,y,window_size=n)` / `pl.rolling_cov`【已验证:Expr 上无此方法,必须走顶层函数,§17】 | +| `ts_quantile(x,n,q)` | 滚动分位(q ∈ (0,1) 常量) | `rolling_quantile` | +| `decay_linear(x,n)` | 线性衰减加权均值(近端权重大) | 手写权重组合表达式 | + +**v1 不提供的时序算子及原因**:`ts_argmax/ts_argmin`——Polars 无向量化实现(`rolling_map` 为逐窗 Python 回调,违反向量化约束,已验证 Expr 无 `rolling_arg_max`);"距极值天数"类需求以具体因子的组合表达式实现(如 `distance_to_high_60d` 模式),确有高频需求再经 numba 扩展。 + +**截面算子**(按日期分组,逐日横截面): + +| 算子 | 语义 | 说明 | +| --- | --- | --- | +| `rank(x)` | 横截面百分位排名 ∈ (0,1] | null 不参与排名 | +| `zscore(x)` | 横截面 (x−μ)/σ | σ=0 → null | +| `winsorize(x,k)` | 截尾至 μ±kσ(k ∈ [1,6] 常量,默认 3) | 截面口径 | + +**算术/工具**:`+ − * /`(除零 → null)、`log abs sign sqrt min max power(x,c) clamp(x,lo,hi)`、三元 `if_else(cond, a, b)`、比较与逻辑 `> >= < <= == != and or not`(产出布尔,配合 if_else)。 + +### 3.2 校验规则(编译期全部强制)【设计】 + +1. 标识符必须在基准列/白名单/已注册因子内,否则报错(防注入,沿用 `custom_signals.py` 白名单哲学)。 +2. `ts_delay`/`ts_delta` 的 n ≥ 0;任何窗口 n ∈ [2, 512];`power` 指数 |c| ≤ 4;AST 深度 ≤ 12;表达式 token 数 ≤ 200。 +3. 常量折叠后若产生 `x/0` 类静态除零 → 编译失败。 +4. 依赖列集合 = 递归展开;warmup_bars = max(各 ts 算子窗口);超出即注册表标记,研究 UI 提示所需历史长度。 +5. 产出类型必须为数值或布尔(布尔经 `cast` 视为 0/1)。 +6. **禁止未来引用的总闸**:所有时序算子 shift 语义已内建,语法层不存在负 shift;code review checklist 补一条"新增算子必须只向后看"。 + +### 3.3 编译流水线【设计】 + +`text → tokenizer → Pratt 解析 → AST → 语义检查(§3.2) → 依赖/预热推导 → Polars Expr 工厂`。产出缓存(表达式文本 → 编译产物 LRU,键含依赖列版本);编译失败返回结构化错误(位置 + 原因),不抛裸异常。 + +### 3.4 形式文法(EBNF)与错误码目录【设计】 + +```ebnf +expr = or_expr ; +or_expr = and_expr { "or" and_expr } ; +and_expr = cmp_expr { "and" cmp_expr } ; +cmp_expr = add_expr [ (">" | ">=" | "<" | "<=" | "==" | "!=") add_expr ] ; +add_expr = mul_expr { ("+" | "-") mul_expr } ; +mul_expr = unary { ("*" | "/") unary } ; +unary = "-" unary | primary ; +primary = NUMBER | IDENT | func_call | "(" expr ")" ; +func_call = IDENT "(" [ arglist ] ")" ; +arglist = expr { "," expr } ; +(* IDENT:基准列/白名单指标列/已注册因子 id/算子名;NUMBER:十进制与负号经 unary 处理 *) +``` + +运算符优先级由产生式层级固定(or < and < 比较 < 加减 < 乘除 < 一元负号 < 原子),与 Python/JS 语义一致,降低用户迁移成本。 + +**错误码目录**(编译与运行校验的唯一错误词汇表,API/编辑器/UI 共用): + +| 码 | 含义 | 触发 | +| --- | --- | --- | +| E001 | 未知标识符 | IDENT 不在白名单/注册表 | +| E002 | 未知函数 | 函数名不在算子表 | +| E003 | 参数数量/类型不符 | 算子签名不匹配(含常量参数位置) | +| E004 | 窗口越界 | n∉[2,512] 或 q∉(0,1) | +| E005 | 负 shift | ts_delay/ts_delta 的 n<0 | +| E006 | 嵌套深度超限 | AST 深度>12 | +| E007 | 规模超限 | token 数>200 | +| E008 | 静态除零 | 常量折叠检出分母恒 0 | +| E009 | 产出类型非法 | 非数值/布尔 | +| E010 | power 指数越界 | \|c\|>4 | +| E011 | winsorize k 越界 | k∉[1,6] | +| E012 | 循环引用 | 因子依赖成环(含自定义因子链) | +| E013 | 依赖列不可用 | 面板缺列(运行时) | +| E014 | 语法错误 | 解析失败(附位置) | +| E015 | 预热不足 | warmup > 研究窗口(运行时) | +| E016 | 常量表达式 | 无任何标识符,拒绝保存 | + +错误响应统一结构:`{"code": "E001", "message": "未知标识符: clos", "position": {"offset": 12, "line": 1}, "detail": {...}}`。 + +### 3.5 用户因子生命周期与存储【设计】 + +- 存储路径:`data/user_data/custom_factors/*.json`(对齐 custom_signals 目录约定),schema: + +```json +{ + "id": "uf_my_rev", // ^uf_[a-z0-9_]{1,40}$,前缀与 csg_ 同哲学 + "version": 1, + "label": "我的反转因子", + "formula": "rank(-ts_sum(change_pct, 5))", + "direction": "low", + "description": "5 日累计涨幅的截面倒数", + "created_at": "2026-09-04T00:00:00", + "updated_at": "2026-09-04T00:00:00" +} +``` + +- 生命周期:草稿(编辑器内试算,不落盘)→ 保存(编译通过 + 试算有非空输出才可保存,fail-closed)→ 引用(策略 scoring / 因子研究 / 复合因子)→ 版本化(公式变更 version+1,旧结果按 version 键隔离)→ 删除(有引用时列出引用方并二次确认,对齐策略删除的 fail-closed 要求)。 +- 加载失败的单个文件只禁用该因子并提示,不影响启动与其他因子(对齐 plugins 隔离要求,CONTRIBUTING 第 4 节)。 + +--- + +## 4. 宇宙构建(L-UNI) + +### 4.1 UniverseSpec 完整 schema【设计】 + +```python +@dataclass(frozen=True) +class UniverseSpec: + exclude_suspended: bool = True # 停牌(tradable 矩阵口径,matrix.py:1476+) + exclude_limit_locked: bool = True # 调仓时点一字涨停不可买入者(buy_limit_up 口径) + exclude_st: bool = False # 非点时(今日名称),开启时报告中必须出现降级注记 + min_listing_days: int = 0 # 次新剔除;listing_date 已入库未使用(api/data.py:765) + min_amount_quantile: float | None = None # 流动性过滤:当日成交额截面分位下限 (0,1) + cap_quantile_range: tuple[float, float] | None = None # 市值分位区间;依赖 f_log_float_mv + max_names: int | None = None # 截面数量上限;排序键由研究上下文显式传入(因子值或复合分,不隐式默认),用于微型宇宙研究 +``` + +### 4.2 执行语义【设计】 + +- 按日生成 `universe[date] -> set[symbol]`,**每个过滤条件独立短路、独立计数**,产出 `filter_stats`(每日各过滤器剔除数),研究报告展示"宇宙从 5200 → 4980 → 4890"漏斗。 +- `universe_id = sha256(canonical_json(spec))[:12]`,进一切下游缓存键。 +- 过滤顺序固定(先便宜的列过滤,后需联表的),顺序本身进 canonical_json。 +- **as_of 语义**:宇宙内一切判定只用当日及以前数据。ST 例外必须显式标注 `degraded: ["st_not_point_in_time"]` 并在报告 UI 渲染黄条。 + +### 4.3 数据缺口降级矩阵【设计】 + +| 过滤器 | 数据缺失时行为 | +| --- | --- | +| exclude_suspended / exclude_limit_locked | 素材必在(enriched 必算列);缺失 = 数据本身异常 → fail 报错 | +| exclude_st | instruments 名称缺失 → 过滤器跳过 + 降级注记(不静默假装过滤了) | +| min_listing_days | listing_date 缺失的标的视为"不满足"剔除(保守),计数展示 | +| cap_quantile_range | 历史股本缺失标的退出该过滤(不参与分位),降级注记 | +| 退市股(未来) | 依赖新 dataset(§9);无数据源时 universe 定义退化为"当前上市 ∪ 本地历史",报告中永久注记幸存者偏差警示 | + +--- + +## 5. 风险调整与基准(L-NEU) + +### 5.1 NeutralizationSpec【设计】 + +```python +@dataclass(frozen=True) +class NeutralizationSpec: + benchmark: str | None = "000001.SH" # 上证指数(index_const.py:12【现状】核心四只之一);扩展指数见 §9 + return_basis: Literal["raw", "excess"] = "excess" # IC/分层收益口径 + method: Literal["none", "industry_demean", "industry_zscore", "regression_industry_size"] = "none" + winsorize_sigma: float | None = 3.0 # 因子值截面截尾;None = 不截尾 + # neutralization_id = sha256(canonical_json)[:12],进缓存键 +``` + +### 5.2 方法规格【设计】 + +- 超额收益:`r_ex = r_stock − r_bench`(基准同日收益;基准停市日沿用最近交易日,日历由数据轴驱动)。 +- `industry_demean`:`f' = f − mean_ind(f)`(THS 行业一级,ext preset 快照)。 +- `industry_zscore`:组内标准化 `f' = (f − μ_ind)/σ_ind`(σ=0 组 → null)。 +- `regression_industry_size`:`f ~ 1 + 行业哑变量 + log_float_mv` 的残差(逐日 OLS,Polars 表达式实现,n<30 或共线 → 回退 demean + 注记)。 +- 固定管线顺序:`宇宙过滤 → winsorize → 中性化 → 标准化(zscore 或 rank)`;顺序进 spec 哈希。 +- **行业快照局限**(当前归属回填历史)写入 `degraded` 注记并在报告显示;点时行业表到位后(§9)仅切换数据源,spec 不变。 + +### 5.3 报告口径并列【设计】 + +IC 报告同时输出三列:`原始 / 超额 / 超额+中性化`,默认排序以最后一列为准——旧结论可查,新结论更严,不静默替换。 + +--- + +## 6. 统计检验(L-INF) + +**依赖原则**:后端当前无 scipy/statsmodels(已验证,§17),运行时保持零新增第三方依赖——NW/BH-FDR/DSR 全部以 numpy 手写实现(各约 20-40 行);statsmodels 仅允许加入 uv dev 依赖组用于测试对拍,不进运行时 import。 + +### 6.1 统计量精确定义【设计】 + +| 统计量 | 定义 | 备注 | +| --- | --- | --- | +| Rank IC | 逐日 Spearman(factor_t, fwd_ret_{t→t+h}),现有口径不变(factor.py:740-750) | — | +| IC t 值(朴素) | `t = mean(IC) / (std(IC, ddof=1)/√N)` | 仅作对照展示 | +| IC t 值(NW) | Newey-West HAC 稳健标准误,滞后 `L = h`(h 日前瞻收益使 IC 序列存在 h−1 阶移动平均自相关) | **主口径**;numpy 手写 Bartlett 核加权,测试用固定黄金参考向量 + 可选 dev 组 statsmodels 对拍 | +| ICIR | mean(IC)/std(IC),已有 | — | +| IC 自相关 & 半衰期 | ACF(1..10);半衰期 = ACF 首次 < 0.5 的滞后(线性插值);无收敛 → null | 换手率预期管理 | +| 分层单调性 | Spearman(组序号, 组均超额收益) + 线性趋势斜率 t 值 | 判定"梯子是否成立" | +| 多空 t 值 | 顶组−底组日超额收益序列的 NW t | 滞后 = 调仓周期的收益重叠阶数 | +| BH-FDR q 值 | 对 optimizer/mining 排行榜全体 p 值(每行 = 其 OOS/IS 最优组合的 IC 或收益 t 值双尾 p)做 Benjamini-Hochberg,q_i = min_{j≥i}(N·p_j/j) 单调化 | 排行榜级,不进单因子报告 | +| Deflated Sharpe | Bailey-López de Prado:以试验次数 N(挖掘 trial 预算已计数,mining.py:1041+)与偏度峰度校正 SR₀,DSR = Φ((SR−SR₀)·√(T−1) / √(1−γ̂₃SR+((γ̂₄−1)/4)SR²)) | 挖掘晋升报告展示"考虑搜索后的置信" | +| 覆盖率/换手率 | 已有(factor.py:1123-1166),补充宇宙过滤后口径 | — | + +### 6.2 显著性标注约定【设计】 + +|t| < 1.645 无标注;≥1.645 `*`(10%);≥1.96 `**`(5%);≥2.576 `***`(1%)。报告 UI 图标化,q ≥ 0.10 的挖掘候选禁止晋升(现有晋升门槛 mining.py:26-31 之上叠加,未达标给出具体差值)。 + +### 6.3 版本命名规则与报告完整字段【设计】 + +**方法论版本命名**(单一规则,全文档统一): + +- `factor_v3` = 本设计交付的因子研究方法论(三口径 + 统计检验 + 宇宙/中性化 spec 进键);现有 `factor_v2`(factor.py:112【现状】)结果按旧版本读取展示,不重算。 +- `metrics_v2` = §7 指标统一后的口径;与 `factor_v3` 独立演进,报告分别携带。 +- 因子个体 `version`(FactorSpec)与研究方法论版本正交:因子公式变更不改方法论版本,反之亦然。 + +**IC 研究报告完整字段定义**(`POST /api/factor-research/ic` 响应,§10 示例为其节选): + +```text +methodology_version: str # "factor_v3" +factor_id / factor_version: str / int +universe_id / neutralization_id: str # 两 spec 哈希 +universe: object # 回显生效 UniverseSpec +neutralization: object # 回显生效 NeutralizationSpec +date_range: {start, end, rebalance, n_groups} +ic: {mean, std, icir, t_naive, t_newey_west, nw_lag, significance, + half_life_days|null, acf: float[10], win_rate, coverage, n_days} +ic_decay: [{horizon, ic_mean, icir}] # 沿用现有 1/3/5 日结构 +ic_yearly: [{year, ic_mean, icir, n_days}] # 沿用现有结构 +ic_by_basis: {raw, excess, excess_neutralized} × {mean, t_newey_west} +monotonicity: {spearman, trend_t, verdict} # verdict ∈ 成立/弱/不成立 +groups: [{group, excess_return_annual, nav, turnover, n_names, + t_stat, avg_name_count}] # 每组含 t 值 +long_short: {annual_return, t_newey_west, max_drawdown, executable_short: false} +turnover_top_group: float +costs: {commission_pct, stamp_tax_pct, slippage_bps, round_trip} +universe_funnel: [{date, raw, after_suspended, after_limit_locked, + after_st|null, after_new_listing|null, after_filters}] +degraded: [str] # 如 st_not_point_in_time / industry_snapshot +warnings: [str] # 非降级类提示(预热边界、覆盖不足等) +``` + +新增字段全部带默认值,历史(factor_v2)缓存结果缺字段时前端显示为空,不报错。 + +--- + +## 7. 绩效指标统一(L-MET) + +### 7.1 唯一口径【设计】 + +- **Sharpe**:净值曲线日收益 `r_t = nav_t/nav_{t−1} − 1`,`Sharpe = mean(r)/std(r, ddof=1) × √A`;无风险利率参数 `rf_annual`(默认 0,单位/年,日化按 A 折算)。引擎三种旧口径(逐笔/仅卖出日/净值)收敛为净值口径;前两者字段保留一个版本周期并标 `deprecated_mode`。 +- **年化天数 A**:默认 243(近五年 A 股实际均值区间),`metrics_methodology_version = "metrics_v2"`;报告展示口径徽章。 +- **年化收益**:`(nav_T/nav_0)^(A/n_bars) − 1`(统一按 K 线数折算,废除 365.25 自然日混用,engine.py:2871-2874 收敛)。 +- **MaxDD**:现有算法(峰值下限 1.0)不变。 +- **基准相对新增**:`excess_annual`、`tracking_error = std(r−r_b)×√A`、`information_ratio = mean(r−r_b)/std(r−r_b)×√A`、`beta/alpha`(OLS,rf 处理同上)、`excess_win_rate`。 +- `engine.py`/`strategy.py`/`factor.py` 全部改 import `app/factors/metrics.py`,禁止本地重算(Ruff 检查加入 noqa 禁用清单之外无豁免)。 + +--- + +## 8. 复合因子与策略接入(L-CMP)——核心章节 + +### 8.1 现有桥(【现状·已可用】,设计的锚点) + +- 策略配置:`"scoring": {"factor_name": weight}` + `scoring_directions` 覆盖高低方向(`builtin/*.py` 均此形态)。 +- `scoring.py`:虚拟因子按需编译 Polars 表达式(`scoring_value_expr`)、依赖展开(`scoring_dependencies`)、预热推导(`scoring_warmup_bars`)。 +- 回测矩阵按 `score` 排序建仓(`engine.py` `max_positions` + `score_min/max`)。 +- 挖掘产物 = 因子排名组合(mining.py),候选库 `candidates.py` 已有 `factor`/`strategy` 双形态。 + +### 8.2 FactorCompositeSpec【设计】 + +```python +@dataclass(frozen=True) +class FactorCompositeSpec: + id: str # ^cf_[a-z0-9_]{1,40}$;策略 scoring 里以 "cf_xxx" 引用 + version: int + label: str + factors: tuple[CompositeMember, ...] # 1..10 个成员 + transform: Literal["rank", "zscore"] = "rank" # 成员标准化方式(截面) + weighting: Literal["manual", "equal", "icir", "max_ic"] = "manual" + auto_weight_window: int = 504 # 自动权重的滚动窗口(交易日) + direction: Literal["high", "low"] = "high" + +@dataclass(frozen=True) +class CompositeMember: + factor: str # 任意已注册因子 id(含 uf_/cf_ 前缀,禁止自引用,环检测) + weight: float | None # manual 模式必填;自动模式忽略 + direction_override: Literal["high", "low"] | None = None +``` + +**计算管线(顺序固定,进 spec 哈希)**: + +``` +宇宙(可选, 默认不过滤以兼容现有策略) +→ 各成员因子值(注册表展开,含 warmup 检查) +→ winsorize(3σ, 可关) +→ 截面 transform(rank/zscore) +→ 方向统一(low → 取负) +→ 加权求和(manual 权重归一化校验 |Σw−1|<1e-9; + icir → w_i ∝ max(ICIR_i, 0),ICIR 取 (t−auto_weight_window, t−1] 窗口——**权重只用于过去,严禁 t 日数据参与 t 日权重**; + max_ic → 同窗口 mean(rank IC) 单调权重) +→ 输出复合分 cf_xxx(rank 基础下近似 ∈ [−1,1],文档声明分布性质) +``` + +**接入策略(零引擎改动)**: + +- 复合因子注册进注册表(kind="composite"),`scoring_value_expr` 机制天然支持:策略写 `"scoring": {"cf_hotmom": 0.6, "vol_ratio_5d": 0.2, "amount": 0.2}` 即生效;依赖/预热自动递归展开进矩阵构建,选股/回测/监控三端无需感知"这是复合因子"。 +- 挖掘产物一键导出为 CompositeSpec:mining 的排名组合本来 = factors+weights,导出即 `cf_mined_`,闭合"挖掘 → 复合因子 → 策略"回路。 +- 前置校验:成员因子任一 warmup 超研究窗口 → 启动期注册成功但使用时返回明确"预热不足"错误(不产出半截分数)。 + +**策略侧引用形态(确切 JSON)**——复合因子编辑器"导出 scoring 片段"产出,直接粘贴进策略配置: + +```json +{ + "scoring": {"cf_hotmom": 0.6, "vol_ratio_5d": 0.2, "amount": 0.2}, + "scoring_directions": {"cf_hotmom": "high", "amount": "high"} +} +``` + +复合因子与普通字段混用、权重语义不变;`scoring_dependencies`/`scoring_warmup_bars` 自动递归展开(`scoring.py:91-104`【现状】机制不动,仅数据源换成注册表)。 + +### 8.3 一致性契约【设计】 + +同一 `cf_xxx` 在**因子研究(IC/分层)、选股、回测、监控**四端必须逐位同值——单测直接断言四路径对同一 (date,symbol) 的输出相等。这是 CONTRIBUTING 5.3"同一候选集和排序方向"的推广。 + +### 8.4 监控端数据流澄清【设计】 + +监控不重算复合因子:`monitor.py:1314-1322`【现状】消费的是**策略结果缓存**里的 `result.scores`(`score_min/score_max` 过滤)。因此复合因子进监控的路径 = 策略执行时算好分 → 结果缓存 → 监控读缓存。**实时行情线程零新增计算**(CONTRIBUTING 6.3 硬约束)。推论:修改复合因子定义后,必须走策略参数变更的既有失效链路(重算策略结果缓存 → 监控实例刷新),该链路已存在(CONTRIBUTING 5.1),设计只复用不新造。 + +### 8.5 自定义/复合因子的盘中行为【设计】 + +- 选股(盘后批量):`incremental_safe=True` 的成员因子照常参与当日计算。 +- `incremental_safe=False` 成员(若有):当日选股对该因子返回"预热/路径不足"的明确不可计算状态(对齐 CONTRIBUTING 5.1"空值不得伪装成零分"),UI 标注原因;**不降级用部分成员算半截复合分**。 +- 盘中增量路径(`pipeline.py:1795` `compute_enriched_today`【现状】):自定义与复合因子默认不进入(§12 缓存策略),分时选股若引用则同样返回不可计算状态,盘后恢复。 + +--- + +## 9. 数据契约扩展(provider dataset) + +新增 dataset 声明(capabilities.py 注册表 + 对应 provider 实现,均【设计】): + +| dataset | 内容 | 解锁能力 | 无数据源时 | +| --- | --- | --- | --- | +| `st_history` | 点时风险警示状态 | 历史涨跌停幅度修正、宇宙 ST 点时过滤 | 涨跌停用当前名推断 + 注记(现状) | +| `delisted_kline` | 退市标的日 K + 退市维表 | 幸存者偏差修复(宇宙回补) | 报告永久幸存者注记 | +| `industry_pit` | 点时行业归属 | 中性化升级为点时 | 行业快照 + 注记(现状) | +| `index_ext` | 扩展指数日 K | 基准升级(当前限核心四只,index_const.py:12-15【现状】) | 基准限核心四只 | + +各 dataset 完整字段 schema(provider 归一后落 Parquet,命名对齐现有 normalized 契约): + +**`st_history`**(分区 `data/parquet/st_history/`,按年): + +| 字段 | 类型 | 语义 | +| --- | --- | --- | +| symbol | str | 标准代码(如 600000.SH) | +| flag_date | date | 状态生效日(戴帽/摘帽公告后的首个交易日) | +| st_flag | bool | true=风险警示(ST/*ST),false=摘帽;行区间语义:自 flag_date 起至下一条记录 | +| flag_type | str | "ST" / "*ST" / "摘帽";缺失填 "ST" | +| source | str | provider 标识 | + +查询语义:`st_at(symbol, t) = flag_date ≤ t 的最后一条记录的 st_flag`(asof-backward)。同步:全量快照 + 增量 append,`(symbol, flag_date)` 去重幂等(对齐 kline_sync:358-367【现状】模式)。 + +**`delisted_kline`**:K 线部分复用 `daily` dataset 完整 schema(symbol/date/OHLC/volume/amount/…);另需维表 `delisted_instruments`: + +| 字段 | 类型 | 语义 | +| --- | --- | --- | +| symbol / name | str | 代码/退市前简称 | +| list_date / delist_date | date | 上市/退市日 | +| delist_reason | str | "面值"/"财务"/"重组"/"主动"/"其他";缺失填 "其他" | + +宇宙回补语义:`as_of=t 的可交易池 = instruments(上市≤t<退市) ∪ delisted(上市≤t<退市)`;`_resolve_universe`(daily_pipeline.py:92-125【现状】)扩展为两源合并。 + +**`industry_pit`**(分区 `data/parquet/industry_pit/`): + +| 字段 | 类型 | 语义 | +| --- | --- | --- | +| symbol | str | 标准代码 | +| effective_date | date | 归属生效日 | +| industry_l1 / industry_l2 | str | 一级/二级行业名(如 计算机/软件开发,对齐 market_mainline.py:35【现状】的两级口径) | +| source | str | provider(如 ths) | + +查询语义:asof-backward join(同 fundamentals.py:107-114【现状】模式);effective_date 缺失 = 供应商不提供历史,整表降级为快照并触发 §5 注记。 + +**`index_ext`**:schema 与 `kline_index_daily` 完全一致(symbol/date/OHLC/volume/amount),仅标的白名单扩展(默认建议:000300.SH 沪深300、000905.SH 中证500、000852.SH 中证1000、000985.SH 中证全指);白名单由 preset 配置声明,不进代码硬编码(对齐 ext_presets 模式)。 + +**能力注册示例**(provider 侧 `plugin.yaml` datasets 声明,对齐 `docs/plugin-development.md` 契约): + +```yaml +datasets: + st_history: + enabled: true + description: 点时风险警示状态(戴帽/摘帽区间) + delisted_kline: + enabled: true + description: 退市标的日K + 退市维表 + industry_pit: + enabled: false # 供应商无历史归属时声明 false,不注册能力 + description: 点时行业归属 +``` + +能力矩阵(`capabilities.py` 注册表)同步各 dataset 的展示元数据与路由偏好字段;provider 未声明 = 该能力全局不可用,研究路径按 §4.3 降级矩阵处理,不静默。 + +--- + +## 10. API 契约【设计】 + +新路由前缀 `/api/factors`(薄层,重计算在 services/factors_research.py 编排层): + +| 端点 | 方法 | 请求要点 | 响应要点 | +| --- | --- | --- | --- | +| `/api/factors/catalog` | GET | asset_type, group, stability 过滤 | 因子清单(含 formula_text/warmup/direction/pit/scale_free/usage_count) | +| `/api/factors/validate` | POST | formula 文本 | 编译错误(位置+原因)或成功(依赖/预热推导) | +| `/api/factors/preview` | POST | formula + symbols + date_range | 试算表格(最新 5 日 × 前 20 标的)+ 非空率 | +| `/api/factors/custom` | GET/POST/DELETE | §3.4 JSON | CRUD;删除带引用清单 | +| `/api/factor-research/ic` | POST | factor_id, universe_id/UniverseSpec, NeutralizationSpec, start/end, rebalance, n_groups | §6 全套统计 + 三口径并列 + 漏斗 filter_stats + degraded 注记 | +| `/api/factor-research/composite` | GET/POST/DELETE | CompositeSpec | CRUD + 一键"作为评分字段试策略"跳转链接 | +| `/api/factor-research/universes` | GET/POST | UniverseSpec 存档 | 命名宇宙 CRUD(研究配置复用) | + +全部响应新增字段带默认值;错误响应含 `code/message/detail`,不泄漏内部栈(CONTRIBUTING 第 8 节)。SSE 进度复用现有回测 SSE 模式(长任务:批量 IC 扫描)。 + +**鉴权**:新路由经 `api/routes.py`【现状】注册,继承应用级部署口令鉴权(`docs/deploy-password.md` 模式),不引入独立权限模型。写操作(自定义因子/复合因子/宇宙存档 CRUD)只落 `data/user_data/`,路径校验沿用策略目录的防穿越规则(CONTRIBUTING 5.1 删除策略 fail-closed 要求同样适用)。 + +**并发**:批量 IC 扫描与复合分批量计算走 `services/heavy_job_limiter.py`【现状】限流,SSE 进度事件结构复用回测现有契约;用户取消走现有回测 worker 取消机制。 + +**核心端点示例**(其余端点按同构风格推导): + +`GET /api/factors/catalog?group=动量&asset_type=stock` → + +```json +{ + "factors": [ + { + "id": "momentum_20d", "version": 1, "label": "20日动量", "group": "动量", + "kind": "base", "formula_text": "20个交易日累计收益率", + "direction": "high", "unit": "ratio", "warmup_bars": 20, + "pit": false, "scale_free": true, "stability": "stable", + "tags": ["momentum"], "usage_count": 7, "custom": false + } + ], + "total": 61, "degraded": [] +} +``` + +`POST /api/factors/validate` `{"formula": "rank(ts_delta(close, -5))"}` → + +```json +{"ok": false, "errors": [{"code": "E005", "message": "负 shift: ts_delay 的 n 必须 ≥ 0(负数即未来函数)", "position": {"offset": 18, "line": 1}, "detail": {"n": -5}}]} +``` + +`POST /api/factor-research/ic`: + +```json +{ + "factor_id": "momentum_20d", + "universe": {"exclude_suspended": true, "exclude_limit_locked": true, "exclude_st": true, "min_listing_days": 60}, + "neutralization": {"benchmark": "000001.SH", "return_basis": "excess", "method": "industry_demean", "winsorize_sigma": 3.0}, + "start": "2023-01-01", "end": "2025-12-31", + "rebalance": "monthly", "n_groups": 5 +} +``` + +响应(节选,完整字段见 §6): + +```json +{ + "methodology_version": "factor_v3", + "universe_id": "a1b2c3d4e5f6", "neutralization_id": "9f8e7d6c5b4a", + "ic": {"mean": 0.031, "icir": 0.42, "t_naive": 2.9, "t_newey_west": 1.87, "nw_lag": 1, + "significance": "*", "half_life_days": 4, "acf": [0.21, 0.08, ...], "coverage": 0.97}, + "ic_by_basis": {"raw": {"mean": 0.041, "t_newey_west": 2.2}, "excess": {"mean": 0.031, "t_newey_west": 1.87}, + "excess_neutralized": {"mean": 0.019, "t_newey_west": 1.02}}, + "monotonicity": {"spearman": 0.9, "trend_t": 2.4, "verdict": "成立"}, + "universe_funnel": [{"date": "2025-12-31", "raw": 5412, "after_suspended": 5390, "after_limit_locked": 5320, "after_st": 5180, "after_new_listing": 5090}], + "degraded": ["st_not_point_in_time", "industry_snapshot"], + "turnover_top_group": 0.31, "costs_round_trip": 0.0013 +} +``` + +--- + +## 11. 前端界面【设计】 + +按此前结论:**不新增顶层页面**,组件级落点。每个组件给出区块级线框与交互流: + +1. **ResearchProfile 共享面板**(新组件,因子回测/挖掘/验证三视图共用):UniverseSpec + NeutralizationSpec 的受控表单,可存档命名(对应 `/api/factor-research/universes`);degraded 注记黄条;查询键含两 spec 哈希(queryKeys.ts 集中新增 `factorResearch` 键族)。 + 线框:`[存档下拉 ▾] [另存为] | 折叠区1·宇宙(6 个过滤器开关/输入 + 漏斗摘要行) | 折叠区2·调整(基准/口径/方法/截尾) | [重置] [应用到当前视图]`;spec 哈希变化即触发查询键切换。 +2. **因子目录对话框**(因子回测 tab 内,仿 ResearchCandidatesDialog):分组树 + 搜索 + 公式/方向/预热/PIT 徽章/引用数;"研究此因子"按钮回填选择器。 + 线框:`左侧分组树(带计数) | 右侧表格[因子/公式/方向/预热/PIT/引用] | 底部[研究此因子][加入复合候选]`;"加入复合候选"把因子暂存到复合编辑器的选择篮(跨组件轻状态,放 TanStack Query 缓存而非全局 store)。 +3. **IC 报告增强**:t 值列(NW 主口径,显著性星标)、三口径并列、单调性判定、IC 半衰期、宇宙漏斗、降级注记条。 + 线框:IC 摘要卡新增 `t(NW)=2.31** 半衰期=4d 单调性=成立(ρ=0.9)` 一行;分层表头新增口径切换 tab(原始/超额/超额+中性化),切换不改数据只换列;宇宙漏斗为横向递减条形(5200→4980→4890,hover 显示过滤器名)。 +4. **自定义因子编辑器**(Settings 新面板"因子库",与信号库并列):公式输入 + 算子速查侧栏 + 实时校验 + 试算预览 + 版本列表 + 引用关系展示。 + 线框:`左列: 版本列表(当前高亮)+元信息表单 | 中列: 公式输入框(等宽,校验错误行内红标+光标定位) + 算子速查(点击插入) | 右列: 试算预览表(最新5日×前20标的+非空率) [校验] [试算] [保存]`;保存按钮在校验+试算双绿前禁用。 +5. **复合因子编辑器**(同 Settings 面板内 tab):成员表(因子搜索、权重、方向)、自动权重开关与窗口、管线预览图;"试用于策略"向导生成 scoring 片段。 + 线框:`上: 成员表[因子搜索器|方向|权重|剔除] + weighting 单选 + 窗口输入 | 中: 成员相关性热力图(§11-8, >0.8 对红标提示去重) + 管线预览(过滤→截尾→中性化→标准化→加权) | 下: [导出 scoring 片段] [试用于策略]`。 +6. **因子相关性探索器**(复合编辑器内嵌 + 因子回测 tab 的独立对话框):任选 2-10 个因子,展示区间内日均截面秩相关矩阵热力图。后端复用 `mining.py:446` `compute_rank_correlation`【现状】抽出的公共函数,不新建第二套计算。 +7. **挖掘工作台**:排行榜加 t/q 值列与 DSR;候选卡新增"导出为复合因子"。 +8. 全部新组件覆盖 加载/空/错误/禁用/无权限 五态(CONTRIBUTING 第 7 节)**并在 1280px 常用宽度与窄屏(≤768px)检查截断、遮挡、弹窗可操作性**;前端类型同步进 `lib/api.ts`;所有轮询/长任务按钮带进行中禁用态。 + +**查询键新增**(`queryKeys.ts` 集中定义,spec 哈希必须进键): + +```text +factorCatalog({assetType, group}) // 目录 +factorValidate() // mutation,无需键 +factorCustomList() / factorCustomPreview({formulaHash}) // CRUD / 试算 +factorIcReport({factorId, factorVersion, universeId, neuId, + methodology, rangeHash, rebalance, nGroups}) +factorCompositeList() / factorCompositeEval({cmpSpecHash, matrixGeneration}) +factorCorrelation({factorIds[], rangeHash}) +researchUniverses() +``` + +--- + +## 12. 缓存与性能【设计】 + +| 缓存 | 键 | 失效 | +| --- | --- | --- | +| 因子 IC 报告 | `fr:ic:{factor_id}:{v}:{universe_id}:{neu_id}:{methodology}:{range_hash}:{rebalance}:{n_groups}` | enriched generation 变更或键任一分量变 | +| 复合因子定义 | `fr:cmp:{id}:{spec_hash}` | 定义编辑 | +| 复合分值(研究期) | `fr:cmpv:{cmp_spec_hash}:{matrix_generation}` | 矩阵重建 | +| DSL 编译产物 | 进程内 LRU(表达式文本 → Expr) | 进程重启 | +| 宇宙快照 | `fr:uni:{universe_id}:{matrix_generation}` | 矩阵重建 | + +约束:复合/自定义因子**默认不物化进 enriched parquet**(避免用户定义污染核心管道与增量路径);只在研究/评分请求期计算并按上表缓存。`incremental_safe=False` 的因子盘中路径直接缺失而非降级计算(对齐 pipeline 增量路径现有行为)。中性化逐日截面计算全部 Polars 表达式化;统计层 O(N·G) 极小。禁止任何新增逻辑进入实时行情线程(CONTRIBUTING 6.3)。 + +**性能预算(实现验收线,超线必须先优化再合入)**: + +| 操作 | 预算 | 基准场景 | +| --- | --- | --- | +| 单因子 IC 全报告(含三口径+t 值+分层) | ≤ 现有报告耗时 × 1.3 | 全 A 股 × 3 年日线(现有 `factor.py` 同窗基线,PR 里附前后数据,CONTRIBUTING 6.3) | +| DSL 编译(含校验) | ≤ 5ms/表达式 | 深度 12、token 200 上限样例 | +| 复合因子单期截面计算 | ≤ 成员因子独立计算耗时之和 × 1.2 | 10 成员 × 全 A 股 | +| 宇宙过滤全期 | ≤ 全期 IC 计算的 10% | 同上基准 | +| 因子目录接口 | ≤ 50ms | 全量 62+ 因子元数据 | +| 相关性探索器 | ≤ 现有 mining 同规模秩相关耗时 × 1.1 | 10 因子 × 1 年 | + +--- + +## 13. 测试矩阵(最低要求清单)【设计】 + +| 模块 | 必测 | +| --- | --- | +| 注册表重构 | 特征化快照(62+ 因子两条计算路径逐位一致);重复 id/未增版本拒绝启动 | +| DSL | 每算子黄金用例(含 null/除零/σ=0/全常数);负 shift 编译失败;深度/窗口/白名单越界拒绝;与手写 Polars 等价性;注入样例(`__import__`、列名穿越)拒绝 | +| 宇宙 | 每过滤器独立单测(构造含 ST/停牌/涨停锁死/次新/微额的合成面板);漏斗计数;降级注记触发 | +| 中性化 | 合成数据数值断言(demean/zscore/回归残差 vs statsmodels 对拍);行业缺组回退;基准停市日 | +| 统计 | NW t:黄金参考向量(离线计算硬编码期望值)+ dev 组 statsmodels 对拍(可选);FDR:BH 已知 p 向量解析解;DSR:已构造解析例(对称正态收益 + 已知试验数);单调性边界(平梯/倒梯) | +| 指标 | Sharpe/年化/超额/IR 已知序列解析解;243 口径回归 | +| 复合因子 | 权重归一;**自动权重无未来函数**(t 日权重不随 t 日数据变化——篡改 t 日数据断言权重不变);四端同值断言(§8.3);环引用拒绝 | +| API | 成功/空数据/编译错误/预热不足/无权限 | +| 缓存 | 键覆盖测试(改 spec 必换键);generation 失效 | +| 前端 | pnpm build + 五态检查 | + +--- + +## 14. 实施路线图(PR 粒度,每 PR 独立可合) + +| PR | 内容 | 依赖 | 主要文件 | 分级 | +| --- | --- | --- | --- | --- | +| 1 | stats 模块:IC t(NW)/单调性/半衰期 + IC 报告新字段 | 无 | 新 `app/factors/stats.py` + factor.py 增量 | L2 | +| 2 | metrics 统一 + metrics_v2 版本化 | PR-1 | 新 metrics.py;engine/strategy/factor 改引用 | L3(热点,最小接线) | +| 3 | 宇宙构建器 + 过滤器 + IC 接入 + 缓存键 | PR-1 | 新 universe.py + factor.py | L2 | +| 4 | 基准超额 + 中性化 + 三口径并列 | PR-3 | 新 neutralize.py + factor.py | L2 | +| 5 | FDR + DSR 进 optimizer/mining 排行榜与晋升门槛 | PR-1 | optimizer/mining 增量 | L3 | +| 6 | 因子注册表重构(特征化测试先行) | 无(可与 1-5 并行) | 新 registry.py;factor.py/scoring.py/pipeline.py 收口 | L3 | +| 7 | 复合因子 + 策略 scoring 桥 + 挖掘导出 | PR-6 | 新 composite.py;scoring.py 最小接线 | L2→L3 | +| 8 | DSL 编译器 + 自定义因子 CRUD + Settings 因子库 UI | PR-6 | 新 dsl/ + api + settings 前端 | L2 | +| 9 | 前端:ResearchProfile + 目录 + 报告增强 | PR-3/4 后端就绪 | 前端组件族 | 前端 | +| 10+ | 数据契约:st_history / delisted / industry_pit / index_ext | provider 侧 | capabilities + 各 provider | L1+L2 | + +每个 PR 按 CONTRIBUTING 第 10 节模板出描述(问题/根因/方案/兼容/性能/验证/界面证据/回滚)。 + +**配置白名单联动**:`backtest/candidates.py:28-49`【现状】的 `_CONFIG_FIELDS["factor"]` 是冻结字段集,PR-3/PR-4 必须同步扩展 `universe`、`neutralization` 两个配置字段(沿用 `_MINING_SOURCE_CONFIG_FIELDS` 的 frozenset 合并模式),否则保存候选会静默丢弃 spec——这是缓存一致性之外的第二个容易漏的接线点,测试须覆盖"保存→载入→spec 哈希不变"。 + +**各 PR 回滚要点**:PR-1/3/4/5 新增模块 + 增量字段,回滚 = revert 即可(旧缓存键不含新分量,自动回旧路径);PR-2 指标统一保留 `metrics_methodology` 开关,回滚 = 切回 v1 计算分支并保留数据;PR-6 注册表重构通过特征化测试保证行为等价,回滚 = revert(无持久化迁移);PR-7/8 用户数据(自定义因子/复合因子 JSON)为新增目录,回滚代码后文件残留但不再加载,重新部署即恢复——**不存在任何需要用户手动清数据的回滚**(CONTRIBUTING 第 12 节红线)。 + +**文档同步任务**(各 PR 内完成,不单开):PR-1/3/4 更新 `docs/features.md` 因子回测章节;PR-5 更新 `docs/mining.md` 门槛说明;PR-7 更新 `docs/strategy.md` 评分字段说明与 `操作说明书.md`;PR-8 更新 `docs/custom-data-source.md` 无关则跳过;本设计文档在每个 PR 合入后把对应条目从【设计】改标【已实现】。 + +--- + +## 15. 兼容性影响与风险清单 + +| 变更 | 功能影响 | 结果口径影响 | 缓解 | +| --- | --- | --- | --- | +| PR-1/5 统计字段 | 无 | 无(纯新增) | 字段默认值 | +| PR-2 指标统一 | 无 | **Sharpe/年化数字变化(有意)** | metrics_v2 版本徽章;旧字段一版周期弃用 | +| PR-3/4 宇宙/中性化 | 无 | IC/分层数字变化(有意,通常回落) | 默认开关显式;三口径并列;缓存键含 spec | +| PR-6 注册表 | 无 | 要求逐位一致 | 特征化测试是合入硬门槛 | +| PR-7/8 复合/DSL | 无(纯新增能力) | 无 | 注入列模式,custom_signals 先例 | +| 数据契约 | 无 | 退市股回补后回测数字变化(修复) | 独立 dataset,无源时明确降级注记 | + +剩余风险:~~① ts_rank 的 Polars 原生可用性需实现期确认~~【已解决:§17 验证 polars 1.40.1 `Expr.rolling_rank` 存在且行为正确】;② 行业快照回填历史的偏差在点时表到位前无法消除(注记透明化);③ 幸存者偏差的根本修复依赖数据源,代码侧已尽(注记 + 回补接口预留);④ ST 非点时在 st_history dataset 到位前仅能注记;⑤ 统计函数运行时零新增第三方依赖(后端当前无 scipy/statsmodels,已验证),NW/BH-FDR/DSR 以 numpy 手写实现,statsmodels 仅允许加入 uv dev 依赖组做测试对拍,不进运行时。 + +--- + +## 16. 明确不做清单(YAGNI 边界) + +以下能力**刻意不在本设计范围内**,防止范围蔓延(依据 `docs/secondary-development.md` 第 10 节:不为未来可能出现的需求预埋框架)。出现真实需求时再按需立项: + +| 不做项 | 理由 | +| --- | --- | +| 因子市场/分享/导入导出社区 | 单用户自托管定位,无真实需求 | +| 全 Barra 风格回归(Beta/动量/流动性/非线性市值等十因子) | 数据与维护成本高;industry+size lite 已覆盖主要混杂,收益边际低 | +| 自动机器学习/遗传规划因子搜索 | 与现有 beam search + 嵌套样本外定位重叠,且加剧多重检验问题 | +| 港美股/加密资产因子 | 数据源与交易规则(T+0/无涨跌停)完全是另一套引擎 | +| Tick 级/高频因子 | 分钟数据集能力有限,且与现有日线研究框架口径不同 | +| 因子值的实时盘中推送(SSE 逐笔更新) | 违反实时热路径约束;监控经由策略结果缓存已覆盖时效需求 | +| 复合因子权重在线学习/逐日再优化 | 自动权重窗口已是点时滚动;更细粒度会显著推高换手且引入过拟合面 | +| ARIMA/VAR 预测、协整与配对交易 | 本平台定位是**横截面因子研究**;时间序列预测与统计套利是另一条业务线(指数择时/配对),数据、引擎与交易规则均不同,混入即范围蔓延 | +| GARCH 全族 / 卡尔曼滤波 / 时变 Beta 状态空间模型 | EWMA 条件波动已覆盖日频主要价值;逐 symbol 递归拟合与全向量化管线冲突,机构级边际收益不抵维护成本(单 GARCH(1,1) 为 §2.4 的 experimental 按需项,不在冲突内) | +| HMM/马尔可夫区制检测 | 现有情绪周期 6 阶段(启发式)+ 分环境 IC(factor.py:813-903【现状】)已覆盖区制条件分析;统计区制模型列为未来探索项不进本期 | +| 独立权限体系(多用户/角色) | 应用级部署口令已满足自托管场景 | + +--- + +## 17. 验证附录(本设计的验证记录) + +> 验证日期 2026-09-04,基准 main@2ce8b4b1,后端 polars 1.40.1。分三部分:代码引用逐条核对、技术可行性实测、内部一致性检查。结论:**全部引用属实或已修正,可行性风险清零或已有替代方案,一致性检查通过**。 + +### 17.1 代码引用核对(【现状】条目逐条对账) + +| 引用 | 核对内容 | 结果 | +| --- | --- | --- | +| factor.py:36-109 | FACTOR_COLUMNS 61 因子目录 | ✓(全文读取) | +| factor.py:111/112/122 | warmup 120 / factor_v2 / n_groups=5 | ✓ | +| factor.py:740-750 / 752-778 / 780-811 / 813-903 | Rank IC / 分年 / 衰减 / 分环境 | ✓(函数定义与实现均在引用区间) | +| factor.py:1059-1067 / 1095-1098 | 双边佣金+印花税+滑点成本 / 每调仓期扣减 | ✓ | +| factor.py:1123-1166 / 1001-1035 / 1294-1295 | 换手率 / tie-aware 分层 / executable_short=False | ✓ | +| factor.py:1184-1186 / 1274-1277 | 年化系数匹配调仓频率的注释与实现 | ✓(注释原文核实) | +| mining.py:26-31 / 91-100 / 1041-1073 / 1503-1530 / 446 | 晋升门槛 / purge30+embargo5 / trial 预算 / 折构造 / 秩相关 | ✓ | +| mining.py:492-493 | `pl.corr(..., method="spearman")` 可用性 | ✓(代码在用,即证 API 存在) | +| engine.py:2871-2874 / 2899-2903 / 2993-3030 / 3116-3122 | 365.25 年化 / 逐笔 Sharpe(含"非严格正确"注释)/ 仅卖出日聚合 / 净值口径 | ✓(四种口径全部原文核实) | +| engine.py:51 / 54-94 / 903-933 | matching 默认 close_t / 成本模型 / 涨跌停与停牌成交闸 | ✓(全文精读) | +| pipeline.py:970-984 | filter_halt_days(函数头 973) | ✓ | +| pipeline.py:1795 | compute_enriched_today 盘中增量入口 | ✓ **(修正:初稿误引 1738+,该行实为复权因子读取;已改)** | +| matrix.py:1476-1495 | _write_tradable_matrix | ✓ | +| repository.py:1488 | get_index_daily | ✓ | +| api/data.py:765 | listing_date 已暴露未用于研究 | ✓ | +| fundamentals.py:107-115 | join_asof backward + date>_announce 严格公告日后 | ✓ | +| share_capital.py:54-56 | announce_date 优先、period_end 兜底 | ✓ | +| price_limits.py:87-101 | numpy_limit_pct_vectors(当前名推断) | ✓ | +| capabilities.py:44-45 | 复权口径一致性"不做路由耦合"注释 | ✓ | +| index_const.py:12-15 | 核心四只代码(000001.SH/399001.SZ/399006.SZ/000680.SH) | ✓ | +| daily_pipeline.py:92-125 | _resolve_universe(CN_Equity_A 当前池) | ✓ | +| kline_sync.py:358-367 | (symbol, trade_date) 去重 keep=last 原子合并 | ✓ | +| scoring.py:13-51 / 53-66 / 91-104 / 108+ | VIRTUAL 依赖 35 项 / 预热表 / 依赖展开与预热推导 / scoring_value_expr | ✓(全文读取,附录A 由其逐字推导) | +| monitor.py:1314-1322 | score_min/max 消费 result.scores | ✓ | +| strategy/builtin/*.py | "scoring": {字段: 权重} 配置形态 | ✓(8 个内置策略抽样) | +| candidates.py | factor/strategy 双形态候选配置字段 | ✓ | +| services/heavy_job_limiter.py、backtest/numba_runtime.py、services/ext_presets.py、services/market_mainline.py:35 | 模块存在性 / 行业两级口径 | ✓ | + +### 17.2 技术可行性实测(`uv run python` 于 backend 环境) + +| 项 | 实测结果 | 设计影响 | +| --- | --- | --- | +| polars 版本 | 1.40.1(pyproject pin >=1.0) | — | +| `Expr.rolling_rank` | **存在**,递增序列 4 点窗输出 [null,null,null,4,4,4] 行为正确 | §15 风险①**解除**,ts_rank 用原生实现 | +| `Expr.ewm_std` / `ewm_var` | 存在 | f_ewma_vol 直接可实现 | +| `pl.rolling_corr`(顶层)/ `pl.rolling_cov` | 存在且可算出正确相关值(3 点窗样例 0.6547) | ts_corr/ts_cov 编译目标为顶层函数 | +| `pl.rolling_corr(...).over("symbol")` 分组组合 | **实测通过**:A/B 两组各自窗口内相关(0.6547 / −1.0),无串组 | ts_corr 多标的面板场景确认可行(初稿未验证,本轮补测) | +| `Expr.rolling_corr` | **不存在**(仅顶层函数) | 算子表已按顶层函数修正 | +| `Expr.rolling_arg_max` | **不存在**;rolling_map 为 Python 回调 | ts_argmax/ts_argmin v1 移除(§3.1 已注) | +| rolling_quantile/var/mean/std/max/min/sum、shift、pct_change、diff、pow、sign、clip、log、cum_prod | 全部存在 | 其余算子无阻碍 | +| `pl.corr` spearman | mining.py:492 在用 | 无阻碍 | +| numba | 已安装(numba_runtime 真实存在) | GARCH/矩阵核扩展路径成立 | +| statsmodels / scipy | **均未安装** | §6 零依赖原则:运行时 numpy 手写 NW/BH-FDR/DSR,statsmodels 仅可进 dev 依赖组 | + +### 17.3 内部一致性检查 + +1. 缓存键 ↔ Spec 字段:UniverseSpec 7 字段、NeutralizationSpec 5 字段(含 winsorize_sigma 与管线顺序)全部进入各自 canonical_json/哈希 → 键覆盖完整(§4.2/§5.2 ↔ §12)。✓ +2. API ↔ 前端组件:catalog↔目录对话框、validate/preview/custom↔因子编辑器、ic↔报告增强、composite↔复合编辑器、universes↔ResearchProfile,无孤立端点或无后端组件。✓ +3. 测试矩阵 ↔ 模块:§13 十行覆盖 §2-§12 全部新增模块,无模块缺测试项。✓ +4. 路线图 ↔ 章节:PR-1↔§6.1、PR-2↔§7、PR-3↔§4、PR-4↔§5、PR-5↔§6.1(FDR/DSR)、PR-6↔§2、PR-7↔§8、PR-8↔§3、PR-9↔§11、PR-10↔§9,全部章节有落点 PR。✓ +5. §2.4 GARCH(1,1) experimental 与 §16"GARCH 全族不做"边界一致(单一按需项 vs 全族排除,§16 已加注)。✓ +6. EBNF ↔ 算子表:if_else/比较/逻辑均以函数调用或中缀产生式覆盖;优先级链 or 0.8」类条件 | +| 回测 → 因子 | 策略回测「因子归因」tab | 见上文回测引擎章节 | + +**版本与生命周期**:公式修改保存为新版本并回草稿态(需重新检验);状态机 active / watch / retired / draft。 + +--- + ## 📡 监控中心(Monitor) 统一规则引擎,一个页面管理**四类监控**: diff --git a/docs/secondary-development.md b/docs/secondary-development.md index c7e1081..ec78234 100644 --- a/docs/secondary-development.md +++ b/docs/secondary-development.md @@ -22,6 +22,7 @@ - 后端源码扩展注册:`backend/app/custom/.py`,支持 FastAPI 路由、启动钩子和通知格式化器。 - 当前前端插槽:`layout.navigation.extra`、`stock-preview.footer`、`watchlist.toolbar`。 - 当前后端继承点:`NotificationFormatter`。 +- 因子平台与策略线的既有桥接(已实现,二开时直接复用、勿重复实现):因子库一键生成单因子排名策略(`GenerateFactorStrategyDialog` → `custom_factor_*`);策略触发器引用因子条件信号(`AddFactorSignalDialog`,`csg_f_*`);自定义信号 AI 提示词含因子分组(`custom_signals_ai.py`);策略回测因子归因(`strategy.py` 的 `factor_attribution`,覆盖 `meta.scoring` 非空的策略)。 尚未实现、只能在真实需求出现后增加的能力: diff --git a/frontend/package.json b/frontend/package.json index ad460d1..4edd5b3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "tickflow-stock-panel-frontend", "private": true, - "version": "0.2.2", + "version": "0.2.3", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index f5eaf0c..d2926cb 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -25,7 +25,7 @@ import { Star, ScanSearch, History, - Pickaxe, + Sigma, FileText, Settings, DatabaseZap, @@ -40,6 +40,7 @@ import { Sparkles, Layers2, Layers3, + Zap, Landmark, RadioTower, CheckCircle2, @@ -84,9 +85,8 @@ const nav = [ { to: '/', label: '看板', icon: LayoutDashboard }, { to: '/watchlist', label: '自选', icon: Star }, { to: '/screener', label: '策略', icon: ScanSearch }, + { to: '/factors', label: '因子', icon: Sigma }, { to: '/backtest', label: '回测', icon: History }, - { to: '/mining', label: '挖掘', icon: Pickaxe }, - { to: '/lots', label: '持仓提醒', icon: Layers2 }, { to: '/stock-analysis', label: '个股分析', icon: TrendingUp }, { to: '/limit-ladder', label: '连板梯队', icon: Flame }, { to: '/concept-analysis', label: '概念分析', icon: Layers3 }, @@ -95,6 +95,8 @@ const nav = [ { to: '/monitor', label: '监控中心', icon: RadioTower }, { to: '/regime', label: '市场环境', icon: Gauge }, { to: '/abnormal', label: '异动监控', icon: Siren }, + { to: '/lots', label: '持仓提醒', icon: Layers2 }, + { to: '/signals', label: '信号库', icon: Zap }, { to: '/review', label: '复盘', icon: BookOpenCheck }, { to: '/indices', label: '指数', icon: BarChart3 }, { to: '/data', label: '数据', icon: Database }, diff --git a/frontend/src/components/StockPreviewDialog.tsx b/frontend/src/components/StockPreviewDialog.tsx index 527d8ed..7b03ab1 100644 --- a/frontend/src/components/StockPreviewDialog.tsx +++ b/frontend/src/components/StockPreviewDialog.tsx @@ -585,7 +585,7 @@ export function StockPreviewDialog({ symbol, name, onClose, triggerInfo, navList }`} > {parseInt(w, 10)}日{' '} - = 0 ? 'text-bull' : 'text-bear'}>{fmtPct(info.value, 1)} + = 0 ? 'text-bull' : 'text-bear'}>{fmtPct(info.value)} / ±{(info.threshold * 100).toFixed(0)}% · 接近{(info.closeness * 100).toFixed(0)}% diff --git a/frontend/src/components/signals/SignalTriggerActions.tsx b/frontend/src/components/signals/SignalTriggerActions.tsx index 14db306..749c0c4 100644 --- a/frontend/src/components/signals/SignalTriggerActions.tsx +++ b/frontend/src/components/signals/SignalTriggerActions.tsx @@ -1,8 +1,9 @@ import { useState } from 'react' import { useNavigate } from 'react-router-dom' -import { Plus, Settings2 } from 'lucide-react' +import { Plus, Settings2, Zap } from 'lucide-react' import type { CustomSignal } from '@/lib/api' import { CustomSignalDialog } from './CustomSignalDialog' +import { AddFactorSignalDialog } from '@/pages/backtest/AddFactorSignalDialog' interface Props { kind: 'entry' | 'exit' @@ -15,19 +16,31 @@ interface Props { export function SignalTriggerActions({ kind, signals, onChange, buttonClassName, iconClassName }: Props) { const navigate = useNavigate() const [open, setOpen] = useState(false) + const [factorOpen, setFactorOpen] = useState(false) const accent = kind === 'entry' ? 'hover:text-accent hover:border-accent/40' : 'hover:text-warning hover:border-warning/40' const btnCls = buttonClassName ?? 'rounded-btn border border-border bg-base p-1 text-muted transition-colors cursor-pointer' const iconCls = iconClassName ?? 'h-3.5 w-3.5' - const handleSaved = (signal: CustomSignal) => { - if (signal.kind !== kind && signal.kind !== 'both') return - const signalId = `csg_${signal.id}` + const enableSignal = (signalId: string, signalKind: 'entry' | 'exit' | 'both') => { + if (signalKind !== kind && signalKind !== 'both') return onChange(signals.includes(signalId) ? signals : [...signals, signalId]) } + const handleSaved = (signal: CustomSignal) => enableSignal(`csg_${signal.id}`, signal.kind) + const handleFactorCreated = (signal: { id: string; kind: 'entry' | 'exit' | 'both' }) => + enableSignal(`csg_${signal.id}`, signal.kind) + return ( <> + +