From b6cf0495e1b663cd24381be07c916833301cb5a5 Mon Sep 17 00:00:00 2001 From: lytem28 Date: Thu, 16 Jul 2026 12:17:27 +0800 Subject: [PATCH] feat: complete matrix-native backtest engine Unify strategy execution across backtest, screener, and monitoring; isolate backtest workloads in spawn workers; and add shared matrix caching plus valid-bar indicator acceleration. --- backend/app/api/backtest.py | 188 +- backend/app/api/monitor_rules.py | 21 +- backend/app/api/screener.py | 233 +- backend/app/api/strategy.py | 114 +- backend/app/backtest/engine.py | 1066 ++++- backend/app/backtest/matrix.py | 3809 +++++++++++++++++ backend/app/backtest/optimizer.py | 256 +- backend/app/backtest/strategy.py | 1105 ++++- backend/app/backtest/walkforward.py | 81 +- backend/app/backtest/worker.py | 311 ++ backend/app/config.py | 8 + backend/app/desktop.py | 3 + backend/app/indicators/pipeline.py | 171 +- backend/app/main.py | 58 +- backend/app/services/screener.py | 345 +- backend/app/services/strategy_cache.py | 8 + backend/app/strategy/ai_generator.py | 45 +- backend/app/strategy/builtin/boll_breakout.py | 69 +- .../strategy/builtin/broken_board_recovery.py | 92 +- .../app/strategy/builtin/bullish_alignment.py | 75 +- .../strategy/builtin/consecutive_limit_ups.py | 58 +- .../strategy/builtin/high_turnover_surge.py | 85 +- .../app/strategy/builtin/limit_up_momentum.py | 72 +- .../strategy/builtin/low_volatility_leader.py | 86 +- .../app/strategy/builtin/ma_golden_cross.py | 85 +- backend/app/strategy/builtin/macd_golden.py | 93 +- .../strategy/builtin/n_day_low_reversal.py | 78 +- backend/app/strategy/builtin/near_limit_up.py | 119 +- .../app/strategy/builtin/oversold_bounce.py | 89 +- .../app/strategy/builtin/oversold_reversal.py | 94 +- .../strategy/builtin/pullback_ma20_bounce.py | 90 +- .../strategy/builtin/pullback_to_support.py | 112 +- backend/app/strategy/builtin/strong_open.py | 96 +- .../app/strategy/builtin/trend_breakout.py | 83 +- .../strategy/builtin/volume_price_surge.py | 79 +- backend/app/strategy/custom_signals.py | 30 +- backend/app/strategy/engine.py | 681 ++- backend/app/strategy/monitor.py | 139 +- backend/app/strategy/prompt_builder.py | 14 +- .../prompts/strategy-builder-step1.md | 4 +- .../prompts/strategy-builder-step2.md | 2 +- .../app/strategy/prompts/strategy-example.md | 3 + .../prompts/strategy-guide-compact.md | 55 +- .../app/strategy/prompts/strategy-guide.md | 12 +- backend/app/tickflow/repository.py | 67 + backend/pyproject.toml | 6 +- backend/tests/backtest/test_dependencies.py | 119 + backend/tests/backtest/test_market_matrix.py | 225 + .../backtest/test_matrix_compute_cache.py | 234 + .../tests/backtest/test_matrix_strategy.py | 972 +++++ backend/tests/backtest/test_optimizer_api.py | 28 + backend/tests/backtest/test_optimizer_run.py | 178 +- .../tests/backtest/test_robustness_metrics.py | 29 +- .../test_strategy_backtest_correctness.py | 304 +- backend/tests/backtest/test_worker_process.py | 188 + backend/tests/test_ai_generator_prompt.py | 17 + backend/tests/test_high_turnover_strategy.py | 31 +- backend/tests/test_indicator_needed.py | 62 + backend/tests/test_screener_builtin_params.py | 193 +- backend/tests/test_screener_etf.py | 220 +- backend/tests/test_st_limit_and_sharpe.py | 23 +- backend/tests/test_strategy_code_save.py | 6 +- .../tests/test_strategy_realtime_refresh.py | 78 +- backend/tests/test_strategy_registry.py | 120 + backend/tests/test_strategy_saved_params.py | 23 +- backend/uv.lock | 4 + .../screener/StrategyBuilderDialog.tsx | 80 +- frontend/src/lib/api.ts | 26 +- frontend/src/lib/optimizerTask.ts | 14 + frontend/src/pages/Backtest.tsx | 2 +- frontend/src/pages/Screener.tsx | 7 +- .../src/pages/backtest/StrategyBacktest.tsx | 35 +- .../src/pages/backtest/StrategyOptimizer.tsx | 4 +- .../pages/backtest/StrategyWalkForward.tsx | 2 +- 74 files changed, 12113 insertions(+), 1501 deletions(-) create mode 100644 backend/app/backtest/matrix.py create mode 100644 backend/app/backtest/worker.py create mode 100644 backend/tests/backtest/test_dependencies.py create mode 100644 backend/tests/backtest/test_market_matrix.py create mode 100644 backend/tests/backtest/test_matrix_compute_cache.py create mode 100644 backend/tests/backtest/test_matrix_strategy.py create mode 100644 backend/tests/backtest/test_worker_process.py create mode 100644 backend/tests/test_indicator_needed.py create mode 100644 backend/tests/test_strategy_registry.py diff --git a/backend/app/api/backtest.py b/backend/app/api/backtest.py index db72beb..9f69da3 100644 --- a/backend/app/api/backtest.py +++ b/backend/app/api/backtest.py @@ -4,7 +4,6 @@ from __future__ import annotations import asyncio import json import logging -import queue import threading from dataclasses import asdict from datetime import date, timedelta @@ -19,7 +18,6 @@ from app.services.backtest import ( BacktestConfig, BacktestService, VectorbtUnavailable, - is_available, ) logger = logging.getLogger(__name__) @@ -213,11 +211,8 @@ class StrategyBacktestRequest(BaseModel): @router.post("/strategy/run") def strategy_run(req: StrategyBacktestRequest, request: Request): """策略回测 — 复用 StrategyDef 体系做全周期回测。""" - from app.backtest.strategy import StrategyBacktestService, StrategyBacktestConfig - - engine = _get_engine(request) - strategy_engine = request.app.state.strategy_engine - svc = StrategyBacktestService(engine, strategy_engine) + from app.backtest.strategy import StrategyBacktestConfig + from app.backtest.worker import make_worker_task, run_worker_task end = req.end or date.today() start = _resolve_start(req, end, FACTOR_DEFAULT_DAYS) @@ -245,8 +240,8 @@ def strategy_run(req: StrategyBacktestRequest, request: Request): holding_days=req.holding_days, asset_type=req.asset_type, ) - result = svc.run(cfg) - return asdict(result) + task = make_worker_task("backtest", settings.data_dir, cfg) + return run_worker_task(task) # ── SSE 流式回测 (实时进度 + 可取消 + 支持重连) ─────────────────── @@ -288,6 +283,26 @@ def _cleanup_stale_jobs(): _running_jobs.pop(k, None) +def _finish_job(job: _BacktestJob, *, result=None, error: str | None = None) -> None: + """Publish the terminal state and proactively drop the reconnect entry after TTL.""" + finished_at = time.time() + with _jobs_lock: + job.result = result + job.error = error + job.done = True + job.finish_ts = finished_at + + def _expire() -> None: + with _jobs_lock: + current = _running_jobs.get(job.key) + if current is job and current.done and current.finish_ts == finished_at: + _running_jobs.pop(job.key, None) + + timer = threading.Timer(_JOB_TTL, _expire) + timer.daemon = True + timer.start() + + def _make_job_key( strategy_id: str, symbols: str | None, start: str | None, end: str | None, matching: str, entry_fill: str | None, exit_fill: str | None, @@ -339,11 +354,8 @@ async def strategy_stream( - done: {result} (完整回测结果) - error: {message} """ - from app.backtest.strategy import StrategyBacktestService, StrategyBacktestConfig - - engine = _get_engine(request) - strategy_engine = request.app.state.strategy_engine - svc = StrategyBacktestService(engine, strategy_engine) + from app.backtest.strategy import StrategyBacktestConfig + from app.backtest.worker import make_worker_task, run_worker_task end_date = date.fromisoformat(end) if end else date.today() if start: @@ -436,14 +448,15 @@ async def strategy_stream( # 仍可置位, svc.run 会据此提前返回 cancelled)。持槽跑完在 finally 释放。 _backtest_semaphore.acquire() try: - result = svc.run(cfg, lambda d: job.progress.append(d), job.cancel_event) - job.result = result - job.done = True - job.finish_ts = time.time() + task = make_worker_task("backtest", settings.data_dir, cfg) + result = run_worker_task( + task, + lambda d: job.progress.append(d), + job.cancel_event, + ) + _finish_job(job, result=result) except Exception as e: - job.error = str(e) - job.done = True - job.finish_ts = time.time() + _finish_job(job, error=str(e)) finally: _backtest_semaphore.release() @@ -462,12 +475,14 @@ async def strategy_stream( yield f"event: error\ndata: {json.dumps({'message': job.error}, ensure_ascii=False)}\n\n" elif job.result is not None: r = job.result - if hasattr(r, "error") and r.error == "cancelled": + error = r.get("error") if isinstance(r, dict) else getattr(r, "error", None) + if error == "cancelled": yield f"event: error\ndata: {json.dumps({'message': '回测已取消'}, ensure_ascii=False)}\n\n" - elif hasattr(r, "error") and r.error: - yield f"event: error\ndata: {json.dumps({'message': r.error}, ensure_ascii=False)}\n\n" + elif error: + yield f"event: error\ndata: {json.dumps({'message': error}, ensure_ascii=False)}\n\n" else: - yield f"event: done\ndata: {json.dumps(asdict(r), ensure_ascii=False, default=str)}\n\n" + payload = r if isinstance(r, dict) else asdict(r) + yield f"event: done\ndata: {json.dumps(payload, ensure_ascii=False, default=str)}\n\n" return # 断开检测: 每 4 轮检查一次 (降低 GIL 抢占频率) @@ -561,8 +576,23 @@ _OPT_BT_FIELDS = [ ] -def _make_opt_job_key(strategy_id, symbols, start, end, param_grid, objective, direction, bt_sig, params=None, overrides=None) -> str: - raw = f"OPT|{strategy_id}|{symbols}|{start}|{end}|{param_grid}|{objective}|{direction}|{bt_sig}|{params}|{overrides}" +def _make_opt_job_key( + strategy_id, + symbols, + start, + end, + param_grid, + objective, + direction, + bt_sig, + params=None, + overrides=None, + matrix_cache_max_mb=512, +) -> str: + raw = ( + f"OPT|{strategy_id}|{symbols}|{start}|{end}|{param_grid}|{objective}|" + f"{direction}|{bt_sig}|{params}|{overrides}|cache={matrix_cache_max_mb}" + ) return hashlib.md5(raw.encode()).hexdigest()[:12] @@ -593,6 +623,7 @@ async def optimize_stream( objective: str = "sortino", direction: str | None = None, max_workers: int = 4, + matrix_cache_max_mb: int = 512, params: str | None = None, # JSON: 未扫描参数固定为用户当前值 (base_params) overrides: str | None = None, # JSON: 策略当前的 basic_filter/signals/风控等覆盖 symbols: str | None = None, @@ -617,12 +648,8 @@ async def optimize_stream( - done: {result} (含 best_params / results 排名) - error: {message} """ - from app.backtest.optimizer import OptimizeConfig, StrategyOptimizer - from app.backtest.strategy import StrategyBacktestService - - engine = _get_engine(request) - strategy_engine = request.app.state.strategy_engine - svc = StrategyBacktestService(engine, strategy_engine) + from app.backtest.optimizer import OptimizeConfig + from app.backtest.worker import make_worker_task, run_worker_task end_date = date.fromisoformat(end) if end else date.today() if start: @@ -642,7 +669,19 @@ async def optimize_stream( max_positions, max_exposure_pct, initial_capital, position_sizing, mode, holding_days, ) bt_sig = "|".join(f"{k}={bt_kwargs[k]}" for k in _OPT_BT_FIELDS) - job_key = _make_opt_job_key(strategy_id, symbols, start, end, param_grid, objective, direction, bt_sig, params, overrides) + job_key = _make_opt_job_key( + strategy_id, + symbols, + start, + end, + param_grid, + objective, + direction, + bt_sig, + params, + overrides, + matrix_cache_max_mb, + ) _cleanup_stale_jobs() with _jobs_lock: @@ -670,9 +709,7 @@ async def optimize_stream( # grid 必须是非空 dict; null/[]/"" 等合法 JSON 但结构错误也在此拦下, # 否则会跳过线程启动却不置 done -> event_generator 永久空转、job 挂死。 if not isinstance(grid, dict) or not grid: - job.error = "param_grid 必须是非空的参数网格对象" - job.done = True - job.finish_ts = time.time() + _finish_job(job, error="param_grid 必须是非空的参数网格对象") grid = None if grid is not None: @@ -698,6 +735,7 @@ async def optimize_stream( objective=objective, direction=direction, max_workers=int(max_workers), + matrix_cache_max_mb=int(matrix_cache_max_mb), base_params=base_params if isinstance(base_params, dict) else {}, overrides=ov if isinstance(ov, dict) else None, backtest_kwargs=bt_kwargs, @@ -705,14 +743,15 @@ async def optimize_stream( def _run_opt(): try: - opt = StrategyOptimizer(svc, strategy_engine) - job.result = opt.optimize(ocfg, lambda d: job.progress.append(d), job.cancel_event) - job.done = True - job.finish_ts = time.time() + task = make_worker_task("optimize", settings.data_dir, ocfg) + result = run_worker_task( + task, + lambda d: job.progress.append(d), + job.cancel_event, + ) + _finish_job(job, result=result) except Exception as e: - job.error = str(e) - job.done = True - job.finish_ts = time.time() + _finish_job(job, error=str(e)) threading.Thread(target=_run_opt, daemon=True).start() @@ -764,8 +803,24 @@ async def optimize_cancel(request: Request): # Walk-forward 优化 — 每折训练区间优化 + 测试区间 OOS 验证 (复用优化器 + job_key 回吐) # ══════════════════════════════════════════════════════════════ -def _make_wf_job_key(strategy_id, symbols, start, end, param_grid, objective, direction, windows, bt_sig, params=None, overrides=None) -> str: - raw = f"WF|{strategy_id}|{symbols}|{start}|{end}|{param_grid}|{objective}|{direction}|{windows}|{bt_sig}|{params}|{overrides}" +def _make_wf_job_key( + strategy_id, + symbols, + start, + end, + param_grid, + objective, + direction, + windows, + bt_sig, + params=None, + overrides=None, + matrix_cache_max_mb=512, +) -> str: + raw = ( + f"WF|{strategy_id}|{symbols}|{start}|{end}|{param_grid}|{objective}|" + f"{direction}|{windows}|{bt_sig}|{params}|{overrides}|cache={matrix_cache_max_mb}" + ) return hashlib.md5(raw.encode()).hexdigest()[:12] @@ -780,6 +835,7 @@ async def walkforward_stream( test_days: int = 63, step_days: int = 63, max_workers: int = 4, + matrix_cache_max_mb: int = 512, params: str | None = None, # JSON: 未扫描参数固定为用户当前值 (base_params) overrides: str | None = None, # JSON: 策略当前的 basic_filter/signals/风控等覆盖 symbols: str | None = None, @@ -801,15 +857,10 @@ async def walkforward_stream( 事件: job {key} / progress {type:walkforward_progress,done,total,fold} / done {result} / error {message} """ - from app.backtest.optimizer import StrategyOptimizer - from app.backtest.strategy import StrategyBacktestService - from app.backtest.walkforward import WalkForwardConfig, WalkForwardService + from app.backtest.walkforward import WalkForwardConfig + from app.backtest.worker import make_worker_task, run_worker_task direction = direction or None - engine = _get_engine(request) - strategy_engine = request.app.state.strategy_engine - svc = StrategyBacktestService(engine, strategy_engine) - optimizer = StrategyOptimizer(svc, strategy_engine) end_date = date.fromisoformat(end) if end else date.today() if start: @@ -824,7 +875,20 @@ async def walkforward_stream( ) bt_sig = "|".join(f"{k}={bt_kwargs[k]}" for k in _OPT_BT_FIELDS) windows = f"{train_days}/{test_days}/{step_days}" - job_key = _make_wf_job_key(strategy_id, symbols, start, end, param_grid, objective, direction, windows, bt_sig, params, overrides) + job_key = _make_wf_job_key( + strategy_id, + symbols, + start, + end, + param_grid, + objective, + direction, + windows, + bt_sig, + params, + overrides, + matrix_cache_max_mb, + ) # guard 作用于单折窗口 (每折训练/测试各是一次回测), 而非总区间 —— WF 总区间可长达数年, # 按总区间拦会误杀; 真正的 OOM 风险在单折窗口过大。 @@ -889,18 +953,20 @@ async def walkforward_stream( base_params=base_params if isinstance(base_params, dict) else {}, overrides=ov if isinstance(ov, dict) else None, backtest_kwargs=bt_kwargs, + matrix_cache_max_mb=int(matrix_cache_max_mb), ) def _run_wf(): try: - wf = WalkForwardService(optimizer, svc, strategy_engine) - job.result = wf.run(wf_cfg, lambda d: job.progress.append(d), job.cancel_event) - job.done = True - job.finish_ts = time.time() + task = make_worker_task("walkforward", settings.data_dir, wf_cfg) + result = run_worker_task( + task, + lambda d: job.progress.append(d), + job.cancel_event, + ) + _finish_job(job, result=result) except Exception as e: - job.error = str(e) - job.done = True - job.finish_ts = time.time() + _finish_job(job, error=str(e)) threading.Thread(target=_run_wf, daemon=True).start() diff --git a/backend/app/api/monitor_rules.py b/backend/app/api/monitor_rules.py index 8abb126..67e15ee 100644 --- a/backend/app/api/monitor_rules.py +++ b/backend/app/api/monitor_rules.py @@ -4,6 +4,7 @@ """ from __future__ import annotations +from datetime import date from pathlib import Path from fastapi import APIRouter, HTTPException, Request @@ -143,6 +144,24 @@ def save_rule(req: RuleModel, request: Request): status_code=403, detail="封单监控需要 Pro+ 套餐 (批量五档能力),请升级后在「设置」页配置", ) + if rule.get("type") == "strategy": + from app.strategy.engine import StrategyDataContext + + strategy_engine = getattr(request.app.state, "strategy_engine", None) + if strategy_engine is None: + raise HTTPException(status_code=503, detail="策略引擎未初始化") + try: + strategy = strategy_engine.get(str(rule.get("strategy_id"))) + strategy_engine.validate_context( + strategy, + StrategyDataContext( + asset_type=str(rule.get("asset_type") or "stock"), + timeframe="1d", + as_of=date.today(), + ), + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e # 编辑现有规则时, 保留原 created_at (避免按时间排序时位置跳动) existing = monitor_rules.load_one(_data_dir(request), rule["id"]) if existing and existing.get("created_at"): @@ -150,7 +169,7 @@ def save_rule(req: RuleModel, request: Request): try: monitor_rules.validate(rule) except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) + raise HTTPException(status_code=400, detail=str(e)) from e monitor_rules.save_one(_data_dir(request), rule) _sync_engine(request) return {"ok": True, "rule": rule} diff --git a/backend/app/api/screener.py b/backend/app/api/screener.py index 4ffb30f..49e349b 100644 --- a/backend/app/api/screener.py +++ b/backend/app/api/screener.py @@ -14,8 +14,8 @@ from typing import Any, Optional from fastapi import APIRouter, HTTPException, Query, Request from pydantic import BaseModel -from app.services.screener import PRESET_STRATEGIES, ScreenerService, strategy_supports_asset from app.services import strategy_cache +from app.services.screener import ScreenerService from app.strategy import config as strategy_config logger = logging.getLogger(__name__) @@ -39,6 +39,7 @@ class PresetRequest(BaseModel): as_of: Optional[date] = None ext_columns: Optional[str] = None asset_type: str = "stock" + timeframe: str = "1d" def _safe(result_dict: dict) -> dict: @@ -100,6 +101,7 @@ def _load_ext_value_maps(repo, ext_columns: Optional[str]) -> dict[str, dict[str return {} import polars as pl + from app.api.ext_data import _read_ext_dataframe from app.services.ext_data import ExtConfigStore @@ -211,39 +213,31 @@ def _update_cache_strategy(data_dir, as_of: str, strategy_id: str, safe_data: di @router.get("/strategies") -def strategies(request: Request, asset_type: str = Query("stock")): - """策略清单（内置 + 自定义 + AI）。按 asset_type 过滤：ETF 仅返回技术类内置策略。""" +def strategies( + request: Request, + asset_type: str = Query("stock"), + timeframe: str = Query("1d"), +): + """兼容策略清单端点；唯一数据源为 StrategyEngine。""" data_dir = request.app.state.repo.store.data_dir - presets = [] - seen_ids: set[str] = set() - - # 内置策略 - for k, v in PRESET_STRATEGIES.items(): - if not strategy_supports_asset(v, asset_type): - continue - overrides = strategy_config.load_override(data_dir, k) - name = (overrides.get("name") or v["name"]) if overrides else v["name"] - desc = (overrides.get("description") or v["description"]) if overrides else v["description"] - presets.append({"id": k, "name": name, "description": desc, "source": "builtin"}) - seen_ids.add(k) - - # 自定义/AI 策略（不在 PRESET_STRATEGIES 中的）; 未标注资产类型, 保守仅 stock 返回 engine = getattr(request.app.state, "strategy_engine", None) - if engine and asset_type == "stock": - for meta in engine.list_strategies(): - sid = meta["id"] - if sid not in seen_ids: - overrides = strategy_config.load_override(data_dir, sid) - name = (overrides.get("name") or meta["name"]) if overrides else meta["name"] - desc = (overrides.get("description") or meta.get("description", "")) if overrides else meta.get("description", "") - presets.append({"id": sid, "name": name, "description": desc, "source": meta.get("source", "custom")}) - seen_ids.add(sid) - # 暴露加载失败的策略,让前端可见(避免"策略静默消失"误判为正常) - load_errors = engine.load_errors() if engine else [] - else: - load_errors = [] + if engine is None: + raise HTTPException(status_code=503, detail="策略引擎未初始化") + presets = [] + for meta in engine.list_strategies(): + if asset_type not in meta.get("asset_types", ["stock"]): + continue + if timeframe not in meta.get("timeframes", ["1d"]): + continue + sid = meta["id"] + overrides = strategy_config.load_override(data_dir, sid) + presets.append({ + **meta, + "name": overrides.get("name") or meta["name"], + "description": overrides.get("description") or meta.get("description", ""), + }) - return {"presets": presets, "load_errors": load_errors} + return {"presets": presets, "load_errors": engine.load_errors()} @router.post("/run") @@ -278,51 +272,34 @@ def run_preset(req: PresetRequest, request: Request): data_dir = request.app.state.repo.store.data_dir ext_values = _load_ext_value_maps(repo, req.ext_columns) overrides = strategy_config.load_override(data_dir, req.strategy_id) - bf = overrides.get("basic_filter") if overrides else None - dl = overrides.get("display_limit") if overrides else None - if dl is None and overrides and "display_limit" in overrides: - dl = 0 - engine = getattr(request.app.state, "strategy_engine", None) - - # 内置策略 - if req.strategy_id in PRESET_STRATEGIES: - filter_fn = None - if req.asset_type == "stock" and engine and engine.has(req.strategy_id): - filter_fn = engine.get(req.strategy_id).filter_fn - try: - result = svc.run_preset( - req.strategy_id, - as_of=as_of, - pool=req.pool, - basic_filter=bf, - filter_fn=filter_fn, - strategy_params=overrides.get("params") if overrides else None, - display_limit=dl, - ) - except ValueError as e: - raise HTTPException(status_code=404, detail=str(e)) from e - safe_data = _safe(asdict(result)) - _update_cache_strategy(data_dir, str(as_of), req.strategy_id, safe_data) - return _result_with_ext(safe_data, ext_values) - - # 自定义/AI 策略 — 通过 StrategyEngine 执行 if not engine: raise HTTPException(status_code=404, detail=f"策略引擎未初始化或策略 {req.strategy_id} 不存在") try: - result = engine.run(req.strategy_id, as_of, pool=req.pool, overrides=overrides or None) + if not engine.has(req.strategy_id): + raise ValueError(f"unknown strategy: {req.strategy_id}") + params = dict(overrides.get("params") or {}) + context = svc.build_strategy_context( + engine, + as_of, + [req.strategy_id], + timeframe=req.timeframe, + params_map={req.strategy_id: params}, + overrides_map={req.strategy_id: overrides or {}}, + ) + result = engine.run( + req.strategy_id, + context, + pool=req.pool, + params=params, + overrides=overrides or None, + ) except ValueError as e: - raise HTTPException(status_code=404, detail=str(e)) from e + status_code = 404 if "unknown strategy" in str(e) else 400 + raise HTTPException(status_code=status_code, detail=str(e)) from e - data = asdict(result) - - if dl is not None and dl > 0: - data["rows"] = data["rows"][:dl] - data["total"] = min(data["total"], dl) - - # 单跑后更新缓存中该策略的结果（保持缓存最新） - safe_data = _safe(data) + safe_data = _safe(asdict(result)) _update_cache_strategy(data_dir, str(as_of), req.strategy_id, safe_data) return _result_with_ext(safe_data, ext_values) @@ -403,18 +380,19 @@ def market_snapshot(request: Request): @router.post("/run_all") def run_all(request: Request, body: Optional[dict] = None): - """批量运行指定策略,只返回每个策略的命中数。 - - 优化: 从 enriched 读取一次目标日期数据, 所有策略共享。 - body.strategy_ids: 只跑指定的策略 ID 列表, 为空则跑全部。 - """ + """批量运行指定策略；注册、路由和执行均由 StrategyEngine 负责。""" from datetime import date as date_type t_total = time.perf_counter() body = body or {} repo = request.app.state.repo - svc = ScreenerService(repo) + asset_type = str(body.get("asset_type") or "stock") + timeframe = str(body.get("timeframe") or "1d") + svc = ScreenerService(repo, asset_type=asset_type) + engine = getattr(request.app.state, "strategy_engine", None) + if engine is None: + raise HTTPException(status_code=503, detail="策略引擎未初始化") # 解析日期 raw_date = body.get("as_of") @@ -425,27 +403,21 @@ def run_all(request: Request, body: Optional[dict] = None): if not as_of: return {"as_of": None, "results": {}} - # 一次读取目标日期的全部数据 - t0 = time.perf_counter() - precomputed = svc._load_enriched_for_date(as_of) - logger.info("run_all: _load_enriched_for_date took %.1fms", (time.perf_counter() - t0) * 1000) - - results: dict[str, dict] = {} data_dir = request.app.state.repo.store.data_dir - # 收集需要运行的策略 ID (如果指定了 strategy_ids 则只跑这些) requested_ids = body.get("strategy_ids") - all_ids = list(PRESET_STRATEGIES.keys()) - engine = getattr(request.app.state, "strategy_engine", None) - if engine: - for meta in engine.list_strategies(): - sid = meta["id"] - if sid not in PRESET_STRATEGIES: - all_ids.append(sid) - if requested_ids and isinstance(requested_ids, list): - id_set = set(requested_ids) - all_ids = [sid for sid in all_ids if sid in id_set] + all_ids = [str(sid) for sid in requested_ids] + unknown = [sid for sid in all_ids if not engine.has(sid)] + if unknown: + raise HTTPException(status_code=404, detail=f"unknown strategies: {unknown}") + else: + all_ids = [ + meta["id"] + for meta in engine.list_strategies() + if asset_type in meta.get("asset_types", ["stock"]) + and timeframe in meta.get("timeframes", ["1d"]) + ] if not all_ids: return {"as_of": str(as_of), "results": {}} @@ -455,54 +427,37 @@ def run_all(request: Request, body: Optional[dict] = None): all_overrides = strategy_config.list_overrides(data_dir) logger.info("run_all: list_overrides took %.1fms (%d overrides)", (time.perf_counter() - t0) * 1000, len(all_overrides)) - # 历史策略: 只在需要时加载 (只加载 all_ids 中包含的 filter_history 策略) - t0 = time.perf_counter() - shared_history = None - id_set = set(all_ids) - if engine: - history_strats = [ - (sid, s) for sid, s in engine._strategies.items() - if s.filter_history_fn and sid in id_set - ] - if history_strats: - max_lb = min(max(s.lookback_days for _, s in history_strats), 30) - shared_history = svc._load_enriched_history(as_of, max(1, max_lb)) - else: - history_strats = [] - logger.info("run_all: _load_enriched_history took %.1fms (history_strats=%d)", (time.perf_counter() - t0) * 1000, len(history_strats)) + params_map = { + sid: dict((all_overrides.get(sid) or {}).get("params") or {}) + for sid in all_ids + } + overrides_map = {sid: all_overrides.get(sid, {}) for sid in all_ids} + try: + context = svc.build_strategy_context( + engine, + as_of, + all_ids, + timeframe=timeframe, + params_map=params_map, + overrides_map=overrides_map, + ) + engine_results = engine.run_all( + context, + params_map=params_map, + overrides_map=overrides_map, + strategy_ids=all_ids, + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e - for sid in all_ids: - try: - overrides = all_overrides.get(sid, {}) - bf = overrides.get("basic_filter") if overrides else None - dl = overrides.get("display_limit") if overrides else None - if dl is None and overrides and "display_limit" in overrides: - dl = 0 - - if sid in PRESET_STRATEGIES: - filter_fn = engine.get(sid).filter_fn if engine and engine.has(sid) else None - r = svc.run_preset( - sid, - as_of=as_of, - precomputed=precomputed, - basic_filter=bf, - filter_fn=filter_fn, - strategy_params=overrides.get("params") if overrides else None, - display_limit=dl, - ) - else: - r = engine.run( - sid, as_of, overrides=overrides or None, - precomputed=precomputed, precomputed_history=shared_history, - ) - if dl is not None and dl > 0: - r.rows = r.rows[:dl] - r.total = min(r.total, dl) - - safe_rows = _safe(asdict(r)).get("rows", []) - results[sid] = {"total": r.total, "as_of": str(as_of), "rows": safe_rows} - except (ValueError, Exception): - continue + results: dict[str, dict] = {} + for sid, result in engine_results.items(): + safe_rows = _safe(asdict(result)).get("rows", []) + results[sid] = { + "total": result.total, + "as_of": str(as_of), + "rows": safe_rows, + } elapsed = (time.perf_counter() - t_total) * 1000 logger.info("run_all: total took %.1fms (%d strategies)", elapsed, len(all_ids)) diff --git a/backend/app/api/strategy.py b/backend/app/api/strategy.py index dc951ad..3706eaf 100644 --- a/backend/app/api/strategy.py +++ b/backend/app/api/strategy.py @@ -46,6 +46,15 @@ def _data_dir(request: Request) -> Path: return request.app.state.repo.store.data_dir +def _invalidate_strategy_runtime(request: Request) -> None: + from app.services import strategy_cache + + strategy_cache.clear_cache(_data_dir(request)) + monitor_engine = getattr(request.app.state, "monitor_engine", None) + if monitor_engine is not None: + monitor_engine.invalidate_strategy_state() + + def _safe(result_dict: dict) -> dict: rows = result_dict.get("rows", []) for r in rows: @@ -80,6 +89,9 @@ def _strategy_detail(s: StrategyDef, overrides: dict | None = None) -> dict: "description": description or s.meta.get("description", ""), "tags": s.meta.get("tags", []), "source": s.source, + "execution_backend": s.execution_backend, + "asset_types": s.meta.get("asset_types", ["stock"]), + "timeframes": s.meta.get("timeframes", ["1d"]), "version": s.meta.get("version", "1.0.0"), "basic_filter": bf, "params": s.meta.get("params", []), @@ -109,10 +121,14 @@ class RunRequest(BaseModel): as_of: date | None = None pool: list[str] | None = None params: dict | None = None + asset_type: str = "stock" + timeframe: str = "1d" class RunAllRequest(BaseModel): as_of: date | None = None + asset_type: str = "stock" + timeframe: str = "1d" class SaveConfigRequest(BaseModel): @@ -155,18 +171,26 @@ class MonitorStartRequest(BaseModel): @router.get("") -def list_strategies(request: Request): +def list_strategies( + request: Request, + asset_type: str | None = None, + timeframe: str | None = None, +): engine = _get_engine(request) data_dir = _data_dir(request) all_overrides = strategy_config.list_overrides(data_dir) result = [] for meta in engine.list_strategies(): + if asset_type and asset_type not in meta.get("asset_types", ["stock"]): + continue + if timeframe and timeframe not in meta.get("timeframes", ["1d"]): + continue sid = meta["id"] s = engine.get(sid) overrides = all_overrides.get(sid) result.append(_strategy_detail(s, overrides)) - return {"strategies": result} + return {"strategies": result, "load_errors": engine.load_errors()} @router.get("/{strategy_id}") @@ -201,14 +225,25 @@ def run_strategy(req: RunRequest, request: Request): as_of = req.as_of if not as_of: from app.services.screener import ScreenerService - svc = ScreenerService(request.app.state.repo) + svc = ScreenerService(request.app.state.repo, asset_type=req.asset_type) as_of = svc.latest_date() if not as_of: raise HTTPException(status_code=400, detail="无可用数据日期") try: + from app.services.screener import ScreenerService + svc = ScreenerService(request.app.state.repo, asset_type=req.asset_type) + context = svc.build_strategy_context( + engine, + as_of, + [req.strategy_id], + timeframe=req.timeframe, + params_map={req.strategy_id: params}, + overrides_map={req.strategy_id: overrides or {}}, + ) result = engine.run( - req.strategy_id, as_of, + req.strategy_id, + context, pool=req.pool, params=params, overrides=overrides or None, @@ -227,14 +262,39 @@ def run_all(req: RunAllRequest, request: Request): as_of = req.as_of if not as_of: from app.services.screener import ScreenerService - svc = ScreenerService(request.app.state.repo) + svc = ScreenerService(request.app.state.repo, asset_type=req.asset_type) as_of = svc.latest_date() if not as_of: return {"as_of": None, "results": {}} all_overrides = strategy_config.list_overrides(data_dir) + strategy_ids = [ + meta["id"] + for meta in engine.list_strategies() + if req.asset_type in meta.get("asset_types", ["stock"]) + and req.timeframe in meta.get("timeframes", ["1d"]) + ] + from app.services.screener import ScreenerService + svc = ScreenerService(request.app.state.repo, asset_type=req.asset_type) + params_map = { + sid: dict((all_overrides.get(sid) or {}).get("params") or {}) + for sid in strategy_ids + } + context = svc.build_strategy_context( + engine, + as_of, + strategy_ids, + timeframe=req.timeframe, + params_map=params_map, + overrides_map={sid: all_overrides.get(sid, {}) for sid in strategy_ids}, + ) results: dict[str, dict] = {} - for sid, result in engine.run_all(as_of, overrides_map=all_overrides).items(): + for sid, result in engine.run_all( + context, + params_map=params_map, + overrides_map={sid: all_overrides.get(sid, {}) for sid in strategy_ids}, + strategy_ids=strategy_ids, + ).items(): results[sid] = {"total": result.total, "as_of": str(as_of)} return {"as_of": str(as_of), "results": results} @@ -301,6 +361,7 @@ class BuildRequest(BaseModel): direction: str = "long" rules: str = "" strategy_id: str = "" + execution_backend: Literal["polars_expr", "matrix_native"] = "polars_expr" # step2 字段 current_code: str = "" instruction: str = "" @@ -514,6 +575,8 @@ def _save_strategy_code(req: StrategyCodeSaveRequest, request: Request, *, legac engine.reload() raise ValueError(f"策略保存失败: {e}") from e + _invalidate_strategy_runtime(request) + return { "ok": True, "strategy_id": sid, @@ -581,7 +644,14 @@ async def ai_test(request: Request): def _build_prompt(req: BuildRequest) -> str: if req.step == 1: - return build_step1(req.name, req.description, req.direction, req.rules, req.strategy_id) + return build_step1( + req.name, + req.description, + req.direction, + req.rules, + req.strategy_id, + req.execution_backend, + ) if req.step == 2: return build_step2(req.current_code, req.instruction) raise ValueError(f"无效步骤: {req.step}") @@ -699,18 +769,22 @@ def delete_strategy(strategy_id: str, request: Request): if s.source == "builtin": raise HTTPException(status_code=403, detail="内置策略不可删除") - # 删除策略文件 - if s.file_path and s.file_path.exists(): - s.file_path.unlink() - - # 删除 overrides data_dir = _data_dir(request) - override_path = data_dir / "user_data" / "strategy_overrides" / f"{strategy_id}.json" - if override_path.exists(): - override_path.unlink() + path = s.file_path + previous_code = path.read_text(encoding="utf-8") if path and path.exists() else None + if path and path.exists(): + path.unlink() + try: + engine.reload() + except Exception as e: + if path is not None and previous_code is not None: + path.write_text(previous_code, encoding="utf-8") + engine.reload() + raise HTTPException(status_code=400, detail=f"策略删除失败: {e}") from e - # 热重载 - engine.reload() + override_path = data_dir / "user_data" / "strategy_overrides" / f"{strategy_id}.json" + override_path.unlink(missing_ok=True) + _invalidate_strategy_runtime(request) return {"ok": True} @@ -725,5 +799,9 @@ def delete_strategy(strategy_id: str, request: Request): @router.post("/reload") def reload_strategies(request: Request): engine = _get_engine(request) - engine.reload() + try: + engine.reload() + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + _invalidate_strategy_runtime(request) return {"ok": True, "count": len(engine.list_strategies())} diff --git a/backend/app/backtest/engine.py b/backend/app/backtest/engine.py index b3b43e8..9bcc25d 100644 --- a/backend/app/backtest/engine.py +++ b/backend/app/backtest/engine.py @@ -9,22 +9,36 @@ import logging import threading import time from collections import OrderedDict +from collections.abc import Callable from dataclasses import dataclass from datetime import date -from typing import Callable - -logger = logging.getLogger(__name__) from typing import Literal import numpy as np import polars as pl +import pyarrow as pa +from app.backtest.matrix import ( + MarketDataMatrix, + MarketMatrix, + MatrixCacheProfile, + build_market_matrix, + load_market_data_matrix_from_parquet, +) +from app.config import settings from app.parquet import scan_enriched_parquet from app.tickflow.repository import KlineRepository logger = logging.getLogger(__name__) +def _matrix_entry_score(matrix: MarketMatrix, time_id: int, asset_id: int) -> float: + source_time = int(matrix.entry_signal_time[time_id, asset_id]) + if source_time < 0: + return 0.0 + return float(matrix.score[source_time, asset_id]) + + # ================================================================ # 数据结构 # ================================================================ @@ -116,6 +130,17 @@ class SimResult: stats: dict +@dataclass(frozen=True) +class SimulationOptions: + """Controls expensive result materialization without changing matching semantics.""" + + include_monte_carlo: bool = True + include_curves: bool = True + include_trades: bool = True + include_per_symbol_stats: bool = True + include_return_distribution: bool = True + + def _resolve_signal_id(panel: pl.DataFrame, idx: int, signal_ids: list[str] | None) -> str | None: """在触发行 idx 上, 从候选信号里找出 panel 列为 True 的那个, 返回其列名。 @@ -288,6 +313,145 @@ class BacktestEngine: """加载 enriched 数据面板，带缓存。asset_type='etf' 时读 ETF enriched。""" return self._cache.get_or_compute(symbols, start, end, columns, self._load_panel_inner, asset_type=asset_type) + def load_panel_for_backtest( + self, + symbols: list[str] | None, + start: date, + end: date, + feature_plan, + asset_type: str = "stock", + ) -> pl.DataFrame: + """按解析后的依赖加载窄基础列并计算回测所需特征。""" + from app.indicators.pipeline import ( + compute_indicators, + compute_limit_signals, + compute_signals, + ) + + df = self.load_panel( + symbols, + start, + end, + columns=sorted(feature_plan.base_columns), + asset_type=asset_type, + ) + if df.is_empty(): + return df + + instruments = ( + self.repo.get_instruments_asset(asset_type) + if self.repo is not None + else pl.DataFrame() + ) + matrix_native = feature_plan.execution_backend == "matrix_native" + if not matrix_native: + df = compute_indicators(df, needed=set(feature_plan.indicator_columns)) + df = compute_signals(df, needed=set(feature_plan.signal_columns)) + if not instruments.is_empty(): + df = compute_limit_signals( + df, + instruments, + needed={"signal_limit_up", "signal_limit_down"} + if matrix_native + else set(feature_plan.signal_columns), + ) + join_cols = ["symbol"] if "symbol" in instruments.columns else [] + join_cols.extend( + col for col in sorted(feature_plan.instrument_columns) + if col in instruments.columns and col not in df.columns + ) + if len(join_cols) > 1: + df = df.join( + instruments.select(join_cols).unique(subset=["symbol"]), + on="symbol", + how="left", + ) + + required_columns = set(feature_plan.base_columns) | set(feature_plan.instrument_columns) + required_columns |= set(feature_plan.signal_columns) + if not matrix_native: + required_columns |= set(feature_plan.indicator_columns) + missing_columns = required_columns - set(df.columns) + if missing_columns: + raise ValueError(f"回测字段依赖未生成: {sorted(missing_columns)}") + + float_cols = [c for c in df.columns if df[c].dtype.is_float()] + if float_cols: + df = df.with_columns([ + pl.when(pl.col(c).is_nan() | pl.col(c).is_infinite()) + .then(None) + .otherwise(pl.col(c)) + .alias(c) + for c in float_cols + ]) + return df + + def load_market_data_matrix_for_backtest( + self, + symbols: list[str] | None, + start: date, + end: date, + feature_plan, + asset_type: str = "stock", + *, + cache_profile: MatrixCacheProfile | None = None, + coverage_start: date | None = None, + coverage_end: date | None = None, + ) -> MarketDataMatrix: + """Load a matrix-native backtest directly from projected parquet batches.""" + if feature_plan.execution_backend != "matrix_native": + raise ValueError("direct market matrix loading requires matrix_native backend") + from app.tickflow.repository import enriched_dirname + + parquet_root = self.repo.store.data_dir / enriched_dirname(asset_type) + instruments = self.repo.get_instruments_asset(asset_type) + field_columns = ( + set(feature_plan.base_columns) + | set(feature_plan.instrument_columns) + | set(feature_plan.matrix_columns) + ) + cache_root = ( + self.repo.store.data_dir / ".backtest_matrix_cache" + if settings.backtest_matrix_disk_cache_enabled + else None + ) + cache_fields = ( + cache_profile.field_columns + if cache_profile is not None + else frozenset(field_columns) + ) + cache_max_bytes = ( + cache_profile.max_disk_bytes + if cache_profile is not None + else settings.backtest_matrix_cache_max_mb * 1024 * 1024 + ) + generation_loader = getattr(self.repo, "get_matrix_data_generation", None) + source_generation = ( + generation_loader(asset_type) + if cache_root is not None and callable(generation_loader) + else None + ) + try: + return load_market_data_matrix_from_parquet( + parquet_root, + start, + end, + field_columns=field_columns, + symbols=symbols, + instruments=instruments, + cache_root=cache_root, + coverage_start=coverage_start, + coverage_end=coverage_end, + cache_field_columns=cache_fields, + cache_max_bytes=cache_max_bytes, + profile_generation=( + cache_profile.generation if cache_profile is not None else "request" + ), + source_generation=source_generation, + ) + except pa.ArrowException as exc: + raise ValueError(f"direct market matrix parquet scan failed: {exc}") from exc + def cache_stats(self) -> dict: """暴露 PanelCache 遥测快照 (扫盘耗时/次数/命中/复用), 供上层量化 IO 占比。""" return self._cache.stats() @@ -304,7 +468,7 @@ class BacktestEngine: # 近期区间优先复用 repository 的预计算 enriched 历史缓存 (仅 stock: 该缓存为股票专用)。 try: - if asset_type == "stock" and self.repo is not None and hasattr(self.repo, "get_enriched_range"): + if columns is None and asset_type == "stock" and self.repo is not None and hasattr(self.repo, "get_enriched_range"): cached = self.repo.get_enriched_range(start, end, symbols=symbols, columns=columns) if cached is not None and not cached.is_empty(): elapsed = (time.perf_counter() - t0) * 1000 @@ -515,6 +679,328 @@ class BacktestEngine: cancel_event: "threading.Event | None" = None, entry_signal_ids: list[str] | None = None, exit_signal_ids: list[str] | None = None, + ) -> SimResult: + """Execute every candidate independently on MarketMatrix asset columns.""" + if panel.is_empty(): + return self._empty_result() + raw_candidates = int(entries.fill_null(False).sum()) if entries is not None and len(entries) == len(panel) else 0 + if raw_candidates <= 0: + return self._empty_result() + matrix = build_market_matrix( + panel, + entries, + exits, + entry_delay_bars=1 if config.entry_fill == "open_t+1" else 0, + exit_delay_bars=1 if config.exit_fill == "open_t+1" else 0, + entry_signal_ids=entry_signal_ids, + exit_signal_ids=exit_signal_ids, + ) + return self._simulate_independent_matrix( + matrix, raw_candidates, config, progress_cb, cancel_event, + ) + + def _simulate_independent_matrix( + self, + matrix: MarketMatrix, + raw_candidates: int, + config: MatcherConfig, + progress_cb: "Callable[[dict], None] | None", + cancel_event: "threading.Event | None", + options: SimulationOptions | None = None, + ) -> SimResult: + options = options or SimulationOptions() + entry_prices = matrix.open if config.entry_fill == "open_t+1" else matrix.close + exit_prices = matrix.open if config.exit_fill == "open_t+1" else matrix.close + buy_cost_pct = config.buy_cost_pct() + sell_cost_pct = config.sell_cost_pct() + trades: list[TradeRecord] = [] + execution_stats = { + "buy_invalid_price": 0, + "buy_suspended": 0, + "buy_limit_up": 0, + "buy_score_filter": 0, + "buy_no_next_bar": max(raw_candidates - int(matrix.entry.sum()), 0), + "sell_invalid_price": 0, + "sell_suspended": 0, + "sell_limit_down": 0, + "sell_no_future": 0, + "pending_exit": 0, + } + + minute_cache: dict = {} + if config.minute_fill: + trigger_times, trigger_assets = np.nonzero(matrix.entry | matrix.exit) + dates = {matrix.timestamp_labels[int(t)][:10] for t in trigger_times} + symbols = {matrix.symbols[int(a)] for a in trigger_assets} + if dates and symbols: + loaded = self._load_minute_for_fills(self.repo, list(symbols), dates, "stock") + minute_cache = {key: value for key, value in loaded.items() if value is not None and len(value) > 0} + + def _count(key: str) -> None: + execution_stats[key] = execution_stats.get(key, 0) + 1 + + def _valid_price(value) -> bool: + return bool(np.isfinite(value) and value > 0) + + def _present(time_id: int, asset_id: int) -> bool: + return bool(np.isfinite([ + matrix.open[time_id, asset_id], matrix.high[time_id, asset_id], + matrix.low[time_id, asset_id], matrix.close[time_id, asset_id], + ]).any()) + + def _signal_id(code: int, ids: tuple[str, ...]) -> str | None: + return ids[code] if 0 <= code < len(ids) else None + + def _signal_date(signal_time: int, fallback: str) -> str: + return matrix.timestamp_labels[signal_time][:10] if signal_time >= 0 else fallback + + def _refill(time_id: int, asset_id: int, side: str, daily_price: float) -> float: + if not config.minute_fill or not minute_cache: + return daily_price + rows = minute_cache.get((matrix.symbols[asset_id], matrix.timestamp_labels[time_id][:10])) + if rows is None: + return daily_price + reference = float(matrix.reference_price[time_id, asset_id]) + precise = self._resolve_minute_fill( + rows, reference if _valid_price(reference) else None, side, + ) + return precise if precise is not None else daily_price + + def _one_price_limit(time_id: int, asset_id: int, direction: str) -> bool: + if not matrix.tradable[time_id, asset_id]: + return False + prices = [ + float(matrix.open[time_id, asset_id]), float(matrix.high[time_id, asset_id]), + float(matrix.low[time_id, asset_id]), float(matrix.close[time_id, asset_id]), + ] + if not all(_valid_price(value) for value in prices): + return False + same = max(prices) - min(prices) <= max(abs(prices[3]) * 1e-4, 0.01) + flags = matrix.limit_up_locked if direction == "up" else matrix.limit_down_locked + return bool(flags[time_id, asset_id]) and same + + def _can_buy(time_id: int, asset_id: int) -> tuple[bool, str]: + if not matrix.tradable[time_id, asset_id]: + return False, "buy_suspended" + if not _valid_price(entry_prices[time_id, asset_id]): + return False, "buy_invalid_price" + if _one_price_limit(time_id, asset_id, "up"): + return False, "buy_limit_up" + return True, "" + + def _can_sell(time_id: int, asset_id: int, override: float | None) -> tuple[bool, str]: + if not matrix.tradable[time_id, asset_id]: + return False, "sell_suspended" + price = override if override is not None else exit_prices[time_id, asset_id] + if not _valid_price(price): + return False, "sell_invalid_price" + if _one_price_limit(time_id, asset_id, "down"): + return False, "sell_limit_down" + return True, "" + + def _risk_exit(pos: dict, time_id: int, asset_id: int) -> tuple[str | None, float | None]: + if pos.get("pending_exit_reason") or pos["entry_time"] == time_id: + return None, None + entry_price = float(pos["entry_price"]) + open_price = float(matrix.open[time_id, asset_id]) + low_price = float(matrix.low[time_id, asset_id]) + high_price = float(matrix.high[time_id, asset_id]) + peak_price = float(pos["max_high"]) + lines: list[tuple[float, str]] = [] + if config.stop_loss_pct is not None: + lines.append((entry_price * (1 - abs(config.stop_loss_pct)), "stop_loss")) + if config.trailing_stop_pct is not None: + lines.append((peak_price * (1 - abs(config.trailing_stop_pct)), "trailing_stop")) + activate = config.trailing_take_profit_activate_pct + drawdown = config.trailing_take_profit_drawdown_pct + if activate is not None and drawdown is not None and peak_price > entry_price: + if peak_price / entry_price - 1 >= abs(float(activate)): + lines.append((peak_price * (1 - abs(float(drawdown))), "trailing_take_profit")) + valid_lines = [(line, reason) for line, reason in lines if _valid_price(line)] + if valid_lines: + stop_price, reason = max(valid_lines, key=lambda item: item[0]) + if _valid_price(open_price) and open_price <= stop_price: + return reason, open_price + if _valid_price(low_price) and low_price <= stop_price: + return reason, stop_price + if config.take_profit_pct is not None: + take_profit = entry_price * (1 + abs(float(config.take_profit_pct))) + if _valid_price(open_price) and open_price >= take_profit: + return "take_profit", open_price + if _valid_price(high_price) and high_price >= take_profit: + return "take_profit", take_profit + return None, None + + def _try_close( + pos: dict, + time_id: int, + asset_id: int, + reason: str, + signal_date: str, + override: float | None = None, + ) -> bool: + ok, blocked = _can_sell(time_id, asset_id, override) + if not ok: + if not pos.get("pending_exit_reason"): + pos["pending_exit_reason"] = reason + pos["pending_exit_signal_date"] = signal_date + _count("pending_exit") + pos["blocked_exit_days"] += 1 + _count(blocked) + return False + exit_price = float(override) if override is not None else _refill( + time_id, asset_id, "sell", float(exit_prices[time_id, asset_id]) + ) + shares = 100.0 + entry_value = shares * pos["entry_price"] * (1 + buy_cost_pct) + exit_value = shares * exit_price * (1 - sell_cost_pct) + pnl_amount = exit_value - entry_value + trades.append(TradeRecord( + symbol=matrix.symbols[asset_id], + name=matrix.names[asset_id], + entry_date=pos["entry_date"], + exit_date=matrix.timestamp_labels[time_id][:10], + entry_price=round(float(pos["entry_price"]), 4), + exit_price=round(exit_price, 4), + pnl_pct=round(float(pnl_amount / entry_value), 6) if entry_value > 0 else 0.0, + duration=int(pos["hold_days"]), + exit_reason=reason, + shares=shares, + lots=1.0, + entry_value=round(float(entry_value), 2), + exit_value=round(float(exit_value), 2), + pnl_amount=round(float(pnl_amount), 2), + entry_score=round(float(pos["entry_score"]), 2), + entry_signal_date=pos["entry_signal_date"], + exit_signal_date=signal_date, + blocked_exit_days=int(pos["blocked_exit_days"]), + entry_signal_id=pos["entry_signal_id"], + exit_signal_id=_signal_id( + int(matrix.exit_signal_code[time_id, asset_id]), matrix.exit_signal_ids + ) if reason == "signal" else None, + )) + return True + + entry_times, entry_assets = np.nonzero(matrix.entry) + order = np.lexsort((entry_times, entry_assets)) + for seq, order_pos in enumerate(order, start=1): + time_id = int(entry_times[order_pos]) + asset_id = int(entry_assets[order_pos]) + if cancel_event is not None and cancel_event.is_set(): + break + if progress_cb is not None and (seq == 1 or seq % 500 == 0): + try: + progress_cb({ + "day": seq, + "total": len(order), + "date": matrix.timestamp_labels[time_id][:10], + "equity": 0, + }) + except Exception: + pass + ok, blocked = _can_buy(time_id, asset_id) + if not ok: + _count(blocked) + continue + score = _matrix_entry_score(matrix, time_id, asset_id) + if config.score_min is not None and score < config.score_min: + _count("buy_score_filter") + continue + if config.score_max is not None and score > config.score_max: + _count("buy_score_filter") + continue + future_times = [ + future for future in range(time_id + 1, matrix.shape[0]) + if _present(future, asset_id) + ] + if not future_times: + _count("sell_no_future") + continue + entry_price = _refill(time_id, asset_id, "buy", float(entry_prices[time_id, asset_id])) + entry_date = matrix.timestamp_labels[time_id][:10] + pos = { + "entry_time": time_id, + "entry_date": entry_date, + "entry_signal_date": _signal_date( + int(matrix.entry_signal_time[time_id, asset_id]), entry_date + ), + "entry_signal_id": _signal_id( + int(matrix.entry_signal_code[time_id, asset_id]), matrix.entry_signal_ids + ), + "entry_price": entry_price, + "entry_score": score, + "hold_days": 0, + "max_high": max(entry_price, float(matrix.high[time_id, asset_id])), + "pending_exit_reason": None, + "pending_exit_signal_date": None, + "blocked_exit_days": 0, + } + closed = False + for future in future_times: + pos["hold_days"] += 1 + date_text = matrix.timestamp_labels[future][:10] + reason, override = _risk_exit(pos, future, asset_id) + if reason and _try_close(pos, future, asset_id, reason, date_text, override): + closed = True + break + reason = None + signal_date = date_text + if pos.get("pending_exit_reason"): + reason = str(pos["pending_exit_reason"]) + signal_date = str(pos.get("pending_exit_signal_date") or date_text) + elif matrix.exit[future, asset_id]: + reason = "signal" + signal_date = _signal_date(int(matrix.exit_signal_time[future, asset_id]), date_text) + elif config.max_hold_days is not None and pos["hold_days"] >= config.max_hold_days: + reason = "max_hold" + elif future == future_times[-1]: + reason = "end" + if reason and _try_close(pos, future, asset_id, reason, signal_date): + closed = True + break + high_price = float(matrix.high[future, asset_id]) + if _valid_price(high_price): + pos["max_high"] = max(float(pos["max_high"]), high_price) + if not closed and not pos.get("pending_exit_reason"): + last_time = future_times[-1] + _try_close( + pos, last_time, asset_id, "end", matrix.timestamp_labels[last_time][:10] + ) + + result = self._calc_independent_candidate_result( + trades, + raw_candidates, + execution_stats, + options=options, + ) + result.stats["market_matrix_shape"] = list(matrix.shape) + result.stats["market_matrix_bytes"] = matrix.nbytes + return result + + def simulate_independent_market_matrix( + self, + matrix: MarketMatrix, + raw_candidates: int, + config: MatcherConfig, + progress_cb: "Callable[[dict], None] | None" = None, + cancel_event: "threading.Event | None" = None, + options: SimulationOptions | None = None, + ) -> SimResult: + """Run independent-candidate simulation on a prebuilt MarketMatrix.""" + return self._simulate_independent_matrix( + matrix, raw_candidates, config, progress_cb, cancel_event, options, + ) + + def simulate_independent_candidates_legacy( + self, + panel: pl.DataFrame, + entries: pl.Series | None, + exits: pl.Series | None, + config: MatcherConfig, + progress_cb: "Callable[[dict], None] | None" = None, + cancel_event: "threading.Event | None" = None, + entry_signal_ids: list[str] | None = None, + exit_signal_ids: list[str] | None = None, ) -> SimResult: """全量候选独立执行：每个买入信号都是独立样本, 不受资金/仓位限制。""" if panel.is_empty(): @@ -1016,6 +1502,464 @@ class BacktestEngine: cancel_event: "threading.Event | None" = None, entry_signal_ids: list[str] | None = None, exit_signal_ids: list[str] | None = None, + ) -> SimResult: + """Account-level matcher backed by immutable ``time x asset`` arrays.""" + if panel.is_empty(): + return self._empty_result() + + matrix = build_market_matrix( + panel, + entries, + exits, + entry_delay_bars=1 if config.entry_fill == "open_t+1" else 0, + exit_delay_bars=1 if config.exit_fill == "open_t+1" else 0, + entry_signal_ids=entry_signal_ids, + exit_signal_ids=exit_signal_ids, + ) + if not matrix.entry.any(): + return self._empty_result() + return self._simulate_portfolio_matrix(matrix, config, progress_cb, cancel_event) + + def simulate_market_matrix( + self, + matrix: MarketMatrix, + config: MatcherConfig, + progress_cb: "Callable[[dict], None] | None" = None, + cancel_event: "threading.Event | None" = None, + options: SimulationOptions | None = None, + ) -> SimResult: + """Run the production Python matcher on a prebuilt MarketMatrix.""" + if not matrix.entry.any(): + return self._empty_result() + return self._simulate_portfolio_matrix(matrix, config, progress_cb, cancel_event, options) + + def _simulate_portfolio_matrix( + self, + matrix: MarketMatrix, + config: MatcherConfig, + progress_cb: "Callable[[dict], None] | None", + cancel_event: "threading.Event | None", + options: SimulationOptions | None = None, + ) -> SimResult: + options = options or SimulationOptions() + time_count, asset_count = matrix.shape + entry_prices = matrix.open if config.entry_fill == "open_t+1" else matrix.close + exit_prices = matrix.open if config.exit_fill == "open_t+1" else matrix.close + buy_cost_pct = config.buy_cost_pct() + sell_cost_pct = config.sell_cost_pct() + cash = float(config.initial_capital) + peak = cash + max_positions = max(int(config.max_positions), 0) + max_exposure_pct = min(max(float(config.max_exposure_pct), 0.0), 1.0) + positions: dict[int, dict] = {} + last_close = np.full(asset_count, np.nan, dtype=np.float64) + trades: list[TradeRecord] = [] + equity_curve: list[dict] = [] + drawdown_curve: list[dict] = [] + equity_values: list[float] = [] + exposure_values: list[float] = [] + execution_stats = { + "buy_invalid_price": 0, + "buy_suspended": 0, + "buy_limit_up": 0, + "buy_no_slot": 0, + "buy_cash": 0, + "buy_lot_size": 0, + "buy_same_day_reentry": 0, + "buy_exposure": 0, + "buy_score_filter": 0, + "sell_invalid_price": 0, + "sell_suspended": 0, + "sell_limit_down": 0, + "pending_exit": 0, + } + + minute_cache: dict = {} + if config.minute_fill: + trigger_times, trigger_assets = np.nonzero(matrix.entry | matrix.exit) + trigger_dates = {matrix.timestamp_labels[int(t)][:10] for t in trigger_times} + trigger_symbols = {matrix.symbols[int(a)] for a in trigger_assets} + if trigger_dates and trigger_symbols: + asset_type = "etf" if all( + symbol.endswith(".SH") and symbol.startswith("5") + for symbol in list(trigger_symbols)[:5] + ) else "stock" + loaded = self._load_minute_for_fills( + self.repo, list(trigger_symbols), trigger_dates, asset_type, + ) + minute_cache = {key: value for key, value in loaded.items() if value is not None and len(value) > 0} + + def _count(key: str) -> None: + execution_stats[key] = execution_stats.get(key, 0) + 1 + + def _valid_price(value) -> bool: + return bool(np.isfinite(value) and value > 0) + + def _signal_id(code: int, signal_ids: tuple[str, ...]) -> str | None: + return signal_ids[code] if 0 <= code < len(signal_ids) else None + + def _signal_date(signal_time: int, fallback: str) -> str: + return matrix.timestamp_labels[signal_time][:10] if signal_time >= 0 else fallback + + def _market_value() -> float: + total = 0.0 + for asset, pos in positions.items(): + mark = last_close[asset] + if not _valid_price(mark): + mark = pos["entry_price"] + total += pos["shares"] * mark + return total + + def _refill_price(time_id: int, asset_id: int, side: str, daily_price: float) -> float: + if not config.minute_fill or not minute_cache: + return daily_price + key = (matrix.symbols[asset_id], matrix.timestamp_labels[time_id][:10]) + minute_rows = minute_cache.get(key) + if minute_rows is None: + return daily_price + reference = float(matrix.reference_price[time_id, asset_id]) + precise = self._resolve_minute_fill( + minute_rows, + reference if _valid_price(reference) else None, + side, + ) + return precise if precise is not None else daily_price + + def _one_price_limit(time_id: int, asset_id: int, direction: str) -> bool: + if not matrix.tradable[time_id, asset_id]: + return False + prices = ( + float(matrix.open[time_id, asset_id]), + float(matrix.high[time_id, asset_id]), + float(matrix.low[time_id, asset_id]), + float(matrix.close[time_id, asset_id]), + ) + if not all(_valid_price(value) for value in prices): + return False + same_price = max(prices) - min(prices) <= max(abs(prices[3]) * 1e-4, 0.01) + flag = matrix.limit_up_locked if direction == "up" else matrix.limit_down_locked + return bool(flag[time_id, asset_id]) and same_price + + def _can_buy(time_id: int, asset_id: int) -> tuple[bool, str]: + if not matrix.tradable[time_id, asset_id]: + return False, "buy_suspended" + if not _valid_price(entry_prices[time_id, asset_id]): + return False, "buy_invalid_price" + if _one_price_limit(time_id, asset_id, "up"): + return False, "buy_limit_up" + return True, "" + + def _can_sell(time_id: int, asset_id: int, override: float | None = None) -> tuple[bool, str]: + if not matrix.tradable[time_id, asset_id]: + return False, "sell_suspended" + price = override if override is not None else exit_prices[time_id, asset_id] + if not _valid_price(price): + return False, "sell_invalid_price" + if _one_price_limit(time_id, asset_id, "down"): + return False, "sell_limit_down" + return True, "" + + def _mark_pending(asset_id: int, reason: str, signal_date: str) -> None: + pos = positions[asset_id] + if not pos.get("pending_exit_reason"): + pos["pending_exit_reason"] = reason + pos["pending_exit_signal_date"] = signal_date + _count("pending_exit") + pos["blocked_exit_days"] += 1 + + def _sell( + time_id: int, + asset_id: int, + reason: str, + signal_date: str, + sold_today: set[int], + override: float | None = None, + ) -> None: + nonlocal cash + pos = positions.pop(asset_id) + exit_price = float(override) if override is not None else _refill_price( + time_id, asset_id, "sell", float(exit_prices[time_id, asset_id]) + ) + exit_value = pos["shares"] * exit_price * (1 - sell_cost_pct) + cash += exit_value + pnl_amount = exit_value - pos["entry_value"] + pnl_pct = pnl_amount / pos["entry_value"] if pos["entry_value"] > 0 else 0.0 + sold_today.add(asset_id) + trades.append(TradeRecord( + symbol=matrix.symbols[asset_id], + name=matrix.names[asset_id], + entry_date=pos["entry_date"], + exit_date=matrix.timestamp_labels[time_id][:10], + entry_price=round(float(pos["entry_price"]), 4), + exit_price=round(exit_price, 4), + pnl_pct=round(float(pnl_pct), 6), + duration=int(pos["hold_days"]), + exit_reason=reason, + shares=round(float(pos["shares"]), 4), + lots=round(float(pos["lots"]), 2), + position_pct=round(float(pos["position_pct"]), 6), + entry_value=round(float(pos["entry_value"]), 2), + exit_value=round(float(exit_value), 2), + pnl_amount=round(float(pnl_amount), 2), + entry_score=round(float(pos["entry_score"]), 2), + entry_signal_date=pos["entry_signal_date"], + exit_signal_date=signal_date, + blocked_exit_days=int(pos["blocked_exit_days"]), + entry_signal_id=pos["entry_signal_id"], + exit_signal_id=_signal_id( + int(matrix.exit_signal_code[time_id, asset_id]), matrix.exit_signal_ids + ) if reason == "signal" else None, + )) + + def _try_sell( + time_id: int, + asset_id: int, + reason: str, + signal_date: str, + sold_today: set[int], + override: float | None = None, + ) -> bool: + ok, blocked = _can_sell(time_id, asset_id, override) + if not ok: + _mark_pending(asset_id, reason, signal_date) + _count(blocked) + return False + _sell(time_id, asset_id, reason, signal_date, sold_today, override) + return True + + for time_id, date_label in enumerate(matrix.timestamp_labels): + date_text = date_label[:10] + if time_id % 20 == 0: + if cancel_event is not None and cancel_event.is_set(): + logger.info("回测被用户取消 (第 %d/%d 天)", time_id, time_count) + break + if progress_cb is not None: + try: + progress_cb({ + "day": time_id + 1, + "total": time_count, + "date": date_text, + "equity": round(cash + _market_value(), 2), + }) + except Exception: + pass + + sold_today: set[int] = set() + for pos in positions.values(): + pos["hold_days"] += 1 + + for asset_id in list(positions): + pos = positions.get(asset_id) + if pos is None or pos.get("pending_exit_reason") or pos["entry_date"] == date_text: + continue + if not matrix.tradable[time_id, asset_id] or pos["entry_price"] <= 0: + continue + open_price = float(matrix.open[time_id, asset_id]) + low_price = float(matrix.low[time_id, asset_id]) + high_price = float(matrix.high[time_id, asset_id]) + entry_price = float(pos["entry_price"]) + peak_price = float(pos["max_high"]) + risk_lines: list[tuple[float, str]] = [] + if config.stop_loss_pct is not None: + risk_lines.append((entry_price * (1 - abs(config.stop_loss_pct)), "stop_loss")) + if config.trailing_stop_pct is not None: + risk_lines.append((peak_price * (1 - abs(config.trailing_stop_pct)), "trailing_stop")) + activate = config.trailing_take_profit_activate_pct + drawdown = config.trailing_take_profit_drawdown_pct + if activate is not None and drawdown is not None and peak_price > entry_price: + if peak_price / entry_price - 1 >= abs(float(activate)): + risk_lines.append((peak_price * (1 - abs(float(drawdown))), "trailing_take_profit")) + valid_lines = [(line, reason) for line, reason in risk_lines if _valid_price(line)] + if valid_lines: + stop_price, reason = max(valid_lines, key=lambda item: item[0]) + override = None + if _valid_price(open_price) and open_price <= stop_price: + override = open_price + elif _valid_price(low_price) and low_price <= stop_price: + override = stop_price + if override is not None: + _try_sell(time_id, asset_id, reason, date_text, sold_today, override) + continue + if config.take_profit_pct is not None: + take_profit = entry_price * (1 + abs(float(config.take_profit_pct))) + if _valid_price(open_price) and open_price >= take_profit: + _try_sell(time_id, asset_id, "take_profit", date_text, sold_today, open_price) + elif _valid_price(high_price) and high_price >= take_profit: + _try_sell(time_id, asset_id, "take_profit", date_text, sold_today, take_profit) + + for asset_id in list(positions): + pos = positions.get(asset_id) + if pos is None: + continue + reason = "" + signal_date = date_text + if pos.get("pending_exit_reason"): + reason = str(pos["pending_exit_reason"]) + signal_date = str(pos.get("pending_exit_signal_date") or date_text) + elif matrix.exit[time_id, asset_id]: + reason = "signal" + signal_date = _signal_date(int(matrix.exit_signal_time[time_id, asset_id]), date_text) + elif config.max_hold_days is not None and pos["hold_days"] >= config.max_hold_days: + reason = "max_hold" + elif time_id == time_count - 1: + reason = "end" + if reason: + _try_sell(time_id, asset_id, reason, signal_date, sold_today) + + if time_id < time_count - 1 and max_positions > 0: + candidates: list[tuple[int, float]] = [] + for asset_id in np.flatnonzero(matrix.entry[time_id]): + asset = int(asset_id) + if asset in positions: + continue + if asset in sold_today: + _count("buy_same_day_reentry") + continue + ok, blocked = _can_buy(time_id, asset) + if not ok: + _count(blocked) + continue + score = _matrix_entry_score(matrix, time_id, asset) + if config.score_min is not None and score < config.score_min: + _count("buy_score_filter") + continue + if config.score_max is not None and score > config.score_max: + _count("buy_score_filter") + continue + candidates.append((asset, score)) + candidates.sort(key=lambda item: item[1], reverse=True) + slots = max_positions - len(positions) + if slots <= 0: + execution_stats["buy_no_slot"] += len(candidates) + elif candidates: + selected = candidates[:slots] + market_value_before = _market_value() + equity_before = cash + market_value_before + target_value = equity_before * max_exposure_pct / max_positions + exposure_capacity = equity_before * max_exposure_pct - market_value_before + if equity_before <= 0 or exposure_capacity <= 0 or max_exposure_pct <= 0: + execution_stats["buy_exposure"] += len(selected) + else: + weights = np.repeat(1 / len(selected), len(selected)) + if config.position_sizing == "score_weight": + raw_weights = np.array([max(item[1], 0.0) for item in selected]) + if raw_weights.sum() > 0: + weights = raw_weights / raw_weights.sum() + total_budget = min(cash, exposure_capacity, target_value * len(selected)) + for (asset_id, entry_score), weight in zip(selected, weights): + if len(positions) >= max_positions: + _count("buy_no_slot") + break + market_value = _market_value() + equity = cash + market_value + capacity = equity * max_exposure_pct - market_value + allocation = min(total_budget * float(weight), target_value, cash, capacity) + if allocation <= 0: + _count("buy_exposure") + continue + entry_price = _refill_price( + time_id, asset_id, "buy", float(entry_prices[time_id, asset_id]) + ) + shares = np.floor(allocation / (entry_price * (1 + buy_cost_pct)) / 100) * 100 + entry_value = shares * entry_price * (1 + buy_cost_pct) + if shares <= 0: + _count("buy_lot_size") + continue + if entry_value > cash + 1e-6: + _count("buy_cash") + continue + if entry_value > capacity + 1e-6: + _count("buy_exposure") + continue + cash -= entry_value + positions[asset_id] = { + "entry_date": date_text, + "entry_signal_date": _signal_date( + int(matrix.entry_signal_time[time_id, asset_id]), date_text + ), + "entry_signal_id": _signal_id( + int(matrix.entry_signal_code[time_id, asset_id]), matrix.entry_signal_ids + ), + "entry_price": entry_price, + "entry_value": entry_value, + "shares": shares, + "lots": shares / 100, + "position_pct": entry_value / equity_before if equity_before > 0 else 0.0, + "entry_score": entry_score, + "max_high": entry_price, + "hold_days": 0, + "pending_exit_reason": None, + "pending_exit_signal_date": None, + "blocked_exit_days": 0, + } + + for asset_id, pos in positions.items(): + high_price = float(matrix.high[time_id, asset_id]) + if _valid_price(high_price): + pos["max_high"] = max(float(pos["max_high"]), high_price) + valid_closes = np.isfinite(matrix.close[time_id]) & (matrix.close[time_id] > 0) + last_close[valid_closes] = matrix.close[time_id, valid_closes] + + market_value = _market_value() + equity = cash + market_value + peak = max(peak, equity) + drawdown = (equity - peak) / peak if peak > 0 else 0.0 + exposure = market_value / equity if equity > 0 else 0.0 + equity_value = round(float(equity), 2) + exposure_value = round(float(exposure), 4) + equity_values.append(equity_value) + exposure_values.append(exposure_value) + if options.include_curves: + equity_curve.append({ + "date": date_text, + "value": equity_value, + "cash": round(float(cash), 2), + "positions": len(positions), + "exposure": exposure_value, + }) + drawdown_curve.append({ + "date": date_text, + "value": round(float(drawdown), 4), + }) + + statistics_started = time.perf_counter() + stats = self._calc_portfolio_stats_from_values( + equity_values, + exposure_values, + trades, + config.initial_capital, + include_monte_carlo=options.include_monte_carlo, + ) + stats["statistics_ms"] = round( + (time.perf_counter() - statistics_started) * 1000, + 1, + ) + stats["execution"] = execution_stats + stats["pending_exit_positions"] = sum(1 for pos in positions.values() if pos.get("pending_exit_reason")) + stats["market_matrix_shape"] = [time_count, asset_count] + stats["market_matrix_bytes"] = matrix.nbytes + return SimResult( + equity_curve=equity_curve if options.include_curves else [], + drawdown_curve=drawdown_curve if options.include_curves else [], + trades=trades if options.include_trades else [], + per_symbol_stats=( + self._calc_per_symbol(trades) + if options.include_per_symbol_stats + else [] + ), + stats=stats, + ) + + def simulate_portfolio_legacy( + self, + panel: pl.DataFrame, + entries: pl.Series | None, + exits: pl.Series | None, + config: MatcherConfig, + progress_cb: "Callable[[dict], None] | None" = None, + cancel_event: "threading.Event | None" = None, + entry_signal_ids: list[str] | None = None, + exit_signal_ids: list[str] | None = None, ) -> SimResult: """账户级组合回测：日线信号 → 成交约束 → 仓位/现金撮合。""" if panel.is_empty(): @@ -1673,6 +2617,8 @@ class BacktestEngine: initial_capital: float, start: date, end: date, + *, + include_monte_carlo: bool = True, ) -> dict: if not trades: return {"total_return": 0, "n_trades": 0} @@ -1727,7 +2673,7 @@ class BacktestEngine: calmar = annual_return / abs(max_dd) if abs(max_dd) > 0.001 else 0.0 durations = np.array([t.duration for t in trades], dtype=float) - return { + stats = { "total_return": round(float(total_return), 4), "annual_return": round(float(annual_return), 4), "max_drawdown": round(float(max_dd), 4), @@ -1741,8 +2687,10 @@ class BacktestEngine: "avg_win": round(avg_win, 4), "avg_loss": round(avg_loss, 4), **BacktestEngine._per_trade_block(pnls, durations), - **BacktestEngine._mc_drawdown_percentiles(pnls), } + if include_monte_carlo: + stats.update(BacktestEngine._mc_drawdown_percentiles(pnls)) + return stats @staticmethod def _calc_per_symbol(trades: list[TradeRecord]) -> list[dict]: @@ -1780,8 +2728,11 @@ class BacktestEngine: trades: list[TradeRecord], n_candidates: int, execution_stats: dict[str, int], + *, + options: SimulationOptions | None = None, ) -> SimResult: """全量独立候选统计：按每个候选样本的实际执行收益聚合。""" + options = options or SimulationOptions() if not trades: return SimResult( equity_curve=[], @@ -1812,6 +2763,7 @@ class BacktestEngine: equity_curve: list[dict] = [] drawdown_curve: list[dict] = [] + equity_values: list[float] = [] equity = 1.0 peak = 1.0 daily_avg: list[float] = [] @@ -1822,14 +2774,21 @@ class BacktestEngine: equity *= (1 + avg_ret) peak = max(peak, equity) dd = (equity - peak) / peak if peak > 0 else 0.0 - equity_curve.append({ - "date": d_str, - "value": round(float(equity), 4), - "positions": len(values), - }) - drawdown_curve.append({"date": d_str, "value": round(float(dd), 4)}) + equity_value = round(float(equity), 4) + equity_values.append(equity_value) + if options.include_curves: + equity_curve.append({ + "date": d_str, + "value": equity_value, + "positions": len(values), + }) + drawdown_curve.append({ + "date": d_str, + "value": round(float(dd), 4), + }) - values = np.array([r["value"] for r in equity_curve], dtype=float) + statistics_started = time.perf_counter() + values = np.array(equity_values, dtype=float) total_return = float(values[-1] - 1.0) if len(values) else 0.0 peaks = np.maximum.accumulate(values) if len(values) else np.array([]) drawdowns = values / peaks - 1 if len(values) else np.array([]) @@ -1838,18 +2797,6 @@ class BacktestEngine: sharpe = float(np.mean(daily) / np.std(daily) * np.sqrt(252)) if len(daily) > 1 and np.std(daily) > 0 else 0.0 sortino = BacktestEngine._sortino_ratio(daily) - lo, hi, nbins = -0.20, 0.20, 20 - clipped = np.clip(pnls, lo, hi) - counts, edges = np.histogram(clipped, bins=nbins, range=(lo, hi)) - dist = [ - { - "range": f"{(edges[i]*100):+.0f}~{(edges[i+1]*100):+.0f}%", - "count": int(counts[i]), - "ratio": round(float(counts[i] / pnls.size), 4) if pnls.size else 0.0, - } - for i in range(nbins) - ] - stats = { "mode": "full", "full_kind": "candidate_execution", @@ -1868,16 +2815,36 @@ class BacktestEngine: "max_drawdown": round(float(max_drawdown), 4), "sharpe": round(float(sharpe), 2), "sortino": round(float(sortino), 2) if sortino is not None else None, - "return_distribution": dist, "execution": execution_stats, - **BacktestEngine._mc_drawdown_percentiles(pnls), } + if options.include_return_distribution: + lo, hi, nbins = -0.20, 0.20, 20 + clipped = np.clip(pnls, lo, hi) + counts, edges = np.histogram(clipped, bins=nbins, range=(lo, hi)) + stats["return_distribution"] = [ + { + "range": f"{(edges[i]*100):+.0f}~{(edges[i+1]*100):+.0f}%", + "count": int(counts[i]), + "ratio": round(float(counts[i] / pnls.size), 4) if pnls.size else 0.0, + } + for i in range(nbins) + ] + if options.include_monte_carlo: + stats.update(BacktestEngine._mc_drawdown_percentiles(pnls)) + stats["statistics_ms"] = round( + (time.perf_counter() - statistics_started) * 1000, + 1, + ) return SimResult( - equity_curve=equity_curve, - drawdown_curve=drawdown_curve, - trades=trades, - per_symbol_stats=BacktestEngine._calc_per_symbol(trades), + equity_curve=equity_curve if options.include_curves else [], + drawdown_curve=drawdown_curve if options.include_curves else [], + trades=trades if options.include_trades else [], + per_symbol_stats=( + BacktestEngine._calc_per_symbol(trades) + if options.include_per_symbol_stats + else [] + ), stats=stats, ) @@ -1886,14 +2853,35 @@ class BacktestEngine: equity_curve: list[dict], trades: list[TradeRecord], initial_capital: float, + *, + include_monte_carlo: bool = True, ) -> dict: - if not equity_curve: + equity_values = [float(row["value"]) for row in equity_curve] + exposure_values = [float(row.get("exposure", 0.0)) for row in equity_curve] + return BacktestEngine._calc_portfolio_stats_from_values( + equity_values, + exposure_values, + trades, + initial_capital, + include_monte_carlo=include_monte_carlo, + ) + + @staticmethod + def _calc_portfolio_stats_from_values( + equity_values: list[float], + exposure_values: list[float], + trades: list[TradeRecord], + initial_capital: float, + *, + include_monte_carlo: bool = True, + ) -> dict: + if not equity_values: return {"total_return": 0, "n_trades": 0} - final_equity = float(equity_curve[-1]["value"]) + final_equity = float(equity_values[-1]) total_return = final_equity / initial_capital - 1 if initial_capital > 0 else 0.0 - values = np.array([float(r["value"]) for r in equity_curve], dtype=float) + values = np.array(equity_values, dtype=float) daily = values[1:] / values[:-1] - 1 if len(values) > 1 else np.array([]) - annual_return = (1 + total_return) ** (252 / max(len(equity_curve), 1)) - 1 if total_return > -1 else total_return + annual_return = (1 + total_return) ** (252 / max(len(equity_values), 1)) - 1 if total_return > -1 else total_return peaks = np.maximum.accumulate(values) drawdowns = values / peaks - 1 max_drawdown = float(drawdowns.min()) if len(drawdowns) else 0.0 @@ -1901,12 +2889,12 @@ class BacktestEngine: sortino = BacktestEngine._sortino_ratio(daily) pnls = np.array([t.pnl_pct for t in trades], dtype=float) if trades else np.array([]) durations = np.array([t.duration for t in trades], dtype=float) if trades else np.array([]) - exposures = np.array([float(r.get("exposure", 0.0)) for r in equity_curve], dtype=float) + exposures = np.array(exposure_values, dtype=float) wins = pnls[pnls > 0] losses = pnls[pnls <= 0] avg_win = float(np.mean(wins)) if len(wins) else 0.0 avg_loss = abs(float(np.mean(losses))) if len(losses) else 0.0 - return { + stats = { "total_return": round(float(total_return), 4), "annual_return": round(float(annual_return), 4), "max_drawdown": round(float(max_drawdown), 4), @@ -1920,12 +2908,14 @@ class BacktestEngine: "avg_win": round(avg_win, 4), "avg_loss": round(avg_loss, 4), **BacktestEngine._per_trade_block(pnls, durations), - **BacktestEngine._mc_drawdown_percentiles(pnls), "final_equity": round(final_equity, 2), "initial_capital": round(float(initial_capital), 2), "avg_exposure": round(float(np.mean(exposures)), 4) if len(exposures) else 0.0, "max_exposure": round(float(np.max(exposures)), 4) if len(exposures) else 0.0, } + if include_monte_carlo: + stats.update(BacktestEngine._mc_drawdown_percentiles(pnls)) + return stats @staticmethod def _date_str(value) -> str: diff --git a/backend/app/backtest/matrix.py b/backend/app/backtest/matrix.py new file mode 100644 index 0000000..2eb91f1 --- /dev/null +++ b/backend/app/backtest/matrix.py @@ -0,0 +1,3809 @@ +"""Matrix structures, builders, NumPy features, and matrix-strategy contract.""" +from __future__ import annotations + +import hashlib +import json +import logging +import os +import shutil +import threading +import time +import uuid +import weakref +from collections import OrderedDict +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager, nullcontext +from contextvars import ContextVar +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path +from types import MappingProxyType +from typing import Any, Protocol, runtime_checkable + +import numpy as np +import polars as pl +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.dataset as pads +from numba import njit, prange + +_MATRIX_CACHE_VERSION = 1 +_DIRECT_MATRIX_LOADER_VERSION = 3 +_MATRIX_AXIS_INDEX_VERSION = 1 +_ARROW_BATCH_SIZE = 131_072 +_SCORE_ASSET_CHUNK_SIZE = 256 +_ROLLING_MATERIALIZED_WINDOW_BUDGET_BYTES = 32 * 1024 * 1024 +_MATRIX_DISK_CACHE_DEFAULT_MAX_BYTES = 512 * 1024 * 1024 + +logger = logging.getLogger(__name__) +_MATRIX_DISK_CACHE_LOCK = threading.RLock() +_MATRIX_DISK_CACHE_LEASES: dict[str, int] = {} +_MATRIX_DISK_CACHE_PENDING_DELETE: set[str] = set() +_ACTIVE_MATRIX_CACHE: ContextVar[MatrixComputeCache | None] = ContextVar( + "active_matrix_compute_cache", + default=None, +) +_ACTIVE_VALID_BAR_INDEX: ContextVar[Any] = ContextVar( + "active_valid_bar_index", + default=None, +) + + +def _freeze_cache_value(value: Any) -> Any: + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, dict): + return tuple( + (str(key), _freeze_cache_value(item)) + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) + ) + if isinstance(value, (list, tuple)): + return tuple(_freeze_cache_value(item) for item in value) + if isinstance(value, set): + return tuple(sorted((_freeze_cache_value(item) for item in value), key=repr)) + if isinstance(value, float): + if np.isnan(value): + return ("float", "nan") + if np.isposinf(value): + return ("float", "inf") + if np.isneginf(value): + return ("float", "-inf") + return ("float", value) + if isinstance(value, (str, int, bool, bytes, type(None))): + return value + return (type(value).__qualname__, repr(value)) + + +class MatrixComputeCache: + """Job-scoped byte-bounded cache for deterministic matrix operations.""" + + def __init__( + self, + *, + max_bytes: int = 512 * 1024 * 1024, + max_item_bytes: int = 256 * 1024 * 1024, + ) -> None: + if max_bytes <= 0: + raise ValueError("matrix cache max_bytes must be positive") + if max_item_bytes <= 0: + raise ValueError("matrix cache max_item_bytes must be positive") + self.max_bytes = int(max_bytes) + self.max_item_bytes = min(int(max_item_bytes), self.max_bytes) + self._entries: OrderedDict[tuple, np.ndarray] = OrderedDict() + self._lineage: dict[int, tuple[weakref.ReferenceType[np.ndarray], tuple]] = {} + self._market_tokens: dict[int, tuple[MarketDataMatrix, tuple]] = {} + self._market_counter = 0 + self._lock = threading.RLock() + self._closed = False + self._current_bytes = 0 + self._peak_bytes = 0 + self._calls = 0 + self._hits = 0 + self._misses = 0 + self._evictions = 0 + self._skipped = 0 + self._fingerprint_bytes = 0 + self._fingerprint_ms = 0.0 + self._operations: dict[str, dict[str, int | float]] = {} + + @contextmanager + def activate(self, market: MarketDataMatrix) -> Iterator[MatrixComputeCache]: + self.register_market(market) + token = _ACTIVE_MATRIX_CACHE.set(self) + try: + yield self + finally: + _ACTIVE_MATRIX_CACHE.reset(token) + + def register_market(self, market: MarketDataMatrix) -> tuple: + self._ensure_open() + market_id = id(market) + with self._lock: + existing = self._market_tokens.get(market_id) + if existing is not None and existing[0] is market: + return existing[1] + self._market_counter += 1 + token = ("market", self._market_counter) + self._market_tokens[market_id] = (market, token) + arrays = { + "timestamps": market.timestamps, + "session_ids": market.session_ids, + "valid_bar_offsets": market.valid_bars.offsets, + "valid_bar_rows": market.valid_bars.rows, + "open": market.open, + "high": market.high, + "low": market.low, + "close": market.close, + "volume": market.volume, + "tradable": market.tradable, + "limit_up_locked": market.limit_up_locked, + "limit_down_locked": market.limit_down_locked, + **{f"field:{name}": values for name, values in market.fields.items()}, + } + for name, values in arrays.items(): + self._register_lineage(values, (token, name)) + return token + + @contextmanager + def suspend(self) -> Iterator[None]: + """Temporarily bypass this cache without changing its retained entries.""" + token = _ACTIVE_MATRIX_CACHE.set(None) + try: + yield + finally: + _ACTIVE_MATRIX_CACHE.reset(token) + + def market_token(self, market: MarketDataMatrix) -> tuple: + return self.register_market(market) + + def get_or_compute( + self, + operation: str, + inputs: tuple[np.ndarray, ...], + params: Any, + compute: Callable[[], np.ndarray], + *, + key_parts: Any = (), + ) -> np.ndarray: + self._ensure_open() + input_tokens = tuple(self._array_token(values) for values in inputs) + key = ( + _MATRIX_CACHE_VERSION, + str(operation), + input_tokens, + _freeze_cache_value(params), + _freeze_cache_value(key_parts), + ) + with self._lock: + self._calls += 1 + op_stats = self._operations.setdefault( + str(operation), + { + "calls": 0, + "hits": 0, + "misses": 0, + "compute_ms": 0.0, + "computed_bytes": 0, + }, + ) + op_stats["calls"] += 1 + cached = self._entries.get(key) + if cached is not None: + self._hits += 1 + op_stats["hits"] += 1 + self._entries.move_to_end(key) + return cached + self._misses += 1 + op_stats["misses"] += 1 + + compute_started = time.perf_counter() + result = np.asarray(compute()) + compute_ms = (time.perf_counter() - compute_started) * 1000.0 + with self._lock: + op_stats = self._operations[str(operation)] + op_stats["compute_ms"] = float(op_stats["compute_ms"]) + compute_ms + op_stats["computed_bytes"] = int(op_stats["computed_bytes"]) + int(result.nbytes) + if result.ndim == 0: + raise ValueError(f"cached matrix operation {operation} returned a scalar") + if result.flags.writeable: + result.flags.writeable = False + if result.nbytes > self.max_item_bytes or result.nbytes > self.max_bytes: + with self._lock: + self._skipped += 1 + return result + + with self._lock: + existing = self._entries.get(key) + if existing is not None: + self._entries.move_to_end(key) + return existing + self._evict_for(result.nbytes) + self._entries[key] = result + self._current_bytes += int(result.nbytes) + self._peak_bytes = max(self._peak_bytes, self._current_bytes) + derived = hashlib.blake2b(repr(key).encode("utf-8"), digest_size=16).digest() + self._register_lineage(result, ("derived", derived)) + return result + + def snapshot(self) -> dict[str, Any]: + with self._lock: + hit_rate = self._hits / self._calls if self._calls else 0.0 + return { + "enabled": True, + "max_bytes": self.max_bytes, + "max_item_bytes": self.max_item_bytes, + "current_bytes": self._current_bytes, + "peak_bytes": self._peak_bytes, + "entries": len(self._entries), + "calls": self._calls, + "hits": self._hits, + "misses": self._misses, + "evictions": self._evictions, + "skipped": self._skipped, + "hit_rate": round(float(hit_rate), 6), + "fingerprint_bytes": self._fingerprint_bytes, + "fingerprint_ms": round(self._fingerprint_ms, 3), + "operations": { + name: { + **values, + "compute_ms": round(float(values["compute_ms"]), 3), + } + for name, values in sorted(self._operations.items()) + }, + } + + @property + def current_bytes(self) -> int: + with self._lock: + return int(self._current_bytes) + + def has_cached_operation(self, operation: str) -> bool: + with self._lock: + return any(key[1] == operation for key in self._entries) + + def close(self) -> None: + with self._lock: + self._entries.clear() + self._lineage.clear() + self._market_tokens.clear() + self._current_bytes = 0 + self._closed = True + + def _ensure_open(self) -> None: + if self._closed: + raise RuntimeError("matrix compute cache is closed") + + def _array_token(self, values: np.ndarray) -> tuple: + array = np.asarray(values) + with self._lock: + existing = self._lineage.get(id(array)) + if existing is not None and existing[0]() is array: + return existing[1] + + contiguous = array if array.flags.c_contiguous else np.ascontiguousarray(array) + started = time.perf_counter() + digest = hashlib.blake2b(contiguous.view(np.uint8), digest_size=16).digest() + elapsed_ms = (time.perf_counter() - started) * 1000.0 + with self._lock: + self._fingerprint_bytes += int(contiguous.nbytes) + self._fingerprint_ms += elapsed_ms + return ("content", array.dtype.str, tuple(array.shape), digest) + + def _register_lineage(self, array: np.ndarray, token: tuple) -> None: + array_id = id(array) + + def _remove(reference: weakref.ReferenceType[np.ndarray]) -> None: + with self._lock: + current = self._lineage.get(array_id) + if current is not None and current[0] is reference: + self._lineage.pop(array_id, None) + + reference = weakref.ref(array, _remove) + self._lineage[array_id] = (reference, token) + + def _evict_for(self, incoming_bytes: int) -> None: + while self._entries and self._current_bytes + incoming_bytes > self.max_bytes: + _, evicted = self._entries.popitem(last=False) + self._current_bytes -= int(evicted.nbytes) + self._evictions += 1 + + +def active_matrix_compute_cache() -> MatrixComputeCache | None: + return _ACTIVE_MATRIX_CACHE.get() + + +@contextmanager +def _activate_valid_bar_index(index: ValidBarIndex) -> Iterator[None]: + token = _ACTIVE_VALID_BAR_INDEX.set(index) + try: + yield + finally: + _ACTIVE_VALID_BAR_INDEX.reset(token) + + +def _cached_matrix_operation( + operation: str, + inputs: tuple[np.ndarray, ...], + params: Any, + compute: Callable[[], np.ndarray], + *, + key_parts: Any = (), +) -> np.ndarray: + cache = active_matrix_compute_cache() + if cache is None: + return compute() + return cache.get_or_compute( + operation, + inputs, + params, + compute, + key_parts=key_parts, + ) + + +@dataclass(frozen=True) +class MatrixCacheProfile: + """Shared disk-cache boundary for one matrix-native asset universe.""" + + field_columns: frozenset[str] + warmup_bars: int + forward_bars: int + max_disk_bytes: int = _MATRIX_DISK_CACHE_DEFAULT_MAX_BYTES + generation: str = "default" + + +@dataclass(frozen=True) +class ValidBarIndex: + """Asset-major CSR index of effective market bars.""" + + shape: tuple[int, int] + offsets: np.ndarray + rows: np.ndarray + + @property + def nbytes(self) -> int: + return int(self.offsets.nbytes + self.rows.nbytes) + + +def _build_valid_bar_index(valid_mask: np.ndarray) -> ValidBarIndex: + valid = np.asarray(valid_mask, dtype=bool) + if valid.ndim != 2: + raise ValueError("valid bar index requires a 2D mask") + counts = np.count_nonzero(valid, axis=0).astype(np.int64, copy=False) + offsets = np.empty(valid.shape[1] + 1, dtype=np.int64) + offsets[0] = 0 + np.cumsum(counts, out=offsets[1:]) + rows = np.empty(int(offsets[-1]), dtype=np.int32) + for asset_id in range(valid.shape[1]): + start = int(offsets[asset_id]) + stop = int(offsets[asset_id + 1]) + rows[start:stop] = np.flatnonzero(valid[:, asset_id]).astype( + np.int32, + copy=False, + ) + offsets.flags.writeable = False + rows.flags.writeable = False + return ValidBarIndex(shape=valid.shape, offsets=offsets, rows=rows) + + +@dataclass(frozen=True) +class MarketDataMatrix: + """Compact base market data shared by matrix-native strategies and matchers.""" + + timestamps: np.ndarray + timestamp_labels: tuple[str, ...] + session_ids: np.ndarray + symbols: tuple[str, ...] + names: tuple[str, ...] + + open: np.ndarray + high: np.ndarray + low: np.ndarray + close: np.ndarray + volume: np.ndarray + tradable: np.ndarray + limit_up_locked: np.ndarray + limit_down_locked: np.ndarray + fields: Mapping[str, np.ndarray] + cache_status: str = "memory" + cache_path: str | None = None + cache_lease: Any | None = field(default=None, compare=False, repr=False) + vector_fields: frozenset[str] = field(default_factory=frozenset) + cache_timing_ms: Mapping[str, float] = field(default_factory=dict) + _valid_bars: ValidBarIndex | None = field( + default=None, + compare=False, + repr=False, + ) + + @property + def shape(self) -> tuple[int, int]: + return self.open.shape + + @property + def nbytes(self) -> int: + arrays = [ + self.timestamps, + self.session_ids, + self.open, + self.high, + self.low, + self.close, + self.volume, + self.tradable, + self.limit_up_locked, + self.limit_down_locked, + *self.fields.values(), + ] + index_bytes = self._valid_bars.nbytes if self._valid_bars is not None else 0 + return int(sum(array.nbytes for array in arrays) + index_bytes) + + @property + def valid_bars(self) -> ValidBarIndex: + index = self._valid_bars + if index is None: + index = _build_valid_bar_index(np.isfinite(self.close)) + object.__setattr__(self, "_valid_bars", index) + return index + + def field(self, name: str) -> np.ndarray: + if name == "open": + return self.open + if name == "high": + return self.high + if name == "low": + return self.low + if name == "close": + return self.close + if name == "volume": + return self.volume + try: + return self.fields[name] + except KeyError as exc: + raise ValueError(f"MarketDataMatrix missing field: {name}") from exc + + +@dataclass(frozen=True) +class SignalMatrix: + """Strategy output before execution delays are applied.""" + + entry: np.ndarray + exit: np.ndarray + score: np.ndarray + entry_signal_code: np.ndarray + exit_signal_code: np.ndarray + entry_signal_ids: tuple[str, ...] = () + exit_signal_ids: tuple[str, ...] = () + + @property + def shape(self) -> tuple[int, int]: + return self.entry.shape + + @property + def nbytes(self) -> int: + return int(sum( + value.nbytes + for value in self.__dict__.values() + if isinstance(value, np.ndarray) + )) + + +@dataclass(frozen=True) +class MarketMatrix: + """Execution matrix consumed by the Python matcher and future Numba kernel.""" + + timestamps: np.ndarray + timestamp_labels: tuple[str, ...] + session_ids: np.ndarray + symbols: tuple[str, ...] + names: tuple[str, ...] + + open: np.ndarray + high: np.ndarray + low: np.ndarray + close: np.ndarray + volume: np.ndarray + score: np.ndarray + entry: np.ndarray + exit: np.ndarray + tradable: np.ndarray + limit_up_locked: np.ndarray + limit_down_locked: np.ndarray + reference_price: np.ndarray + + entry_signal_time: np.ndarray + exit_signal_time: np.ndarray + entry_signal_code: np.ndarray + exit_signal_code: np.ndarray + entry_signal_ids: tuple[str, ...] + exit_signal_ids: tuple[str, ...] + + @property + def shape(self) -> tuple[int, int]: + return self.open.shape + + @property + def nbytes(self) -> int: + return int(sum( + value.nbytes + for value in self.__dict__.values() + if isinstance(value, np.ndarray) + )) + + +def build_market_data_matrix( + panel: pl.DataFrame, + *, + field_columns: set[str] | frozenset[str] | None = None, +) -> MarketDataMatrix: + """Encode a long base panel into immutable ``time x asset`` arrays.""" + if panel.is_empty(): + raise ValueError("cannot build MarketDataMatrix from an empty panel") + + timestamp_col, unique_timestamps, symbol_values, time_id, asset_id = _encode_axes(panel) + shape = (len(unique_timestamps), len(symbol_values)) + + def float_matrix( + column: str, + default: float = np.nan, + null_fill: float | None = None, + ) -> np.ndarray: + return _float_matrix(panel, column, shape, time_id, asset_id, default, null_fill) + + open_ = float_matrix("open") + high = float_matrix("high") + low = float_matrix("low") + close = float_matrix("close") + volume = float_matrix("volume", 0.0 if "volume" in panel.columns else 1.0, 0.0) + limit_up_locked = _bool_matrix(panel, "signal_limit_up", shape, time_id, asset_id) + limit_down_locked = _bool_matrix(panel, "signal_limit_down", shape, time_id, asset_id) + tradable = _tradable_matrix(open_, high, low, close, volume) + + core_columns = { + timestamp_col, + "symbol", + "name", + "open", + "high", + "low", + "close", + "volume", + "signal_limit_up", + "signal_limit_down", + } + wanted_fields = set(field_columns or ()) - core_columns + fields: dict[str, np.ndarray] = {} + for column in sorted(wanted_fields): + if column in panel.columns and panel[column].dtype.is_numeric(): + fields[column] = float_matrix(column) + elif column == "raw_close": + # A live quote is already an unadjusted price when no separate raw + # field is supplied. Keep this explicit compatibility contract for + # strategies that estimate market value from historical raw prices. + fields[column] = np.array(close, copy=True) + + names = [""] * len(symbol_values) + if "name" in panel.columns: + row_names = panel["name"].fill_null("").cast(pl.Utf8).to_numpy() + for row, aid in enumerate(asset_id): + if not names[int(aid)] and row_names[row]: + names[int(aid)] = str(row_names[row]) + + timestamp_labels = tuple(str(value)[:19] for value in unique_timestamps.to_numpy()) + timestamps = _timestamp_int64(unique_timestamps) + session_dates = unique_timestamps.cast(pl.Date).to_numpy() + session_values = np.unique(session_dates) + session_ids = np.searchsorted(session_values, session_dates).astype(np.int32) + + arrays = ( + timestamps, + session_ids, + open_, + high, + low, + close, + volume, + tradable, + limit_up_locked, + limit_down_locked, + *fields.values(), + ) + _make_read_only(*arrays) + + return MarketDataMatrix( + timestamps=timestamps, + timestamp_labels=timestamp_labels, + session_ids=session_ids, + symbols=tuple(str(value) for value in symbol_values), + names=tuple(names), + open=open_, + high=high, + low=low, + close=close, + volume=volume, + tradable=tradable, + limit_up_locked=limit_up_locked, + limit_down_locked=limit_down_locked, + fields=MappingProxyType(fields), + ) + + +def load_market_data_matrix_from_parquet( + parquet_root: Path, + start: date, + end: date, + *, + field_columns: set[str] | frozenset[str], + symbols: list[str] | None = None, + instruments: pl.DataFrame | None = None, + batch_size: int = _ARROW_BATCH_SIZE, + cache_root: Path | None = None, + coverage_start: date | None = None, + coverage_end: date | None = None, + cache_field_columns: set[str] | frozenset[str] | None = None, + cache_max_bytes: int = _MATRIX_DISK_CACHE_DEFAULT_MAX_BYTES, + profile_generation: str = "default", + source_generation: str | None = None, +) -> MarketDataMatrix: + """Load a daily market matrix, reusing a covering read-only mmap when possible.""" + if start > end: + raise ValueError("matrix parquet range start must not exceed end") + root = Path(parquet_root) + if not root.exists(): + raise ValueError(f"matrix parquet root does not exist: {root}") + available_start, available_end = _partition_date_bounds(root) + if available_start is None or available_end is None: + raise ValueError("matrix parquet root contains no dated partitions") + effective_start = max(start, available_start) + effective_end = min(end, available_end) + if effective_start > effective_end: + raise ValueError("matrix parquet range contains no market data") + requested_fields = _normalize_matrix_cache_fields(field_columns) + requested_coverage_start = coverage_start or start + requested_coverage_end = coverage_end or end + if requested_coverage_start > start or requested_coverage_end < end: + raise ValueError("matrix cache coverage must include the requested range") + build_start = max(requested_coverage_start, available_start) + build_end = min(requested_coverage_end, available_end) + build_fields = frozenset( + requested_fields + | _normalize_matrix_cache_fields(cache_field_columns or field_columns) + ) + normalized_symbols = _normalize_symbol_request(symbols) + instrument_fingerprint = _instrument_fingerprint(instruments).hex() + + partitioning = pads.partitioning( + pa.schema([("date", pa.date32())]), + flavor="hive", + ) + dataset = pads.dataset( + str(root), + format="parquet", + partitioning=partitioning, + ) + _validate_matrix_dataset_schema(dataset) + + if cache_root is None: + return _build_market_data_matrix_from_dataset( + dataset, + root, + effective_start, + effective_end, + requested_fields, + normalized_symbols, + instruments, + batch_size=batch_size, + cache_status="disabled", + ) + + cache_dir = Path(cache_root) + cache_dir.mkdir(parents=True, exist_ok=True) + requested_partitions = ( + {} + if source_generation + else _partition_fingerprints( + root, + effective_start, + effective_end, + include_predecessor=True, + ) + ) + covering = _find_covering_matrix_cache( + cache_dir, + root, + effective_start, + effective_end, + requested_fields, + normalized_symbols, + requested_partitions, + instrument_fingerprint, + source_generation, + ) + if covering is not None: + path, status = covering + try: + market = _load_market_data_matrix_cache(path, cache_status=status) + return _slice_and_project_market_data_matrix( + market, + effective_start, + effective_end, + requested_fields, + ) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + logger.warning("invalid matrix disk cache %s: %s", path, exc) + shutil.rmtree(path, ignore_errors=True) + + build_partitions = _partition_fingerprints(root, build_start, build_end) + if not build_partitions: + raise ValueError("matrix parquet range contains no market data") + cache_path = _matrix_disk_cache_path( + cache_dir, + root, + build_start, + build_end, + build_fields, + normalized_symbols, + build_partitions, + instrument_fingerprint, + profile_generation, + source_generation, + ) + if (cache_path / "manifest.json").exists(): + market = _load_market_data_matrix_cache(cache_path, cache_status="exact") + return _slice_and_project_market_data_matrix( + market, + effective_start, + effective_end, + requested_fields, + ) + + _build_market_data_matrix_cache_from_dataset( + dataset, + root, + cache_path, + build_start, + build_end, + build_fields, + normalized_symbols, + instruments, + build_partitions, + instrument_fingerprint, + profile_generation, + source_generation, + batch_size=batch_size, + axis_cache_root=cache_dir, + ) + _prune_matrix_disk_cache( + cache_dir, + keep=cache_path, + max_bytes=int(cache_max_bytes), + current_source_generation=source_generation, + ) + market = _load_market_data_matrix_cache(cache_path, cache_status="built") + return _slice_and_project_market_data_matrix( + market, + effective_start, + effective_end, + requested_fields, + ) + + +def _validate_matrix_dataset_schema(dataset: pads.Dataset) -> None: + available = set(dataset.schema.names) + required = {"symbol", "date", "open", "high", "low", "close", "volume"} + missing = required - available + if missing: + raise ValueError(f"matrix parquet missing columns: {sorted(missing)}") + + +def _normalize_matrix_cache_fields( + field_columns: set[str] | frozenset[str], +) -> frozenset[str]: + ignored = { + "symbol", + "date", + "name", + "open", + "high", + "low", + "close", + "volume", + "signal_limit_up", + "signal_limit_down", + } + return frozenset(str(name) for name in field_columns if str(name) not in ignored) + + +def _normalize_symbol_request(symbols: list[str] | None) -> tuple[str, ...] | None: + if symbols is None: + return None + return tuple(sorted({str(symbol) for symbol in symbols})) + + +def _matrix_filter_expression( + start: date, + end: date, + symbols: tuple[str, ...] | None, +): + expression = (pads.field("date") >= pa.scalar(start)) & ( + pads.field("date") <= pa.scalar(end) + ) + if symbols is not None: + expression &= pads.field("symbol").isin(list(symbols)) + return expression + + +def _resolve_matrix_storage_fields( + dataset: pads.Dataset, + wanted_fields: frozenset[str], + instruments: pl.DataFrame | None, +) -> tuple[list[str], list[str], list[str]]: + available = set(dataset.schema.names) + parquet_fields = sorted( + name + for name in wanted_fields + if name in available and _arrow_numeric(dataset.schema.field(name).type) + ) + instrument_columns = set(instruments.columns) if instruments is not None else set() + matrix_fields = set(parquet_fields) + vector_fields = { + name + for name in ("total_shares", "float_shares") + if name in wanted_fields + and name in instrument_columns + and name not in parquet_fields + } + if "raw_close" in wanted_fields: + matrix_fields.add("raw_close") + if "turnover_rate" in wanted_fields: + matrix_fields.add("turnover_rate") + if "turnover_rate" not in parquet_fields and "float_shares" in instrument_columns: + vector_fields.add("float_shares") + resolved = matrix_fields | vector_fields + unresolved = wanted_fields - resolved + if unresolved: + raise ValueError(f"matrix parquet fields unavailable: {sorted(unresolved)}") + return parquet_fields, sorted(matrix_fields), sorted(vector_fields) + + +def _build_market_data_matrix_from_dataset( + dataset: pads.Dataset, + root: Path, + start: date, + end: date, + wanted_fields: frozenset[str], + symbols: tuple[str, ...] | None, + instruments: pl.DataFrame | None, + *, + batch_size: int, + cache_status: str, +) -> MarketDataMatrix: + filter_expr = _matrix_filter_expression(start, end, symbols) + actual_dates, actual_symbols = _collect_parquet_axes( + dataset, + filter_expr, + batch_size=batch_size, + ) + if not actual_dates or not actual_symbols: + raise ValueError("matrix parquet range contains no market data") + parquet_fields, matrix_fields, vector_fields = _resolve_matrix_storage_fields( + dataset, + wanted_fields, + instruments, + ) + shape = (len(actual_dates), len(actual_symbols)) + arrays = { + "open": np.full(shape, np.nan, dtype=np.float32), + "high": np.full(shape, np.nan, dtype=np.float32), + "low": np.full(shape, np.nan, dtype=np.float32), + "close": np.full(shape, np.nan, dtype=np.float32), + "volume": np.zeros(shape, dtype=np.float32), + } + fields = { + name: np.full(shape, np.nan, dtype=np.float32) + for name in matrix_fields + } + seen = np.zeros(shape, dtype=bool) + _scan_matrix_values( + dataset, + filter_expr, + actual_dates, + actual_symbols, + arrays, + fields, + parquet_fields, + seen, + batch_size=batch_size, + ) + names, latest_limits = _populate_matrix_derived_arrays( + actual_symbols, + arrays, + fields, + wanted_fields, + instruments, + seen, + parquet_fields=parquet_fields, + vector_fields=vector_fields, + ) + for name in vector_fields: + fields[name] = np.where(seen, fields[name], np.nan).astype(np.float32, copy=False) + tradable = _tradable_matrix( + arrays["open"], + arrays["high"], + arrays["low"], + arrays["close"], + arrays["volume"], + ) + raw_close = fields.get("raw_close", arrays["close"]) + limit_up_locked, limit_down_locked = _limit_lock_matrices( + arrays["close"], + raw_close, + seen, + actual_symbols, + names, + latest_limits, + apply_latest_limits=actual_dates[-1] == _latest_partition_date(root), + ) + timestamps, session_ids = _matrix_time_axes(actual_dates) + _make_read_only( + timestamps, + session_ids, + *arrays.values(), + tradable, + limit_up_locked, + limit_down_locked, + *fields.values(), + ) + return MarketDataMatrix( + timestamps=timestamps, + timestamp_labels=tuple(value.isoformat() for value in actual_dates), + session_ids=session_ids, + symbols=tuple(actual_symbols), + names=tuple(names), + open=arrays["open"], + high=arrays["high"], + low=arrays["low"], + close=arrays["close"], + volume=arrays["volume"], + tradable=tradable, + limit_up_locked=limit_up_locked, + limit_down_locked=limit_down_locked, + fields=MappingProxyType(fields), + cache_status=cache_status, + ) + + +def _build_market_data_matrix_cache_from_dataset( + dataset: pads.Dataset, + root: Path, + cache_path: Path, + start: date, + end: date, + wanted_fields: frozenset[str], + symbols: tuple[str, ...] | None, + instruments: pl.DataFrame | None, + source_partitions: Mapping[str, str], + instrument_fingerprint: str, + profile_generation: str, + source_generation: str | None, + *, + batch_size: int, + axis_cache_root: Path, +) -> None: + build_started = time.perf_counter() + timing_ms: dict[str, float] = {} + cache_path.parent.mkdir(parents=True, exist_ok=True) + temporary = cache_path.parent / f".{cache_path.name}.{uuid.uuid4().hex}.tmp" + temporary.mkdir() + mapped: list[np.memmap] = [] + try: + filter_expr = _matrix_filter_expression(start, end, symbols) + stage_started = time.perf_counter() + actual_dates, actual_symbols = _load_or_build_matrix_axes( + dataset, + root, + start, + end, + symbols, + source_partitions, + filter_expr, + batch_size=batch_size, + cache_root=axis_cache_root, + ) + if not actual_dates or not actual_symbols: + raise ValueError("matrix parquet range contains no market data") + timing_ms["axes"] = round((time.perf_counter() - stage_started) * 1000, 1) + stage_started = time.perf_counter() + parquet_fields, matrix_fields, vector_fields = _resolve_matrix_storage_fields( + dataset, + wanted_fields, + instruments, + ) + shape = (len(actual_dates), len(actual_symbols)) + array_specs, field_specs, total_bytes = _matrix_binary_layout( + shape, + matrix_fields, + vector_fields, + ) + data_path = temporary / "matrix.bin" + with data_path.open("wb") as stream: + stream.truncate(total_bytes) + + arrays = { + name: _open_matrix_memmap(data_path, spec, mapped) + for name, spec in array_specs.items() + } + fields = { + name: _open_matrix_memmap(data_path, spec, mapped) + for name, spec in field_specs.items() + } + timestamps, session_ids = _matrix_time_axes(actual_dates) + arrays["timestamps"][:] = timestamps + arrays["session_ids"][:] = session_ids + timing_ms["layout"] = round((time.perf_counter() - stage_started) * 1000, 1) + stage_started = time.perf_counter() + seen = np.zeros(shape, dtype=bool) + _scan_matrix_values( + dataset, + filter_expr, + actual_dates, + actual_symbols, + arrays, + fields, + parquet_fields, + seen, + batch_size=batch_size, + ) + if not seen.any(): + raise ValueError("matrix parquet range contains no requested market data") + timing_ms["scan"] = round((time.perf_counter() - stage_started) * 1000, 1) + stage_started = time.perf_counter() + _mask_unseen_staging_core(arrays, fields, parquet_fields, seen) + names, latest_limits = _populate_matrix_derived_arrays( + actual_symbols, + arrays, + fields, + wanted_fields, + instruments, + seen, + parquet_fields=parquet_fields, + vector_fields=vector_fields, + ) + _mask_unseen_staging_fields(fields, seen) + _write_tradable_matrix( + arrays["tradable"], + arrays["open"], + arrays["high"], + arrays["low"], + arrays["close"], + arrays["volume"], + ) + _limit_lock_matrices( + arrays["close"], + fields.get("raw_close", arrays["close"]), + seen, + actual_symbols, + names, + latest_limits, + out_up=arrays["limit_up_locked"], + out_down=arrays["limit_down_locked"], + apply_latest_limits=actual_dates[-1] == _latest_partition_date(root), + ) + timing_ms["derived"] = round((time.perf_counter() - stage_started) * 1000, 1) + stage_started = time.perf_counter() + for values in mapped: + values.flush() + _close_matrix_memmaps(mapped) + mapped.clear() + timing_ms["flush_close"] = round((time.perf_counter() - stage_started) * 1000, 1) + timing_ms["total_before_publish"] = round( + (time.perf_counter() - build_started) * 1000, + 1, + ) + + manifest = { + "version": _DIRECT_MATRIX_LOADER_VERSION, + "storage": "matrix.bin", + "parquet_root": str(root.resolve()), + "coverage_start": start.isoformat(), + "coverage_end": end.isoformat(), + "cache_field_columns": sorted(wanted_fields), + "symbols_request": None if symbols is None else list(symbols), + "source_partitions": dict(source_partitions), + "instrument_fingerprint": instrument_fingerprint, + "profile_generation": str(profile_generation), + "source_generation": source_generation, + "build_timing_ms": timing_ms, + "timestamp_labels": [value.isoformat() for value in actual_dates], + "symbols": list(actual_symbols), + "names": list(names), + "arrays": array_specs, + "fields": field_specs, + } + (temporary / "manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, separators=(",", ":")), + encoding="utf-8", + ) + try: + os.replace(temporary, cache_path) + except OSError: + if (cache_path / "manifest.json").exists(): + shutil.rmtree(temporary, ignore_errors=True) + else: + raise + except BaseException: + _close_matrix_memmaps(mapped) + shutil.rmtree(temporary, ignore_errors=True) + raise + + +def _matrix_binary_layout( + shape: tuple[int, int], + matrix_fields: list[str], + vector_fields: list[str], +) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]], int]: + time_count = shape[0] + arrays = [ + ("timestamps", np.dtype(np.int64), (time_count,)), + ("session_ids", np.dtype(np.int32), (time_count,)), + ("open", np.dtype(np.float32), shape), + ("high", np.dtype(np.float32), shape), + ("low", np.dtype(np.float32), shape), + ("close", np.dtype(np.float32), shape), + ("volume", np.dtype(np.float32), shape), + ("tradable", np.dtype(np.uint8), shape), + ("limit_up_locked", np.dtype(np.uint8), shape), + ("limit_down_locked", np.dtype(np.uint8), shape), + ] + offset = 0 + + def add_spec(dtype: np.dtype, value_shape: tuple[int, ...]) -> dict[str, Any]: + nonlocal offset + offset += (-offset) % 64 + spec = { + "offset": offset, + "dtype": dtype.str, + "shape": list(value_shape), + } + offset += int(np.prod(value_shape, dtype=np.int64)) * dtype.itemsize + return spec + + array_specs = {name: add_spec(dtype, value_shape) for name, dtype, value_shape in arrays} + field_specs = { + name: add_spec(np.dtype(np.float32), shape) + for name in matrix_fields + } + field_specs.update({ + name: add_spec(np.dtype(np.float32), (shape[1],)) + for name in vector_fields + }) + return array_specs, field_specs, offset + + +def _open_matrix_memmap( + path: Path, + spec: Mapping[str, Any], + mapped: list[np.memmap], +) -> np.memmap: + values = np.memmap( + path, + dtype=np.dtype(str(spec["dtype"])), + mode="r+", + offset=int(spec["offset"]), + shape=tuple(int(value) for value in spec["shape"]), + order="C", + ) + mapped.append(values) + return values + + +def _mask_unseen_staging_core( + arrays: Mapping[str, np.ndarray], + fields: Mapping[str, np.ndarray], + parquet_fields: list[str], + seen: np.ndarray, +) -> None: + rows_per_chunk = max(1, (32 * 1024 * 1024) // max(1, seen.shape[1])) + targets = [ + arrays["open"], + arrays["high"], + arrays["low"], + arrays["close"], + *(fields[name] for name in parquet_fields), + ] + for start in range(0, seen.shape[0], rows_per_chunk): + stop = min(seen.shape[0], start + rows_per_chunk) + missing = ~seen[start:stop] + for target in targets: + target[start:stop][missing] = np.nan + + +def _mask_unseen_staging_fields( + fields: Mapping[str, np.ndarray], + seen: np.ndarray, +) -> None: + rows_per_chunk = max(1, (32 * 1024 * 1024) // max(1, seen.shape[1])) + for start in range(0, seen.shape[0], rows_per_chunk): + stop = min(seen.shape[0], start + rows_per_chunk) + missing = ~seen[start:stop] + for target in fields.values(): + if target.ndim == 2: + target[start:stop][missing] = np.nan + + +def _close_matrix_memmaps(mapped: list[np.memmap]) -> None: + for values in reversed(mapped): + try: + values.flush() + except (OSError, ValueError): + pass + mmap_obj = getattr(values, "_mmap", None) + if mmap_obj is not None: + try: + mmap_obj.close() + except (OSError, ValueError): + pass + + +def _scan_matrix_values( + dataset: pads.Dataset, + filter_expr, + actual_dates: list[date], + actual_symbols: list[str], + arrays: Mapping[str, np.ndarray], + fields: Mapping[str, np.ndarray], + parquet_fields: list[str], + seen: np.ndarray, + *, + batch_size: int, +) -> None: + date_to_id = {value: index for index, value in enumerate(actual_dates)} + symbol_to_id = {value: index for index, value in enumerate(actual_symbols)} + scan_columns = [ + "symbol", + "date", + "open", + "high", + "low", + "close", + "volume", + *parquet_fields, + ] + scanner = dataset.scanner( + columns=scan_columns, + filter=filter_expr, + batch_size=int(batch_size), + use_threads=True, + ) + flat_seen = seen.ravel() + asset_count = len(actual_symbols) + scan_targets = { + "open": arrays["open"], + "high": arrays["high"], + "low": arrays["low"], + "close": arrays["close"], + "volume": arrays["volume"], + **{name: fields[name] for name in parquet_fields}, + } + for batch in scanner.to_batches(): + time_ids = _arrow_axis_ids(_batch_column(batch, "date"), date_to_id) + asset_ids = _arrow_axis_ids(_batch_column(batch, "symbol"), symbol_to_id) + flat_ids = time_ids.astype(np.int64) * asset_count + asset_ids + if np.unique(flat_ids).size != flat_ids.size or flat_seen[flat_ids].any(): + raise ValueError("MarketDataMatrix requires unique date/symbol rows") + flat_seen[flat_ids] = True + for name, target in scan_targets.items(): + values = _arrow_float_values( + _batch_column(batch, name), + null_fill=0.0 if name == "volume" else np.nan, + ) + target[time_ids, asset_ids] = values + + +def _populate_matrix_derived_arrays( + actual_symbols: list[str], + arrays: Mapping[str, np.ndarray], + fields: dict[str, np.ndarray], + wanted_fields: frozenset[str], + instruments: pl.DataFrame | None, + seen: np.ndarray, + *, + parquet_fields: list[str], + vector_fields: list[str], +) -> tuple[list[str], Mapping[str, np.ndarray]]: + instrument_wanted = set(wanted_fields) + if "turnover_rate" in wanted_fields and "turnover_rate" not in fields: + instrument_wanted.add("float_shares") + names, instrument_fields, latest_limits = _instrument_axis_values( + actual_symbols, + instrument_wanted, + instruments, + time_count=seen.shape[0], + ) + for name, values in instrument_fields.items(): + if name in parquet_fields: + continue + if name not in fields: + fields[name] = values + continue + if name in vector_fields: + fields[name][:] = values[0] + else: + np.copyto(fields[name], values, where=seen) + if "raw_close" in wanted_fields and "raw_close" not in parquet_fields: + np.copyto(fields["raw_close"], arrays["close"]) + if "turnover_rate" in wanted_fields and "turnover_rate" not in parquet_fields: + float_shares = fields.get("float_shares") + if float_shares is None: + raise ValueError("matrix turnover_rate requires float_shares") + _write_turnover_rate_matrix( + fields["turnover_rate"], + arrays["volume"], + float_shares, + ) + return names, latest_limits + + +def _write_turnover_rate_matrix( + target: np.ndarray, + volume: np.ndarray, + float_shares: np.ndarray, +) -> None: + shares = ( + float_shares + if float_shares.ndim == 2 + else np.broadcast_to(float_shares.reshape(1, -1), volume.shape) + ) + rows_per_chunk = max(1, (32 * 1024 * 1024) // max(1, volume.shape[1] * 4)) + for start in range(0, volume.shape[0], rows_per_chunk): + stop = min(volume.shape[0], start + rows_per_chunk) + out = target[start:stop] + np.multiply(volume[start:stop], np.float32(10_000.0), out=out) + shares_chunk = shares[start:stop] + valid = np.isfinite(shares_chunk) & (shares_chunk != 0) + np.divide( + out, + shares_chunk, + out=out, + where=valid, + ) + out[~valid] = np.nan + + +def _write_tradable_matrix( + target: np.ndarray, + open_: np.ndarray, + high: np.ndarray, + low: np.ndarray, + close: np.ndarray, + volume: np.ndarray, +) -> None: + rows_per_chunk = max(1, (32 * 1024 * 1024) // max(1, open_.shape[1] * 4)) + for start in range(0, open_.shape[0], rows_per_chunk): + stop = min(open_.shape[0], start + rows_per_chunk) + values = ( + np.isfinite(open_[start:stop]) + & np.isfinite(high[start:stop]) + & np.isfinite(low[start:stop]) + & np.isfinite(close[start:stop]) + & np.isfinite(volume[start:stop]) + & (volume[start:stop] > 0) + ) + target[start:stop] = values.astype(np.uint8, copy=False) + + +def _matrix_time_axes(actual_dates: list[date]) -> tuple[np.ndarray, np.ndarray]: + epoch = date(1970, 1, 1) + timestamps = np.asarray( + [(value - epoch).days * 86_400_000 for value in actual_dates], + dtype=np.int64, + ) + return timestamps, np.arange(len(actual_dates), dtype=np.int32) + + +def _matrix_disk_cache_path( + cache_root: Path, + parquet_root: Path, + start: date, + end: date, + field_columns: frozenset[str], + symbols: tuple[str, ...] | None, + source_partitions: Mapping[str, str], + instrument_fingerprint: str, + profile_generation: str, + source_generation: str | None, +) -> Path: + payload = { + "version": _DIRECT_MATRIX_LOADER_VERSION, + "parquet_root": str(parquet_root.resolve()), + "coverage_start": start.isoformat(), + "coverage_end": end.isoformat(), + "fields": sorted(field_columns), + "symbols": symbols, + "source_partitions": dict(source_partitions), + "instrument_fingerprint": instrument_fingerprint, + "profile_generation": str(profile_generation), + "source_generation": source_generation, + } + digest = hashlib.blake2b( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8"), + digest_size=20, + ) + return cache_root / f"v{_DIRECT_MATRIX_LOADER_VERSION}-{digest.hexdigest()}" + + +def _partition_fingerprints( + root: Path, + start: date, + end: date, + *, + include_predecessor: bool = False, +) -> dict[str, str]: + selected: list[tuple[date, Path]] = [] + predecessor: tuple[date, Path] | None = None + for partition in root.glob("date=*"): + try: + partition_date = date.fromisoformat(partition.name.removeprefix("date=")) + except ValueError: + continue + if partition_date < start: + if predecessor is None or partition_date > predecessor[0]: + predecessor = (partition_date, partition) + continue + if partition_date <= end: + selected.append((partition_date, partition)) + if include_predecessor and predecessor is not None: + selected.append(predecessor) + result: dict[str, str] = {} + for partition_date, partition in sorted(selected): + digest = hashlib.blake2b(digest_size=20) + files = sorted(partition.rglob("*.parquet")) + if not files: + continue + for path in files: + stat = path.stat() + digest.update(str(path.relative_to(root)).encode("utf-8")) + digest.update(int(stat.st_size).to_bytes(8, "little", signed=False)) + digest.update(int(stat.st_mtime_ns).to_bytes(8, "little", signed=False)) + result[partition_date.isoformat()] = digest.hexdigest() + return result + + +def _partition_date_bounds(root: Path) -> tuple[date | None, date | None]: + earliest: date | None = None + latest: date | None = None + for partition in root.glob("date=*"): + try: + value = date.fromisoformat(partition.name.removeprefix("date=")) + except ValueError: + continue + if earliest is None or value < earliest: + earliest = value + if latest is None or value > latest: + latest = value + return earliest, latest + + +def _latest_partition_date(root: Path) -> date | None: + return _partition_date_bounds(root)[1] + + +def _instrument_fingerprint(instruments: pl.DataFrame | None) -> bytes: + if instruments is None or instruments.is_empty() or "symbol" not in instruments.columns: + return b"no-instruments" + columns = [ + name + for name in ("symbol", "name", "total_shares", "float_shares", "limit_up", "limit_down") + if name in instruments.columns + ] + payload = instruments.select(columns).sort("symbol").to_dicts() + return hashlib.blake2b( + json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8"), + digest_size=20, + ).digest() + + +def _find_covering_matrix_cache( + root: Path, + parquet_root: Path, + start: date, + end: date, + requested_fields: frozenset[str], + symbols: tuple[str, ...] | None, + source_partitions: Mapping[str, str], + instrument_fingerprint: str, + source_generation: str | None, +) -> tuple[Path, str] | None: + matches: list[tuple[int, int, Path, str]] = [] + for path in root.glob(f"v{_DIRECT_MATRIX_LOADER_VERSION}-*"): + manifest_path = path / "manifest.json" + if not manifest_path.exists(): + continue + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if int(manifest.get("version", -1)) != _DIRECT_MATRIX_LOADER_VERSION: + continue + if manifest.get("parquet_root") != str(parquet_root.resolve()): + continue + if manifest.get("symbols_request") != (None if symbols is None else list(symbols)): + continue + if manifest.get("instrument_fingerprint") != instrument_fingerprint: + continue + if source_generation is not None: + if manifest.get("source_generation") != source_generation: + continue + cached_start = date.fromisoformat(str(manifest["coverage_start"])) + cached_end = date.fromisoformat(str(manifest["coverage_end"])) + if cached_start > start or cached_end < end: + continue + cached_fields = frozenset(str(name) for name in manifest["cache_field_columns"]) + if not requested_fields.issubset(cached_fields): + continue + cached_partitions = manifest.get("source_partitions", {}) + if source_generation is None: + relevant_partitions = { + key: value + for key, value in source_partitions.items() + if date.fromisoformat(key) >= cached_start + } + if any( + cached_partitions.get(key) != value + for key, value in relevant_partitions.items() + ): + continue + storage = path / str(manifest.get("storage", "matrix.bin")) + size = storage.stat().st_size + exact = ( + cached_start == start + and cached_end == end + and cached_fields == requested_fields + ) + matches.append((size, -path.stat().st_mtime_ns, path, "exact" if exact else "covering")) + except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError): + continue + if not matches: + return None + _size, _mtime, path, status = min(matches) + return path, status + + +class _MatrixDiskCacheLease: + def __init__(self, path: Path) -> None: + self.path = str(path) + with _MATRIX_DISK_CACHE_LOCK: + _MATRIX_DISK_CACHE_LEASES[self.path] = ( + _MATRIX_DISK_CACHE_LEASES.get(self.path, 0) + 1 + ) + + def __del__(self) -> None: + path = self.path + should_delete = False + with _MATRIX_DISK_CACHE_LOCK: + remaining = _MATRIX_DISK_CACHE_LEASES.get(path, 0) - 1 + if remaining > 0: + _MATRIX_DISK_CACHE_LEASES[path] = remaining + else: + _MATRIX_DISK_CACHE_LEASES.pop(path, None) + should_delete = path in _MATRIX_DISK_CACHE_PENDING_DELETE + if should_delete: + _try_delete_matrix_cache_path(Path(path)) + + +def _try_delete_matrix_cache_path(path: Path) -> bool: + try: + shutil.rmtree(path) + except OSError as exc: + logger.debug("matrix disk cache prune skipped %s: %s", path, exc) + with _MATRIX_DISK_CACHE_LOCK: + _MATRIX_DISK_CACHE_PENDING_DELETE.add(str(path)) + return False + with _MATRIX_DISK_CACHE_LOCK: + _MATRIX_DISK_CACHE_PENDING_DELETE.discard(str(path)) + return True + + +def _load_market_data_matrix_cache( + path: Path, + *, + cache_status: str, +) -> MarketDataMatrix: + manifest = json.loads((path / "manifest.json").read_text(encoding="utf-8")) + if int(manifest["version"]) != _DIRECT_MATRIX_LOADER_VERSION: + raise ValueError("matrix disk cache version mismatch") + storage_path = path / str(manifest["storage"]) + + def load_array(spec: Mapping[str, Any]) -> np.ndarray: + values = np.memmap( + storage_path, + dtype=np.dtype(str(spec["dtype"])), + mode="r", + offset=int(spec["offset"]), + shape=tuple(int(value) for value in spec["shape"]), + order="C", + ) + values.flags.writeable = False + return values + + arrays = {name: load_array(spec) for name, spec in manifest["arrays"].items()} + stored_fields = {name: load_array(spec) for name, spec in manifest["fields"].items()} + shape = arrays["open"].shape + if any( + values.shape != shape + for name, values in arrays.items() + if name not in {"timestamps", "session_ids"} + ): + raise ValueError("matrix disk cache contains inconsistent array shapes") + if any( + values.shape not in {shape, (shape[1],)} + for values in stored_fields.values() + ): + raise ValueError("matrix disk cache contains inconsistent field shapes") + vector_field_names = frozenset( + name + for name, values in stored_fields.items() + if values.shape == (shape[1],) + ) + fields = { + name: ( + values + if values.shape == shape + else np.broadcast_to(values.reshape(1, -1), shape) + ) + for name, values in stored_fields.items() + } + _make_read_only(*fields.values()) + os.utime(path, None) + return MarketDataMatrix( + timestamps=arrays["timestamps"], + timestamp_labels=tuple(str(value) for value in manifest["timestamp_labels"]), + session_ids=arrays["session_ids"], + symbols=tuple(str(value) for value in manifest["symbols"]), + names=tuple(str(value) for value in manifest["names"]), + open=arrays["open"], + high=arrays["high"], + low=arrays["low"], + close=arrays["close"], + volume=arrays["volume"], + tradable=arrays["tradable"], + limit_up_locked=arrays["limit_up_locked"], + limit_down_locked=arrays["limit_down_locked"], + fields=MappingProxyType(fields), + cache_status=cache_status, + cache_path=str(path), + cache_lease=_MatrixDiskCacheLease(path), + vector_fields=vector_field_names, + cache_timing_ms=MappingProxyType({ + str(name): float(value) + for name, value in manifest.get("build_timing_ms", {}).items() + }), + ) + + +def _slice_and_project_market_data_matrix( + market: MarketDataMatrix, + start: date, + end: date, + requested_fields: frozenset[str], +) -> MarketDataMatrix: + labels = market.timestamp_labels + start_label = start.isoformat() + end_label = end.isoformat() + start_id = 0 + while start_id < len(labels) and labels[start_id] < start_label: + start_id += 1 + stop_id = start_id + while stop_id < len(labels) and labels[stop_id] <= end_label: + stop_id += 1 + if start_id >= stop_id: + raise ValueError("matrix parquet range contains no market data") + sliced = slice_market_data_matrix(market, start_id, stop_id) + missing = requested_fields - set(sliced.fields) + if missing: + raise ValueError(f"matrix disk cache missing requested fields: {sorted(missing)}") + projected_values: dict[str, np.ndarray] = {} + for name in sorted(requested_fields): + values = sliced.fields[name] + if name in sliced.vector_fields: + values = np.where(np.isfinite(sliced.close), values, np.nan).astype( + np.float32, + copy=False, + ) + values.flags.writeable = False + projected_values[name] = values + projected = MappingProxyType(projected_values) + return MarketDataMatrix( + timestamps=sliced.timestamps, + timestamp_labels=sliced.timestamp_labels, + session_ids=sliced.session_ids, + symbols=sliced.symbols, + names=sliced.names, + open=sliced.open, + high=sliced.high, + low=sliced.low, + close=sliced.close, + volume=sliced.volume, + tradable=sliced.tradable, + limit_up_locked=sliced.limit_up_locked, + limit_down_locked=sliced.limit_down_locked, + fields=projected, + cache_status=sliced.cache_status, + cache_path=sliced.cache_path, + cache_lease=sliced.cache_lease, + vector_fields=frozenset(), + cache_timing_ms=sliced.cache_timing_ms, + ) + + +def _prune_matrix_disk_cache( + root: Path, + *, + keep: Path, + max_bytes: int, + current_source_generation: str | None = None, +) -> None: + if max_bytes <= 0: + raise ValueError("matrix disk cache max_bytes must be positive") + with _MATRIX_DISK_CACHE_LOCK: + pending = [Path(value) for value in _MATRIX_DISK_CACHE_PENDING_DELETE] + for path in pending: + with _MATRIX_DISK_CACHE_LOCK: + leased = _MATRIX_DISK_CACHE_LEASES.get(str(path), 0) > 0 + if not leased: + _try_delete_matrix_cache_path(path) + + entries: list[tuple[Path, int, int]] = [] + for path in root.glob("v*-*"): + if not path.is_dir(): + continue + try: + size = sum(item.stat().st_size for item in path.rglob("*") if item.is_file()) + entries.append((path, size, path.stat().st_mtime_ns)) + except OSError: + continue + if current_source_generation is not None: + try: + keep_manifest = json.loads( + (keep / "manifest.json").read_text(encoding="utf-8") + ) + keep_parquet_root = keep_manifest.get("parquet_root") + except (OSError, ValueError, TypeError, json.JSONDecodeError): + keep_parquet_root = None + for path, _size, _mtime in list(entries): + if path == keep: + continue + try: + manifest = json.loads( + (path / "manifest.json").read_text(encoding="utf-8") + ) + same_universe = ( + keep_parquet_root is not None + and manifest.get("parquet_root") == keep_parquet_root + and manifest.get("symbols_request") is None + ) + old_generation = manifest.get("source_generation") != current_source_generation + except (OSError, ValueError, TypeError, json.JSONDecodeError): + same_universe = False + old_generation = False + if not (same_universe and old_generation): + continue + with _MATRIX_DISK_CACHE_LOCK: + leased = _MATRIX_DISK_CACHE_LEASES.get(str(path), 0) > 0 + if leased: + _MATRIX_DISK_CACHE_PENDING_DELETE.add(str(path)) + if not leased: + _try_delete_matrix_cache_path(path) + entries = [entry for entry in entries if entry[0] == keep or entry[0].exists()] + total = sum(size for _path, size, _mtime in entries) + for path, size, _mtime in sorted(entries, key=lambda item: item[2]): + if total <= max_bytes: + break + if path == keep: + continue + with _MATRIX_DISK_CACHE_LOCK: + leased = _MATRIX_DISK_CACHE_LEASES.get(str(path), 0) > 0 + if leased: + _MATRIX_DISK_CACHE_PENDING_DELETE.add(str(path)) + if leased: + continue + if _try_delete_matrix_cache_path(path): + total -= size + + +def _matrix_axis_cache_path( + cache_root: Path, + parquet_root: Path, + start: date, + end: date, + symbols: tuple[str, ...] | None, +) -> Path: + payload = json.dumps( + { + "root": str(parquet_root.resolve()), + "start": start.isoformat(), + "end": end.isoformat(), + "symbols": symbols, + }, + sort_keys=True, + separators=(",", ":"), + ) + digest = hashlib.blake2b(payload.encode("utf-8"), digest_size=16).hexdigest() + return cache_root / f".axes-v{_MATRIX_AXIS_INDEX_VERSION}-{digest}.json" + + +def _load_or_build_matrix_axes( + dataset: pads.Dataset, + parquet_root: Path, + start: date, + end: date, + symbols: tuple[str, ...] | None, + source_partitions: Mapping[str, str], + filter_expr, + *, + batch_size: int, + cache_root: Path, +) -> tuple[list[date], list[str]]: + path = _matrix_axis_cache_path(cache_root, parquet_root, start, end, symbols) + previous: dict[str, Any] | None = None + if path.exists(): + try: + previous = json.loads(path.read_text(encoding="utf-8")) + if ( + int(previous.get("version", -1)) == _MATRIX_AXIS_INDEX_VERSION + and previous.get("source_partitions") == dict(source_partitions) + ): + return ( + [date.fromisoformat(value) for value in previous["dates"]], + [str(value) for value in previous["symbols"]], + ) + except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError): + previous = None + + if previous is not None: + previous_partitions = previous.get("source_partitions", {}) + changed_labels = { + value + for value, fingerprint in source_partitions.items() + if previous_partitions.get(value) != fingerprint + } + removed_labels = set(previous_partitions) - set(source_partitions) + rewritten_labels = { + value for value in changed_labels if value in previous_partitions + } + if removed_labels or rewritten_labels: + actual_dates, actual_symbols = _collect_parquet_axes( + dataset, + filter_expr, + batch_size=batch_size, + ) + changed_labels = set() + retained_dates = {value.isoformat() for value in actual_dates} + else: + retained_dates = { + str(value) + for value in previous.get("dates", []) + if str(value) in source_partitions and str(value) not in changed_labels + } + actual_symbols = sorted({str(value) for value in previous.get("symbols", [])}) + if changed_labels: + changed_dates = [date.fromisoformat(value) for value in sorted(changed_labels)] + changed_filter = _matrix_filter_expression( + min(changed_dates), + max(changed_dates), + symbols, + ) & pads.field("date").isin(changed_dates) + scanner = dataset.scanner( + columns=["date", "symbol"], + filter=changed_filter, + batch_size=int(batch_size), + use_threads=True, + ) + symbols_set = set(actual_symbols) + for batch in scanner.to_batches(): + retained_dates.update( + value.isoformat() + for value in pc.unique(_batch_column(batch, "date")).to_pylist() + ) + symbols_set.update( + str(value) + for value in pc.unique(_batch_column(batch, "symbol")).to_pylist() + ) + actual_symbols = sorted(symbols_set) + actual_dates = [date.fromisoformat(value) for value in sorted(retained_dates)] + else: + actual_dates, actual_symbols = _collect_parquet_axes( + dataset, + filter_expr, + batch_size=batch_size, + ) + + payload = { + "version": _MATRIX_AXIS_INDEX_VERSION, + "source_partitions": dict(source_partitions), + "dates": [value.isoformat() for value in actual_dates], + "symbols": list(actual_symbols), + } + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + temporary.write_text( + json.dumps(payload, ensure_ascii=False, separators=(",", ":")), + encoding="utf-8", + ) + os.replace(temporary, path) + return actual_dates, actual_symbols + + +def _collect_parquet_axes( + dataset: pads.Dataset, + filter_expr, + *, + batch_size: int, +) -> tuple[list[date], list[str]]: + dates: set[date] = set() + symbols: set[str] = set() + scanner = dataset.scanner( + columns=["date", "symbol"], + filter=filter_expr, + batch_size=int(batch_size), + use_threads=True, + ) + for batch in scanner.to_batches(): + dates.update(pc.unique(_batch_column(batch, "date")).to_pylist()) + symbols.update( + str(value) + for value in pc.unique(_batch_column(batch, "symbol")).to_pylist() + ) + return sorted(dates), sorted(symbols) + + +def _arrow_axis_ids(values: pa.Array, mapping: Mapping[Any, int]) -> np.ndarray: + encoded = pc.dictionary_encode(values) + dictionary_ids = np.asarray( + [mapping[value] for value in encoded.dictionary.to_pylist()], + dtype=np.int32, + ) + indices = encoded.indices.to_numpy(zero_copy_only=False) + return dictionary_ids[np.asarray(indices, dtype=np.int32)] + + +def _batch_column(batch: pa.RecordBatch, name: str) -> pa.Array: + return batch.column(batch.schema.get_field_index(name)) + + +def _arrow_float_values(values: pa.Array, *, null_fill: float) -> np.ndarray: + casted = pc.cast(values, pa.float32(), safe=False) + if casted.null_count: + casted = pc.fill_null(casted, pa.scalar(null_fill, type=pa.float32())) + result = np.array( + casted.to_numpy(zero_copy_only=False), + dtype=np.float32, + copy=True, + ) + replacement = np.float32(null_fill) + result[~np.isfinite(result)] = replacement + return result + + +def _arrow_numeric(value_type: pa.DataType) -> bool: + return bool( + pa.types.is_integer(value_type) + or pa.types.is_floating(value_type) + or pa.types.is_decimal(value_type) + ) + + +def _instrument_axis_values( + symbols: list[str], + wanted_fields: set[str], + instruments: pl.DataFrame | None, + *, + time_count: int, +) -> tuple[list[str], dict[str, np.ndarray], dict[str, np.ndarray]]: + names = [""] * len(symbols) + fields: dict[str, np.ndarray] = {} + limits = { + "limit_up": np.full(len(symbols), np.nan, dtype=np.float32), + "limit_down": np.full(len(symbols), np.nan, dtype=np.float32), + } + if instruments is None or instruments.is_empty() or "symbol" not in instruments.columns: + return names, fields, limits + + by_symbol = { + str(row["symbol"]): row + for row in instruments.unique(subset=["symbol"]).iter_rows(named=True) + } + numeric_fields = [ + name + for name in sorted(wanted_fields) + if name in instruments.columns and instruments[name].dtype.is_numeric() + ] + vectors = { + name: np.full(len(symbols), np.nan, dtype=np.float32) + for name in numeric_fields + } + for asset_id, symbol in enumerate(symbols): + row = by_symbol.get(symbol) + if row is None: + continue + names[asset_id] = str(row.get("name") or "") + for name, target in vectors.items(): + value = row.get(name) + if value is not None: + target[asset_id] = np.float32(value) + for name, target in limits.items(): + value = row.get(name) + if value is not None: + target[asset_id] = np.float32(value) + + shape = (1, len(symbols)) + fields.update({ + name: np.broadcast_to(values.reshape(shape), (time_count, len(symbols))) + for name, values in vectors.items() + }) + return names, fields, limits + + +def _limit_lock_matrices( + close: np.ndarray, + raw_close: np.ndarray, + seen: np.ndarray, + symbols: list[str], + names: list[str], + latest_limits: Mapping[str, np.ndarray], + *, + out_up: np.ndarray | None = None, + out_down: np.ndarray | None = None, + apply_latest_limits: bool = True, +) -> tuple[np.ndarray, np.ndarray]: + shape = close.shape + up_locked = out_up if out_up is not None else np.zeros(shape, dtype=np.uint8) + down_locked = out_down if out_down is not None else np.zeros(shape, dtype=np.uint8) + if up_locked.shape != shape or down_locked.shape != shape: + raise ValueError("limit lock output shape mismatch") + up_locked.fill(0) + down_locked.fill(0) + board_pct = np.full(shape[1], 0.10, dtype=np.float64) + for asset_id, symbol in enumerate(symbols): + if symbol.startswith(("300", "301", "688", "689")): + board_pct[asset_id] = 0.20 + elif symbol.endswith(".BJ"): + board_pct[asset_id] = 0.30 + elif "ST" in names[asset_id]: + board_pct[asset_id] = 0.05 + + previous_close = np.full(shape[1], np.nan, dtype=np.float64) + previous_raw = np.full(shape[1], np.nan, dtype=np.float64) + previous_adjustment = np.full(shape[1], np.nan, dtype=np.float64) + for time_id in range(shape[0]): + present = seen[time_id] + current_close = close[time_id].astype(np.float64, copy=False) + current_raw = raw_close[time_id].astype(np.float64, copy=False) + current_adjustment = np.full(shape[1], np.nan, dtype=np.float64) + np.divide( + current_close, + current_raw, + out=current_adjustment, + where=np.isfinite(current_raw) & (current_raw != 0), + ) + adjustment_changed = ( + np.isfinite(current_adjustment) + & np.isfinite(previous_adjustment) + & (np.abs(current_adjustment - previous_adjustment) > 1e-6) + ) + reference = np.where(adjustment_changed, previous_close, previous_raw) + valid = ( + present + & np.isfinite(reference) + & (reference > 0) + & np.isfinite(current_raw) + & (current_raw > 0) + ) + if valid.any(): + up_price = _numpy_limit_price(reference, board_pct, up=True) + down_price = _numpy_limit_price(reference, board_pct, up=False) + if apply_latest_limits and time_id == shape[0] - 1: + latest_up = latest_limits["limit_up"] + latest_down = latest_limits["limit_down"] + use_up = np.isfinite(latest_up) & (latest_up < 10_000.0) + use_down = np.isfinite(latest_down) & (latest_down < 10_000.0) + up_price = np.where(use_up, latest_up, up_price) + down_price = np.where(use_down, latest_down, down_price) + up_locked[time_id, valid] = ( + current_raw[valid] >= up_price[valid] - 0.005 + ).astype(np.uint8) + down_locked[time_id, valid] = ( + current_raw[valid] <= down_price[valid] + 0.005 + ).astype(np.uint8) + + previous_close[present] = current_close[present] + previous_raw[present] = current_raw[present] + previous_adjustment[present] = current_adjustment[present] + return up_locked, down_locked + + +def _numpy_limit_price( + previous: np.ndarray, + limit_pct: np.ndarray, + *, + up: bool, +) -> np.ndarray: + sign = 1 if up else -1 + numerator = np.rint((1.0 + sign * limit_pct) * 100.0).astype(np.int64) + result = np.full(previous.shape, np.nan, dtype=np.float64) + finite = np.isfinite(previous) + cents = np.floor(previous[finite] * 100.0 + 0.5).astype(np.int64) + result[finite] = ( + ((cents * numerator[finite] + 50) // 100).astype(np.float64) / 100.0 + ) + return result + + +def make_signal_matrix( + shape: tuple[int, int], + *, + entry: np.ndarray | None = None, + exit: np.ndarray | None = None, + score: np.ndarray | None = None, + entry_signal_code: np.ndarray | None = None, + exit_signal_code: np.ndarray | None = None, + entry_signal_ids: tuple[str, ...] = (), + exit_signal_ids: tuple[str, ...] = (), +) -> SignalMatrix: + """Create a compact read-only signal matrix with canonical dtypes.""" + entry_array = _coerce_array(entry, shape, np.uint8, 0) + exit_array = _coerce_array(exit, shape, np.uint8, 0) + score_array = _coerce_array(score, shape, np.float32, 0.0) + entry_codes = _coerce_array(entry_signal_code, shape, np.int16, -1) + exit_codes = _coerce_array(exit_signal_code, shape, np.int16, -1) + return _finalize_signal_matrix( + entry_array, + exit_array, + score_array, + entry_codes, + exit_codes, + entry_signal_ids=entry_signal_ids, + exit_signal_ids=exit_signal_ids, + ) + + +def _finalize_signal_matrix( + entry: np.ndarray, + exit_: np.ndarray, + score: np.ndarray, + entry_signal_code: np.ndarray, + exit_signal_code: np.ndarray, + *, + entry_signal_ids: tuple[str, ...] = (), + exit_signal_ids: tuple[str, ...] = (), +) -> SignalMatrix: + shape = entry.shape + _make_read_only(entry, exit_, score, entry_signal_code, exit_signal_code) + result = SignalMatrix( + entry=entry, + exit=exit_, + score=score, + entry_signal_code=entry_signal_code, + exit_signal_code=exit_signal_code, + entry_signal_ids=tuple(entry_signal_ids), + exit_signal_ids=tuple(exit_signal_ids), + ) + validate_signal_matrix(result, shape) + return result + + +def validate_signal_matrix(signals: SignalMatrix, shape: tuple[int, int]) -> None: + """Fail explicitly when a matrix strategy violates the shared output contract.""" + specs = { + "entry": (signals.entry, np.dtype(np.uint8)), + "exit": (signals.exit, np.dtype(np.uint8)), + "score": (signals.score, np.dtype(np.float32)), + "entry_signal_code": (signals.entry_signal_code, np.dtype(np.int16)), + "exit_signal_code": (signals.exit_signal_code, np.dtype(np.int16)), + } + for name, (array, dtype) in specs.items(): + if not isinstance(array, np.ndarray): + raise TypeError(f"SignalMatrix.{name} must be a numpy array") + if array.shape != shape: + raise ValueError( + f"SignalMatrix.{name} shape {array.shape} does not match market {shape}" + ) + if array.dtype != dtype: + raise TypeError(f"SignalMatrix.{name} must use {dtype}, got {array.dtype}") + if array.flags.writeable: + raise ValueError(f"SignalMatrix.{name} must be read-only") + if not np.isfinite(signals.score).all(): + raise ValueError("SignalMatrix.score must contain only finite values") + + +def build_market_matrix_from_signals( + market: MarketDataMatrix, + signals: SignalMatrix, + *, + entry_delay_bars: int = 0, + exit_delay_bars: int = 0, + reference_price: np.ndarray | None = None, +) -> MarketMatrix: + """Combine base data and strategy signals into the matcher input matrix.""" + if entry_delay_bars not in (0, 1) or exit_delay_bars not in (0, 1): + raise ValueError("phase-two MarketMatrix supports only zero or one bar delay") + validate_signal_matrix(signals, market.shape) + + present = _present_matrix(market.open, market.high, market.low, market.close, market.volume) + entry, entry_signal_time, entry_signal_code = _delay_signal_matrix( + signals.entry, + signals.entry_signal_code, + present, + entry_delay_bars, + ) + exit_, exit_signal_time, exit_signal_code = _delay_signal_matrix( + signals.exit, + signals.exit_signal_code, + present, + exit_delay_bars, + ) + + if reference_price is not None: + if reference_price.shape != market.shape: + raise ValueError("reference_price shape does not match MarketDataMatrix") + resolved_reference_price = np.array(reference_price, dtype=np.float32, copy=True) + else: + resolved_reference_price = np.full(market.shape, np.nan, dtype=np.float32) + for column in ("ma5", "ma10", "ma20"): + values = market.fields.get(column) + if values is None: + continue + use = ~np.isfinite(resolved_reference_price) & np.isfinite(values) & (values > 0) + resolved_reference_price[use] = values[use] + + _make_read_only( + entry, + exit_, + resolved_reference_price, + entry_signal_time, + exit_signal_time, + entry_signal_code, + exit_signal_code, + ) + + return MarketMatrix( + timestamps=market.timestamps, + timestamp_labels=market.timestamp_labels, + session_ids=market.session_ids, + symbols=market.symbols, + names=market.names, + open=market.open, + high=market.high, + low=market.low, + close=market.close, + volume=market.volume, + score=signals.score, + entry=entry, + exit=exit_, + tradable=market.tradable, + limit_up_locked=market.limit_up_locked, + limit_down_locked=market.limit_down_locked, + reference_price=resolved_reference_price, + entry_signal_time=entry_signal_time, + exit_signal_time=exit_signal_time, + entry_signal_code=entry_signal_code, + exit_signal_code=exit_signal_code, + entry_signal_ids=signals.entry_signal_ids, + exit_signal_ids=signals.exit_signal_ids, + ) + + +def build_market_matrix( + panel: pl.DataFrame, + entries: pl.Series | None, + exits: pl.Series | None, + *, + entry_delay_bars: int = 0, + exit_delay_bars: int = 0, + entry_signal_ids: list[str] | None = None, + exit_signal_ids: list[str] | None = None, +) -> MarketMatrix: + """Backward-compatible long-panel boundary used by legacy/Polars strategies.""" + if panel.is_empty(): + raise ValueError("cannot build MarketMatrix from an empty panel") + market = build_market_data_matrix( + panel, + field_columns={"score", "ma5", "ma10", "ma20"}, + ) + _, _, _, time_id, asset_id = _encode_axes(panel) + shape = market.shape + + raw_entry = _scatter_bool_series(entries, len(panel), shape, time_id, asset_id) + raw_exit = _scatter_bool_series(exits, len(panel), shape, time_id, asset_id) + entry_codes, normalized_entry_ids = _signal_code_matrix( + panel, + entry_signal_ids, + shape, + time_id, + asset_id, + ) + exit_codes, normalized_exit_ids = _signal_code_matrix( + panel, + exit_signal_ids, + shape, + time_id, + asset_id, + ) + score = market.fields.get("score") + signals = make_signal_matrix( + shape, + entry=raw_entry, + exit=raw_exit, + score=np.nan_to_num(score, nan=0.0) if score is not None else None, + entry_signal_code=entry_codes, + exit_signal_code=exit_codes, + entry_signal_ids=normalized_entry_ids, + exit_signal_ids=normalized_exit_ids, + ) + return build_market_matrix_from_signals( + market, + signals, + entry_delay_bars=entry_delay_bars, + exit_delay_bars=exit_delay_bars, + ) + + +def slice_market_data_matrix(market: MarketDataMatrix, start: int, stop: int) -> MarketDataMatrix: + """Return a read-only time slice without copying the underlying market arrays.""" + fields = {name: values[start:stop] for name, values in market.fields.items()} + result = MarketDataMatrix( + timestamps=market.timestamps[start:stop], + timestamp_labels=market.timestamp_labels[start:stop], + session_ids=market.session_ids[start:stop], + symbols=market.symbols, + names=market.names, + open=market.open[start:stop], + high=market.high[start:stop], + low=market.low[start:stop], + close=market.close[start:stop], + volume=market.volume[start:stop], + tradable=market.tradable[start:stop], + limit_up_locked=market.limit_up_locked[start:stop], + limit_down_locked=market.limit_down_locked[start:stop], + fields=MappingProxyType(fields), + cache_status=market.cache_status, + cache_path=market.cache_path, + cache_lease=market.cache_lease, + vector_fields=market.vector_fields, + cache_timing_ms=market.cache_timing_ms, + ) + _make_read_only( + result.timestamps, + result.session_ids, + result.open, + result.high, + result.low, + result.close, + result.volume, + result.tradable, + result.limit_up_locked, + result.limit_down_locked, + *fields.values(), + ) + return result + + +def slice_signal_matrix(signals: SignalMatrix, start: int, stop: int) -> SignalMatrix: + return _finalize_signal_matrix( + signals.entry[start:stop], + signals.exit[start:stop], + signals.score[start:stop], + signals.entry_signal_code[start:stop], + signals.exit_signal_code[start:stop], + entry_signal_ids=signals.entry_signal_ids, + exit_signal_ids=signals.exit_signal_ids, + ) + + +class RealtimeMarketDataMatrix: + """Mutable staging buffer for one live asset universe. + + Historical rows are built once. Repeated snapshots for the current bar only + overwrite the last row; a later timestamp appends one row. Strategies only + receive read-only views through :meth:`snapshot`. + """ + + def __init__( + self, + panel: pl.DataFrame, + *, + field_columns: set[str] | frozenset[str], + build_count: int = 1, + ) -> None: + self.field_columns = frozenset(field_columns) + self.market = _writable_market_copy( + build_market_data_matrix(panel, field_columns=self.field_columns) + ) + self.generation = 1 + self.build_count = int(build_count) + self.update_count = 0 + + def update(self, latest_panel: pl.DataFrame) -> None: + if latest_panel.is_empty(): + raise ValueError("cannot update live matrix from an empty panel") + timestamp_col = "datetime" if "datetime" in latest_panel.columns else "date" + if timestamp_col not in latest_panel.columns: + raise ValueError("live matrix update requires date or datetime") + if latest_panel[timestamp_col].n_unique() != 1: + raise ValueError("live matrix update must contain exactly one timestamp") + + latest = build_market_data_matrix( + latest_panel, + field_columns=self.field_columns, + ) + target_ids = _live_target_asset_ids(self.market.symbols, latest.symbols) + latest_timestamp = int(latest.timestamps[0]) + current_timestamp = int(self.market.timestamps[-1]) + if latest_timestamp < current_timestamp: + raise ValueError("live matrix update timestamp is older than current snapshot") + + if latest_timestamp == current_timestamp: + _overwrite_latest_market_row(self.market, latest, target_ids) + else: + self.market = _append_market_row(self.market, latest, target_ids) + self.generation += 1 + self.update_count += 1 + + def snapshot(self) -> MarketDataMatrix: + return _readonly_market_view(self.market) + + +def _live_target_asset_ids( + symbols: tuple[str, ...], + latest_symbols: tuple[str, ...], +) -> np.ndarray: + symbol_array = np.asarray(symbols) + latest_array = np.asarray(latest_symbols) + target_ids = np.searchsorted(symbol_array, latest_array) + valid = target_ids < len(symbol_array) + if not valid.all() or not np.array_equal(symbol_array[target_ids], latest_array): + raise ValueError("live matrix symbol axis changed; rebuild required") + return target_ids.astype(np.int32, copy=False) + + +def _market_array_fields() -> tuple[str, ...]: + return ( + "open", + "high", + "low", + "close", + "volume", + "tradable", + "limit_up_locked", + "limit_down_locked", + ) + + +def _writable_market_copy(market: MarketDataMatrix) -> MarketDataMatrix: + fields = {name: np.array(values, copy=True) for name, values in market.fields.items()} + return MarketDataMatrix( + timestamps=np.array(market.timestamps, copy=True), + timestamp_labels=market.timestamp_labels, + session_ids=np.array(market.session_ids, copy=True), + symbols=market.symbols, + names=market.names, + open=np.array(market.open, copy=True), + high=np.array(market.high, copy=True), + low=np.array(market.low, copy=True), + close=np.array(market.close, copy=True), + volume=np.array(market.volume, copy=True), + tradable=np.array(market.tradable, copy=True), + limit_up_locked=np.array(market.limit_up_locked, copy=True), + limit_down_locked=np.array(market.limit_down_locked, copy=True), + fields=MappingProxyType(fields), + ) + + +def _readonly_view(values: np.ndarray) -> np.ndarray: + view = values.view() + view.flags.writeable = False + return view + + +def _readonly_market_view(market: MarketDataMatrix) -> MarketDataMatrix: + fields = {name: _readonly_view(values) for name, values in market.fields.items()} + return MarketDataMatrix( + timestamps=_readonly_view(market.timestamps), + timestamp_labels=market.timestamp_labels, + session_ids=_readonly_view(market.session_ids), + symbols=market.symbols, + names=market.names, + open=_readonly_view(market.open), + high=_readonly_view(market.high), + low=_readonly_view(market.low), + close=_readonly_view(market.close), + volume=_readonly_view(market.volume), + tradable=_readonly_view(market.tradable), + limit_up_locked=_readonly_view(market.limit_up_locked), + limit_down_locked=_readonly_view(market.limit_down_locked), + fields=MappingProxyType(fields), + ) + + +def _overwrite_latest_market_row( + market: MarketDataMatrix, + latest: MarketDataMatrix, + target_ids: np.ndarray, +) -> None: + for name in _market_array_fields(): + getattr(market, name)[-1, target_ids] = getattr(latest, name)[0] + for name, values in market.fields.items(): + latest_values = latest.fields.get(name) + if latest_values is None: + raise ValueError(f"live matrix update missing field: {name}") + values[-1, target_ids] = latest_values[0] + object.__setattr__(market, "_valid_bars", None) + + +def _append_market_row( + market: MarketDataMatrix, + latest: MarketDataMatrix, + target_ids: np.ndarray, +) -> MarketDataMatrix: + asset_count = len(market.symbols) + + def aligned(values: np.ndarray, fill: float | int) -> np.ndarray: + row = np.full(asset_count, fill, dtype=values.dtype) + row[target_ids] = values[0] + return row + + arrays: dict[str, np.ndarray] = {} + for name in _market_array_fields(): + source = getattr(latest, name) + fill = np.nan if np.issubdtype(source.dtype, np.floating) else 0 + arrays[name] = np.concatenate( + [getattr(market, name), aligned(source, fill)[None, :]], + axis=0, + ) + + fields: dict[str, np.ndarray] = {} + for name, old_values in market.fields.items(): + latest_values = latest.fields.get(name) + if latest_values is None: + raise ValueError(f"live matrix update missing field: {name}") + fields[name] = np.concatenate( + [old_values, aligned(latest_values, np.nan)[None, :]], + axis=0, + ) + + latest_label = latest.timestamp_labels[0] + same_session = latest_label[:10] == market.timestamp_labels[-1][:10] + next_session = int(market.session_ids[-1]) if same_session else int(market.session_ids[-1]) + 1 + return MarketDataMatrix( + timestamps=np.concatenate([market.timestamps, latest.timestamps[:1]]), + timestamp_labels=(*market.timestamp_labels, latest_label), + session_ids=np.concatenate([ + market.session_ids, + np.array([next_session], dtype=np.int32), + ]), + symbols=market.symbols, + names=market.names, + fields=MappingProxyType(fields), + **arrays, + ) + + +def _encode_axes( + panel: pl.DataFrame, +) -> tuple[str, pl.Series, np.ndarray, np.ndarray, np.ndarray]: + timestamp_col = "datetime" if "datetime" in panel.columns else "date" + required = {timestamp_col, "symbol", "open", "high", "low", "close"} + missing = required - set(panel.columns) + if missing: + raise ValueError(f"MarketDataMatrix missing columns: {sorted(missing)}") + + timestamp_series = panel[timestamp_col] + unique_timestamps = timestamp_series.unique().sort() + symbol_series = panel["symbol"].cast(pl.Utf8) + unique_symbols = symbol_series.unique().sort() + row_timestamps = timestamp_series.to_numpy() + timestamp_values = unique_timestamps.to_numpy() + row_symbols = symbol_series.to_numpy() + symbol_values = unique_symbols.to_numpy() + time_id = np.searchsorted(timestamp_values, row_timestamps).astype(np.int32) + asset_id = np.searchsorted(symbol_values, row_symbols).astype(np.int32) + keys = time_id.astype(np.int64) * len(symbol_values) + asset_id + if np.unique(keys).size != len(panel): + raise ValueError("MarketDataMatrix requires unique timestamp/symbol rows") + return timestamp_col, unique_timestamps, symbol_values, time_id, asset_id + + +def _float_matrix( + panel: pl.DataFrame, + column: str, + shape: tuple[int, int], + time_id: np.ndarray, + asset_id: np.ndarray, + default: float = np.nan, + null_fill: float | None = None, +) -> np.ndarray: + out = np.full(shape, default, dtype=np.float32) + if column not in panel.columns: + return out + values = np.array( + panel[column].cast(pl.Float32, strict=False).to_numpy(), + dtype=np.float32, + copy=True, + ) + values[~np.isfinite(values)] = np.nan if null_fill is None else null_fill + out[time_id, asset_id] = values + return out + + +def _timestamp_int64(series: pl.Series) -> np.ndarray: + if series.dtype == pl.Date: + return series.cast(pl.Datetime("ms")).cast(pl.Int64).to_numpy() + if isinstance(series.dtype, pl.Datetime): + return series.cast(pl.Datetime("ms")).cast(pl.Int64).to_numpy() + return series.cast(pl.Int64, strict=False).to_numpy() + + +def _bool_matrix( + panel: pl.DataFrame, + column: str, + shape: tuple[int, int], + time_id: np.ndarray, + asset_id: np.ndarray, +) -> np.ndarray: + out = np.zeros(shape, dtype=np.uint8) + if column in panel.columns: + out[time_id, asset_id] = panel[column].fill_null(False).cast(pl.UInt8).to_numpy() + return out + + +def _tradable_matrix( + open_: np.ndarray, + high: np.ndarray, + low: np.ndarray, + close: np.ndarray, + volume: np.ndarray, +) -> np.ndarray: + present = _present_matrix(open_, high, low, close, volume) + valid_ohlc = present & ((open_ > 0) | (high > 0) | (low > 0) | (close > 0)) + max_price = np.array(open_, dtype=np.float32, copy=True) + min_price = np.array(open_, dtype=np.float32, copy=True) + for values in (high, low, close): + np.fmax(max_price, values, out=max_price) + np.fmin(min_price, values, out=min_price) + spread = max_price - min_price + tolerance = np.maximum(np.abs(close) * np.float32(1e-4), np.float32(0.01)) + suspended_zero_volume = ((volume <= 0) | np.isnan(volume)) & (spread <= tolerance) + return (valid_ohlc & ~suspended_zero_volume).astype(np.uint8) + + +def _present_matrix( + open_: np.ndarray, + high: np.ndarray, + low: np.ndarray, + close: np.ndarray, + volume: np.ndarray, +) -> np.ndarray: + del volume + return ( + np.isfinite(open_) + | np.isfinite(high) + | np.isfinite(low) + | np.isfinite(close) + ) + + +def _normalize_signal(signal: str) -> str: + return signal if signal.startswith(("signal_", "csg_")) else f"signal_{signal}" + + +def _signal_code_matrix( + panel: pl.DataFrame, + signal_ids: list[str] | None, + shape: tuple[int, int], + time_id: np.ndarray, + asset_id: np.ndarray, +) -> tuple[np.ndarray, tuple[str, ...]]: + normalized = tuple(_normalize_signal(signal) for signal in (signal_ids or [])) + row_codes = np.full(len(panel), -1, dtype=np.int16) + for code, column in enumerate(normalized): + if column not in panel.columns: + continue + mask = panel[column].fill_null(False).cast(pl.Boolean).to_numpy() + row_codes[(row_codes < 0) & mask] = code + codes = np.full(shape, -1, dtype=np.int16) + codes[time_id, asset_id] = row_codes + return codes, normalized + + +def _scatter_bool_series( + series: pl.Series | None, + length: int, + shape: tuple[int, int], + time_id: np.ndarray, + asset_id: np.ndarray, +) -> np.ndarray: + out = np.zeros(shape, dtype=np.uint8) + if series is None or len(series) != length: + return out + out[time_id, asset_id] = series.fill_null(False).cast(pl.UInt8).to_numpy() + return out + + +def _delay_signal_matrix( + raw: np.ndarray, + codes: np.ndarray, + present: np.ndarray, + delay_bars: int, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + shape = raw.shape + output = np.zeros(shape, dtype=np.uint8) + signal_time = np.full(shape, -1, dtype=np.int32) + signal_code = np.full(shape, -1, dtype=np.int16) + for asset_id in range(shape[1]): + rows = np.flatnonzero(present[:, asset_id]) + if len(rows) <= delay_bars: + continue + source_rows = rows[: len(rows) - delay_bars] if delay_bars else rows + target_rows = rows[delay_bars:] if delay_bars else rows + active = raw[source_rows, asset_id] != 0 + if not active.any(): + continue + sources = source_rows[active] + targets = target_rows[active] + output[targets, asset_id] = 1 + signal_time[targets, asset_id] = sources.astype(np.int32) + signal_code[targets, asset_id] = codes[sources, asset_id] + return output, signal_time, signal_code + + +def _coerce_array( + value: np.ndarray | None, + shape: tuple[int, int], + dtype: np.dtype | type, + fill: int | float, +) -> np.ndarray: + if value is None: + return np.full(shape, fill, dtype=dtype) + array = np.asarray(value, dtype=dtype) + if array.shape != shape: + raise ValueError(f"matrix shape {array.shape} does not match expected {shape}") + return np.array(array, dtype=dtype, copy=True) + + +def _make_read_only(*arrays: np.ndarray) -> None: + for array in arrays: + array.flags.writeable = False + + +# ============================================================================ +# Shared NumPy feature primitives +# ============================================================================ + + +def shift(values: np.ndarray, periods: int = 1) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + out = np.full(source.shape, np.nan, dtype=np.float32) + if periods == 0: + out[:] = source + elif periods > 0 and periods < source.shape[0]: + out[periods:] = source[:-periods] + elif periods < 0 and -periods < source.shape[0]: + out[:periods] = source[-periods:] + return out + + +def _resolve_valid_bar_index( + source: np.ndarray, + valid: np.ndarray, + bar_index: ValidBarIndex | None, +) -> ValidBarIndex: + if bar_index is not None: + if bar_index.shape != source.shape: + raise ValueError("valid bar index shape does not match values") + return bar_index + active = _ACTIVE_VALID_BAR_INDEX.get() + if isinstance(active, ValidBarIndex) and active.shape == source.shape: + return active + return _build_valid_bar_index(valid) + + +@njit(cache=True, nogil=True, parallel=True) +def _valid_shift_kernel( + source: np.ndarray, + valid: np.ndarray, + offsets: np.ndarray, + rows: np.ndarray, + periods: int, +) -> np.ndarray: + out = np.full(source.shape, np.nan, dtype=np.float32) + distance = abs(periods) + for asset_id in prange(source.shape[1]): + start = int(offsets[asset_id]) + stop = int(offsets[asset_id + 1]) + ring = np.empty(distance, dtype=np.int32) + seen = 0 + if periods > 0: + for position in range(start, stop): + row = int(rows[position]) + if not valid[row, asset_id] or not np.isfinite(source[row, asset_id]): + continue + slot = seen % distance + if seen >= distance: + out[row, asset_id] = source[int(ring[slot]), asset_id] + ring[slot] = row + seen += 1 + else: + for position in range(stop - 1, start - 1, -1): + row = int(rows[position]) + if not valid[row, asset_id] or not np.isfinite(source[row, asset_id]): + continue + slot = seen % distance + if seen >= distance: + out[row, asset_id] = source[int(ring[slot]), asset_id] + ring[slot] = row + seen += 1 + return out + + +def valid_shift( + values: np.ndarray, + periods: int = 1, + valid_mask: np.ndarray | None = None, + *, + bar_index: ValidBarIndex | None = None, +) -> np.ndarray: + """Shift by effective observations, skipping missing market bars. + + The matrix keeps a shared time axis, so a suspended asset can have NaN + rows between two valid bars. Standard per-symbol indicators must treat + those rows as absent rather than as observations. + """ + source = np.asarray(values, dtype=np.float32) + valid = ( + np.isfinite(source) + if valid_mask is None + else np.asarray(valid_mask, dtype=bool) & np.isfinite(source) + ) + if valid.shape != source.shape: + raise ValueError("valid_shift mask shape does not match values") + if periods == 0: + out = np.full(source.shape, np.nan, dtype=np.float32) + out[valid] = source[valid] + return out + index = _resolve_valid_bar_index(source, valid, bar_index) + + return _cached_matrix_operation( + "valid_shift", + (source, valid, index.offsets, index.rows), + {"periods": int(periods)}, + lambda: _valid_shift_kernel( + source, + valid, + index.offsets, + index.rows, + int(periods), + ), + ) + + +def rolling_min(values: np.ndarray, window: int) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + return _cached_matrix_operation( + "rolling_min", + (source,), + {"window": int(window)}, + lambda: _rolling_reduce(source, window, np.min), + ) + + +def rolling_max(values: np.ndarray, window: int) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + return _cached_matrix_operation( + "rolling_max", + (source,), + {"window": int(window)}, + lambda: _rolling_reduce(source, window, np.max), + ) + + +def rolling_mean(values: np.ndarray, window: int) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + return _cached_matrix_operation( + "rolling_mean", + (source,), + {"window": int(window)}, + lambda: _rolling_reduce(source, window, np.mean), + ) + + +def rolling_sum(values: np.ndarray, window: int) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + return _cached_matrix_operation( + "rolling_sum", + (source,), + {"window": int(window)}, + lambda: _rolling_reduce(source, window, np.sum), + ) + + +def rolling_std(values: np.ndarray, window: int, *, ddof: int = 0) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + return _cached_matrix_operation( + "rolling_std", + (source,), + {"window": int(window), "ddof": int(ddof)}, + lambda: _rolling_reduce( + source, + window, + lambda view, axis: np.std(view, axis=axis, ddof=int(ddof)), + materialized_window_budget_bytes=_ROLLING_MATERIALIZED_WINDOW_BUDGET_BYTES, + ), + ) + + +_VALID_REDUCE_MIN = 0 +_VALID_REDUCE_MAX = 1 +_VALID_REDUCE_MEAN = 2 +_VALID_REDUCE_STD = 3 + + +@njit(cache=True, nogil=True, parallel=True) +def _valid_rolling_kernel( + source: np.ndarray, + valid: np.ndarray, + offsets: np.ndarray, + rows: np.ndarray, + window: int, + operation: int, + ddof: int, +) -> np.ndarray: + out = np.full(source.shape, np.nan, dtype=np.float32) + window_value = float(window) + denominator = float(window - ddof) + for asset_id in prange(source.shape[1]): + start = int(offsets[asset_id]) + stop = int(offsets[asset_id + 1]) + ring = np.empty(window, dtype=np.float32) + seen = 0 + for position in range(start, stop): + row = int(rows[position]) + value = source[row, asset_id] + if not valid[row, asset_id] or not np.isfinite(value): + continue + ring[seen % window] = value + seen += 1 + if seen < window: + continue + first = seen - window + if operation == _VALID_REDUCE_MIN: + result = ring[first % window] + for offset in range(1, window): + candidate = ring[(first + offset) % window] + if candidate < result: + result = candidate + out[row, asset_id] = result + elif operation == _VALID_REDUCE_MAX: + result = ring[first % window] + for offset in range(1, window): + candidate = ring[(first + offset) % window] + if candidate > result: + result = candidate + out[row, asset_id] = result + else: + total = 0.0 + for offset in range(window): + total += float(ring[(first + offset) % window]) + mean = total / window_value + if operation == _VALID_REDUCE_MEAN: + out[row, asset_id] = mean + else: + squared = 0.0 + for offset in range(window): + delta = float(ring[(first + offset) % window]) - mean + squared += delta * delta + out[row, asset_id] = np.sqrt(squared / denominator) + return out + + +def _valid_rolling_reduce( + values: np.ndarray, + valid_mask: np.ndarray, + window: int, + operation: int, + *, + ddof: int = 0, + bar_index: ValidBarIndex | None = None, +) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + valid = np.asarray(valid_mask, dtype=bool) + if source.ndim != 2 or valid.shape != source.shape: + raise ValueError("valid rolling inputs must be matching 2D arrays") + if window <= 0: + raise ValueError("valid rolling window must be positive") + if ddof < 0 or ddof >= window: + raise ValueError("valid rolling ddof must be in [0, window)") + index = _resolve_valid_bar_index(source, valid, bar_index) + return _valid_rolling_kernel( + source, + valid, + index.offsets, + index.rows, + int(window), + int(operation), + int(ddof), + ) + + +def valid_rolling_min( + values: np.ndarray, + valid_mask: np.ndarray, + window: int, + *, + bar_index: ValidBarIndex | None = None, +) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + valid = np.asarray(valid_mask, dtype=bool) & np.isfinite(source) + index = _resolve_valid_bar_index(source, valid, bar_index) + return _cached_matrix_operation( + "valid_rolling_min", + (source, valid, index.offsets, index.rows), + {"window": int(window)}, + lambda: _valid_rolling_reduce( + source, + valid, + window, + _VALID_REDUCE_MIN, + bar_index=index, + ), + ) + + +def valid_rolling_max( + values: np.ndarray, + valid_mask: np.ndarray, + window: int, + *, + bar_index: ValidBarIndex | None = None, +) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + valid = np.asarray(valid_mask, dtype=bool) & np.isfinite(source) + index = _resolve_valid_bar_index(source, valid, bar_index) + return _cached_matrix_operation( + "valid_rolling_max", + (source, valid, index.offsets, index.rows), + {"window": int(window)}, + lambda: _valid_rolling_reduce( + source, + valid, + window, + _VALID_REDUCE_MAX, + bar_index=index, + ), + ) + + +def valid_rolling_mean( + values: np.ndarray, + valid_mask: np.ndarray, + window: int, + *, + bar_index: ValidBarIndex | None = None, +) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + valid = np.asarray(valid_mask, dtype=bool) & np.isfinite(source) + index = _resolve_valid_bar_index(source, valid, bar_index) + return _cached_matrix_operation( + "valid_rolling_mean", + (source, valid, index.offsets, index.rows), + {"window": int(window)}, + lambda: _valid_rolling_reduce( + source, + valid, + window, + _VALID_REDUCE_MEAN, + bar_index=index, + ), + ) + + +def valid_rolling_std( + values: np.ndarray, + valid_mask: np.ndarray, + window: int, + *, + ddof: int = 0, + bar_index: ValidBarIndex | None = None, +) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + valid = np.asarray(valid_mask, dtype=bool) & np.isfinite(source) + index = _resolve_valid_bar_index(source, valid, bar_index) + return _cached_matrix_operation( + "valid_rolling_std", + (source, valid, index.offsets, index.rows), + {"window": int(window), "ddof": int(ddof)}, + lambda: _valid_rolling_reduce( + source, + valid, + window, + _VALID_REDUCE_STD, + ddof=int(ddof), + bar_index=index, + ), + ) + + +def rolling_quantile(values: np.ndarray, window: int, quantile: float) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + q = float(quantile) + if not 0.0 <= q <= 1.0: + raise ValueError("rolling quantile must be in [0, 1]") + return _cached_matrix_operation( + "rolling_quantile", + (source,), + {"window": int(window), "quantile": q}, + lambda: _rolling_reduce( + source, + window, + lambda view, axis: np.quantile(view, q, axis=axis), + materialized_window_budget_bytes=_ROLLING_MATERIALIZED_WINDOW_BUDGET_BYTES, + ), + ) + + +def ewm_adjust_false( + values: np.ndarray, + *, + span: int | None = None, + alpha: float | None = None, +) -> np.ndarray: + """Pandas-compatible EWM mean for ``adjust=False, ignore_na=False``.""" + if alpha is None: + if span is None or span <= 0: + raise ValueError("span must be positive when alpha is omitted") + alpha = 2.0 / (float(span) + 1.0) + if not 0.0 < float(alpha) <= 1.0: + raise ValueError("alpha must be in (0, 1]") + + source = np.asarray(values, dtype=np.float32) + alpha_value = float(alpha) + + def _compute() -> np.ndarray: + out = np.full(source.shape, np.nan, dtype=np.float32) + weighted = np.zeros(source.shape[1], dtype=np.float64) + old_weight = np.ones(source.shape[1], dtype=np.float64) + initialized = np.zeros(source.shape[1], dtype=bool) + decay = 1.0 - alpha_value + + for time_id in range(source.shape[0]): + row = source[time_id].astype(np.float64, copy=False) + finite = np.isfinite(row) + continuing = finite & initialized + starting = finite & ~initialized + + old_weight[initialized] *= decay + if continuing.any(): + denominator = old_weight[continuing] + alpha_value + weighted[continuing] = ( + old_weight[continuing] * weighted[continuing] + + alpha_value * row[continuing] + ) / denominator + old_weight[continuing] = 1.0 + if starting.any(): + weighted[starting] = row[starting] + old_weight[starting] = 1.0 + initialized[starting] = True + out[time_id, initialized] = weighted[initialized].astype(np.float32) + return out + + return _cached_matrix_operation( + "ewm_adjust_false", + (source,), + {"alpha": alpha_value}, + _compute, + ) + + +@njit(cache=True, nogil=True, parallel=True) +def _valid_ewm_kernel( + source: np.ndarray, + valid: np.ndarray, + offsets: np.ndarray, + rows: np.ndarray, + alpha: float, +) -> np.ndarray: + out = np.full(source.shape, np.nan, dtype=np.float32) + decay = 1.0 - alpha + for asset_id in prange(source.shape[1]): + start = int(offsets[asset_id]) + stop = int(offsets[asset_id + 1]) + initialized = False + state = 0.0 + for position in range(start, stop): + row = int(rows[position]) + value = source[row, asset_id] + if not valid[row, asset_id] or not np.isfinite(value): + continue + if initialized: + state = decay * state + alpha * float(value) + else: + state = float(value) + initialized = True + out[row, asset_id] = np.float32(state) + return out + + +def valid_ewm_adjust_false( + values: np.ndarray, + valid_mask: np.ndarray, + *, + span: int | None = None, + alpha: float | None = None, + bar_index: ValidBarIndex | None = None, +) -> np.ndarray: + """Pandas-compatible EWM that advances only on effective observations.""" + if alpha is None: + if span is None or span <= 0: + raise ValueError("span must be positive when alpha is omitted") + alpha = 2.0 / (float(span) + 1.0) + if not 0.0 < float(alpha) <= 1.0: + raise ValueError("alpha must be in (0, 1]") + source = np.asarray(values, dtype=np.float32) + valid = np.asarray(valid_mask, dtype=bool) & np.isfinite(source) + if valid.shape != source.shape: + raise ValueError("valid_ewm mask shape does not match values") + alpha_value = float(alpha) + index = _resolve_valid_bar_index(source, valid, bar_index) + + return _cached_matrix_operation( + "valid_ewm_adjust_false", + (source, valid, index.offsets, index.rows), + {"alpha": alpha_value}, + lambda: _valid_ewm_kernel( + source, + valid, + index.offsets, + index.rows, + alpha_value, + ), + ) + + +def safe_divide(numerator: np.ndarray, denominator: np.ndarray) -> np.ndarray: + out = np.full(np.broadcast_shapes(numerator.shape, denominator.shape), np.nan, dtype=np.float32) + np.divide( + numerator, + denominator, + out=out, + where=np.isfinite(denominator) & (denominator != 0), + ) + return out + + +def _rolling_reduce( + values: np.ndarray, + window: int, + reducer: Callable[..., np.ndarray], + *, + asset_chunk_size: int = 256, + materialized_window_budget_bytes: int | None = None, +) -> np.ndarray: + if window <= 0: + raise ValueError("rolling window must be positive") + source = np.asarray(values, dtype=np.float32) + if source.ndim != 2: + raise ValueError("matrix rolling features require a 2D array") + out = np.full(source.shape, np.nan, dtype=np.float32) + if source.shape[0] < window: + return out + + if materialized_window_budget_bytes is not None: + output_rows = source.shape[0] - window + 1 + logical_bytes_per_asset = output_rows * window * source.itemsize + if logical_bytes_per_asset > 0: + budget_chunk_size = max( + 1, + int(materialized_window_budget_bytes) // logical_bytes_per_asset, + ) + asset_chunk_size = min(asset_chunk_size, budget_chunk_size) + + for start in range(0, source.shape[1], asset_chunk_size): + stop = min(start + asset_chunk_size, source.shape[1]) + view = np.lib.stride_tricks.sliding_window_view( + source[:, start:stop], + window_shape=window, + axis=0, + ) + out[window - 1 :, start:stop] = reducer(view, axis=-1).astype(np.float32, copy=False) + return out + + +# ============================================================================ +# Matrix-native strategy protocol and framework-owned pipeline +# ============================================================================ + + +@runtime_checkable +class MatrixStrategy(Protocol): + def required_fields(self) -> frozenset[str]: ... + + def required_warmup_bars(self, params: dict) -> int: ... + + def compute_signals( + self, + market: MarketDataMatrix, + params: dict, + ) -> SignalMatrix: ... + + +@dataclass(frozen=True) +class MatrixPipelineConfig: + basic_filter: dict + scoring: dict[str, float] + order_by: str | None + descending: bool + asset_mask: np.ndarray | None = None + protect_strategy_cache: bool = False + + +class MatrixStrategyPipeline: + """Apply framework-owned filtering and scoring to a matrix strategy output.""" + + def run( + self, + strategy: MatrixStrategy, + market: MarketDataMatrix, + params: dict, + config: MatrixPipelineConfig, + timing_ms: dict[str, float] | None = None, + ) -> SignalMatrix: + with _activate_valid_bar_index(market.valid_bars): + return self._run_with_valid_bars( + strategy, + market, + params, + config, + timing_ms, + ) + + def _run_with_valid_bars( + self, + strategy: MatrixStrategy, + market: MarketDataMatrix, + params: dict, + config: MatrixPipelineConfig, + timing_ms: dict[str, float] | None, + ) -> SignalMatrix: + strategy_started = time.perf_counter() + signals = strategy.compute_signals(market, params) + validate_signal_matrix(signals, market.shape) + if timing_ms is not None: + timing_ms["strategy_signals"] = round( + (time.perf_counter() - strategy_started) * 1000, + 1, + ) + + filter_started = time.perf_counter() + cache = active_matrix_compute_cache() + protect_cache = ( + cache is not None + and config.protect_strategy_cache + and not cache.has_cached_operation("basic_filter_mask") + and cache.current_bytes + _estimate_pipeline_cache_bytes(market, config) + > cache.max_bytes + ) + cache_scope = ( + cache.suspend() + if protect_cache + else nullcontext() + ) + with cache_scope: + basic_mask = build_pipeline_filter_mask(market, config) + entry = (signals.entry.astype(bool) & basic_mask).astype(np.uint8) + score = build_matrix_score( + market, + entry.astype(bool), + config.scoring, + config.order_by, + config.descending, + fallback=signals.score, + ) + entry_codes = np.where(entry != 0, signals.entry_signal_code, -1).astype(np.int16) + exit_codes = np.where(signals.exit != 0, signals.exit_signal_code, -1).astype(np.int16) + if timing_ms is not None: + timing_ms["filter_score"] = round( + (time.perf_counter() - filter_started) * 1000, + 1, + ) + return make_signal_matrix( + market.shape, + entry=entry, + exit=signals.exit, + score=score, + entry_signal_code=entry_codes, + exit_signal_code=exit_codes, + entry_signal_ids=signals.entry_signal_ids, + exit_signal_ids=signals.exit_signal_ids, + ) + + +def _estimate_pipeline_cache_bytes( + market: MarketDataMatrix, + config: MatrixPipelineConfig, +) -> int: + float_bytes = int(market.close.nbytes) + bool_bytes = int(market.shape[0] * market.shape[1]) + estimated = bool_bytes + if config.asset_mask is not None: + estimated += bool_bytes + + feature_names = { + name + for name, weight in config.scoring.items() + if float(weight) != 0.0 + } + if not feature_names and config.order_by and config.order_by != "score": + feature_names.add(str(config.order_by)) + for name in feature_names: + if name in {"open", "high", "low", "close", "volume"} or name in market.fields: + continue + if name == "vol_ratio_5d": + estimated += 2 * float_bytes + elif name == "change_pct" or ( + name.startswith("momentum_") and name.endswith("d") + ): + estimated += float_bytes + return estimated + + +def build_pipeline_filter_mask( + market: MarketDataMatrix, + config: MatrixPipelineConfig, +) -> np.ndarray: + basic_mask = build_basic_filter_mask(market, config.basic_filter) + if config.asset_mask is None: + return basic_mask + + asset_mask = np.asarray(config.asset_mask, dtype=bool) + if asset_mask.shape != (market.shape[1],): + raise ValueError("matrix strategy asset mask length does not match market") + return _cached_matrix_operation( + "pipeline_filter_mask", + (basic_mask, asset_mask), + {}, + lambda: basic_mask & asset_mask[None, :], + ) + + +def build_basic_filter_mask(market: MarketDataMatrix, config: dict) -> np.ndarray: + cache = active_matrix_compute_cache() + if cache is None: + return _build_basic_filter_mask_uncached(market, config) + return cache.get_or_compute( + "basic_filter_mask", + (), + config, + lambda: _build_basic_filter_mask_uncached(market, config), + key_parts=cache.market_token(market), + ) + + +def _build_basic_filter_mask_uncached(market: MarketDataMatrix, config: dict) -> np.ndarray: + if not config or not config.get("enabled", True): + return np.ones(market.shape, dtype=bool) + + mask = np.ones(market.shape, dtype=bool) + close = market.close + if config.get("price_min") is not None: + mask &= close >= float(config["price_min"]) + if config.get("price_max") is not None: + mask &= close <= float(config["price_max"]) + + _apply_bound(mask, close * _optional_field(market, "total_shares"), config, "market_cap") + _apply_bound(mask, close * _optional_field(market, "float_shares"), config, "float_cap") + _apply_bound(mask, _required_field_for_bound(market, config, "amount"), config, "amount") + _apply_bound(mask, _optional_field(market, "turnover_rate"), config, "turnover") + + if config.get("exclude_st"): + asset_mask = np.array( + [ + not any(token in name.upper() for token in ("ST", "*ST", "退")) + for name in market.names + ], + dtype=bool, + ) + mask &= asset_mask[None, :] + + boards = config.get("boards") + if isinstance(boards, list) and boards: + board_mask = np.zeros(len(market.symbols), dtype=bool) + for asset_id, symbol in enumerate(market.symbols): + board_mask[asset_id] = _symbol_in_boards(symbol, boards) + mask &= board_mask[None, :] + return mask + + +def build_matrix_score( + market: MarketDataMatrix, + universe: np.ndarray, + scoring: dict[str, float], + order_by: str | None, + descending: bool, + *, + fallback: np.ndarray, +) -> np.ndarray: + weights = {name: float(weight) for name, weight in scoring.items() if float(weight) != 0.0} + total_weight = sum(weights.values()) + if weights and total_weight > 0: + score = np.zeros(market.shape, dtype=np.float32) + all_finite = universe.copy() + row_count, asset_count = market.shape + chunk_size = min(_SCORE_ASSET_CHUNK_SIZE, asset_count) + finite_scratch = np.empty((row_count, chunk_size), dtype=bool) + work_mask = np.empty((row_count, chunk_size), dtype=bool) + value_scratch = np.empty((row_count, chunk_size), dtype=np.float32) + for name, weight in weights.items(): + values = matrix_feature(market, name) + row_min = np.full(row_count, np.inf, dtype=np.float32) + row_max = np.full(row_count, -np.inf, dtype=np.float32) + for start in range(0, asset_count, chunk_size): + stop = min(start + chunk_size, asset_count) + width = stop - start + finite = finite_scratch[:, :width] + values_chunk = values[:, start:stop] + np.isfinite(values_chunk, out=finite) + all_finite[:, start:stop] &= finite + finite &= universe[:, start:stop] + np.minimum( + row_min, + np.min(values_chunk, axis=1, where=finite, initial=np.inf), + out=row_min, + ) + np.maximum( + row_max, + np.max(values_chunk, axis=1, where=finite, initial=-np.inf), + out=row_max, + ) + row_range = row_max - row_min + varying_rows = np.isfinite(row_range) & (row_range > 0) + normalized_weight = np.float32(weight / total_weight) + for start in range(0, asset_count, chunk_size): + stop = min(start + chunk_size, asset_count) + width = stop - start + finite = finite_scratch[:, :width] + mask = work_mask[:, :width] + scratch = value_scratch[:, :width] + values_chunk = values[:, start:stop] + np.isfinite(values_chunk, out=finite) + finite &= universe[:, start:stop] + scratch.fill(0.0) + np.logical_and(finite, varying_rows[:, None], out=mask) + np.subtract(values_chunk, row_min[:, None], out=scratch, where=mask) + np.divide(scratch, row_range[:, None], out=scratch, where=mask) + np.logical_and(finite, ~varying_rows[:, None], out=mask) + scratch[mask] = np.float32(0.5) + scratch *= normalized_weight + score[:, start:stop] += scratch + score *= np.float32(100.0) + score[~universe | ~all_finite] = 0.0 + return score + + if order_by and order_by != "score": + values = matrix_feature(market, order_by) + result = np.zeros(market.shape, dtype=np.float32) + direction = np.float32(1.0 if descending else -1.0) + for start in range(0, market.shape[1], _SCORE_ASSET_CHUNK_SIZE): + stop = min(start + _SCORE_ASSET_CHUNK_SIZE, market.shape[1]) + values_chunk = values[:, start:stop] + valid = universe[:, start:stop] & np.isfinite(values_chunk) + np.multiply( + values_chunk, + direction, + out=result[:, start:stop], + where=valid, + ) + return result + result = np.zeros(market.shape, dtype=np.float32) + np.copyto(result, fallback, where=universe) + return result + + +def matrix_feature(market: MarketDataMatrix, name: str) -> np.ndarray: + if name in {"open", "high", "low", "close", "volume"} or name in market.fields: + return market.field(name) + close_feature = ( + name in { + "prev_close", + "change_pct", + "change_amount", + "amplitude", + "boll_upper", + "boll_lower", + "high_60d", + "low_60d", + "annual_vol_20d", + } + or (name.startswith("ma") and name[2:].isdigit()) + or (name.startswith("rsi_") and name[4:].isdigit()) + or ( + name.startswith("momentum_") and name.endswith("d") + ) + ) + if close_feature: + source = market.close + elif name == "vol_ratio_5d": + source = market.volume + else: + raise ValueError(f"unsupported matrix feature: {name}") + with _activate_valid_bar_index(market.valid_bars): + return _cached_matrix_operation( + "matrix_feature", + (source,), + {"name": name}, + lambda: _compute_matrix_feature(market, name), + ) + + +def _compute_matrix_feature(market: MarketDataMatrix, name: str) -> np.ndarray: + close_valid = np.isfinite(market.close) + if name == "prev_close": + return valid_shift(market.close, 1, close_valid) + if name == "change_pct": + return _valid_return_over_bars(market.close, close_valid, 1) + if name == "change_amount": + previous = valid_shift(market.close, 1, close_valid) + out = np.full(market.shape, np.nan, dtype=np.float32) + np.subtract(market.close, previous, out=out, where=np.isfinite(previous)) + return out + if name == "amplitude": + previous = valid_shift(market.close, 1, close_valid) + out = np.full(market.shape, np.nan, dtype=np.float32) + np.divide( + market.high - market.low, + previous, + out=out, + where=np.isfinite(previous) & (previous > 0), + ) + return out + if name.startswith("momentum_") and name.endswith("d"): + try: + bars = int(name.removeprefix("momentum_").removesuffix("d")) + except ValueError as exc: + raise ValueError(f"unsupported matrix feature: {name}") from exc + return _valid_return_over_bars(market.close, close_valid, bars) + if name == "vol_ratio_5d": + volume_valid = close_valid & np.isfinite(market.volume) + previous_volume = valid_shift(market.volume, 1, volume_valid) + previous_mean = valid_rolling_mean( + previous_volume, + np.isfinite(previous_volume), + 5, + ) + out = np.full(market.shape, np.nan, dtype=np.float32) + np.divide( + market.volume, + previous_mean, + out=out, + where=volume_valid & np.isfinite(previous_mean) & (previous_mean != 0), + ) + return out + if name.startswith("ma") and name[2:].isdigit(): + return valid_rolling_mean(market.close, close_valid, int(name[2:])) + if name == "boll_upper" or name == "boll_lower": + middle = valid_rolling_mean(market.close, close_valid, 20) + deviation = valid_rolling_std(market.close, close_valid, 20, ddof=1) + offset = np.float32(2.0) * deviation + return middle + offset if name == "boll_upper" else middle - offset + if name == "high_60d": + return valid_rolling_max(market.close, close_valid, 60) + if name == "low_60d": + return valid_rolling_min(market.close, close_valid, 60) + if name == "annual_vol_20d": + daily = _valid_return_over_bars(market.close, close_valid, 1) + return valid_rolling_std( + daily, + np.isfinite(daily), + 20, + ddof=1, + ) * np.float32(252 ** 0.5) + if name.startswith("rsi_") and name[4:].isdigit(): + window = int(name[4:]) + delta = market.close - valid_shift(market.close, 1, close_valid) + delta_valid = close_valid + gain = np.where(delta > 0, delta, 0.0).astype(np.float32, copy=False) + loss = np.where(delta < 0, -delta, 0.0).astype(np.float32, copy=False) + gain[~delta_valid] = np.nan + loss[~delta_valid] = np.nan + average_gain = valid_ewm_adjust_false( + gain, + delta_valid, + alpha=1.0 / window, + ) + average_loss = valid_ewm_adjust_false( + loss, + delta_valid, + alpha=1.0 / window, + ) + denominator = np.where(average_loss == 0, np.float32(1e-12), average_loss) + out = np.full(market.shape, np.nan, dtype=np.float32) + np.divide(average_gain, denominator, out=out, where=np.isfinite(denominator)) + out = np.float32(100.0) - np.float32(100.0) / (np.float32(1.0) + out) + return out + raise ValueError(f"unsupported matrix feature: {name}") + + +def apply_time_masks( + signals: SignalMatrix, + entry_time_mask: np.ndarray, + exit_time_mask: np.ndarray, +) -> SignalMatrix: + if entry_time_mask.shape != (signals.shape[0],) or exit_time_mask.shape != (signals.shape[0],): + raise ValueError("time mask length does not match SignalMatrix") + entry_mask = np.asarray(entry_time_mask, dtype=bool) + exit_mask = np.asarray(exit_time_mask, dtype=bool) + entry = np.array(signals.entry, dtype=np.uint8, copy=True) + exit_ = np.array(signals.exit, dtype=np.uint8, copy=True) + entry[~entry_mask] = 0 + exit_[~exit_mask] = 0 + entry_codes = np.array(signals.entry_signal_code, dtype=np.int16, copy=True) + exit_codes = np.array(signals.exit_signal_code, dtype=np.int16, copy=True) + entry_codes[entry == 0] = -1 + exit_codes[exit_ == 0] = -1 + return _finalize_signal_matrix( + entry, + exit_, + signals.score, + entry_codes, + exit_codes, + entry_signal_ids=signals.entry_signal_ids, + exit_signal_ids=signals.exit_signal_ids, + ) + + +def _valid_return_over_bars( + values: np.ndarray, + valid_mask: np.ndarray, + bars: int, +) -> np.ndarray: + previous = valid_shift(values, bars, valid_mask) + out = np.full(values.shape, np.nan, dtype=np.float32) + np.divide(values, previous, out=out, where=np.isfinite(previous) & (previous != 0)) + out -= np.float32(1.0) + return out + + +def _optional_field(market: MarketDataMatrix, name: str) -> np.ndarray: + values = market.fields.get(name) + if values is None: + return np.full(market.shape, np.nan, dtype=np.float32) + return values + + +def _required_field_for_bound( + market: MarketDataMatrix, + config: dict, + name: str, +) -> np.ndarray: + if config.get(f"{name}_min") is None and config.get(f"{name}_max") is None: + return np.full(market.shape, np.nan, dtype=np.float32) + return market.field(name) + + +def _apply_bound(mask: np.ndarray, values: np.ndarray, config: dict, prefix: str) -> None: + minimum = config.get(f"{prefix}_min") + maximum = config.get(f"{prefix}_max") + if minimum is not None and np.isfinite(values).any(): + mask &= values >= float(minimum) + if maximum is not None and np.isfinite(values).any(): + mask &= values <= float(maximum) + + +def _symbol_in_boards(symbol: str, boards: list[str]) -> bool: + for board in boards: + if board == "沪主板" and symbol.startswith("60"): + return True + if board == "深主板" and symbol.startswith(("00", "001")): + return True + if board == "创业板" and symbol.startswith(("300", "301")): + return True + if board == "科创板" and symbol.startswith("688"): + return True + if board == "北交所" and symbol.endswith(".BJ"): + return True + return False diff --git a/backend/app/backtest/optimizer.py b/backend/app/backtest/optimizer.py index fdfbbc9..51323d5 100644 --- a/backend/app/backtest/optimizer.py +++ b/backend/app/backtest/optimizer.py @@ -3,7 +3,7 @@ 给定策略 + 参数网格, 遍历所有参数组合各跑一次回测, 按目标指标排序, 返回最优参数。 - 参数网格校验对齐 StrategyDef.meta["params"] (类型/范围/选项)。 -- 多线程并行执行, 复用 PanelCache: 同一 symbols/日期的面板只加载一次, 其余组合命中缓存。 +- 单个优化任务在一个 worker 内串行执行; matrix_native 策略共享一份 MarketDataMatrix。 - 支持进度回调 (第 i/N 组完成) 与取消。 """ from __future__ import annotations @@ -12,9 +12,9 @@ import itertools import logging import threading import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field, is_dataclass from datetime import date +from typing import Protocol logger = logging.getLogger(__name__) @@ -190,10 +190,19 @@ class OptimizeConfig: base_params: dict = field(default_factory=dict) # 不扫的固定策略参数 overrides: dict | None = None backtest_kwargs: dict = field(default_factory=dict) # matching/fees/mode/initial_capital 等 + matrix_cache_max_mb: int = 512 + + +class PhaseRssSampler(Protocol): + """Resource probe supplied by the outer worker process.""" + + def reset_phase(self) -> None: ... + + def phase_peak_rss_bytes(self) -> int: ... class StrategyOptimizer: - """遍历参数组合并行回测, 按目标排序。""" + """在单 worker 内遍历参数组合, 并按目标排序。""" def __init__(self, service, strategy_engine) -> None: self.service = service @@ -204,14 +213,19 @@ class StrategyOptimizer: cfg: OptimizeConfig, progress_cb=None, cancel_event: threading.Event | None = None, + *, + rss_sampler: PhaseRssSampler | None = None, + prepared_market_data=None, ) -> dict: - from app.backtest.strategy import StrategyBacktestConfig + from app.backtest.strategy import BacktestResultPolicy, StrategyBacktestConfig t0 = time.perf_counter() if cfg.objective not in VALID_OBJECTIVES: raise ValueError(f"不支持的优化目标 '{cfg.objective}', 可选: {sorted(VALID_OBJECTIVES)}") direction = cfg.direction or default_direction(cfg.objective) _validate_backtest_kwargs(cfg.backtest_kwargs) + if int(cfg.matrix_cache_max_mb) <= 0: + raise ValueError("matrix_cache_max_mb 必须为正整数") s = self.strategy_engine.get(cfg.strategy_id) # 可能抛 ValueError params_meta = s.meta.get("params", []) @@ -219,36 +233,69 @@ class StrategyOptimizer: n_total = len(combos) results: list[dict] = [] - done = 0 - lock = threading.Lock() + backtest_configs = [ + StrategyBacktestConfig( + strategy_id=cfg.strategy_id, + symbols=cfg.symbols, + start=cfg.start, + end=cfg.end, + params={**cfg.base_params, **combo}, + overrides=cfg.overrides, + **cfg.backtest_kwargs, + ) + for combo in combos + ] - def _run_one(idx: int, combo: dict) -> dict | None: + prepared = None + prepare_ms = 0.0 + trials_ms = 0.0 + final_backtest_ms = 0.0 + trial_peak_rss_bytes = None + final_backtest_peak_rss_bytes = None + cache_summary = None + output: dict = {} + trial_policy = BacktestResultPolicy.optimizer_trial(cfg.objective) + + def _run_one(combo: dict, bt_cfg: StrategyBacktestConfig) -> dict | None: if cancel_event is not None and cancel_event.is_set(): return None - # 单组异常必须隔离: 加了并行后, 一组抛异常若冒泡会拖垮整批 (丢弃全部已完成结果)。 + # 单组异常必须隔离: 一组失败不能丢弃全部已完成结果。 try: - merged = {**cfg.base_params, **combo} - bt_cfg = StrategyBacktestConfig( - strategy_id=cfg.strategy_id, - symbols=cfg.symbols, - start=cfg.start, - end=cfg.end, - params=merged, - overrides=cfg.overrides, - **cfg.backtest_kwargs, - ) - res = self.service.run(bt_cfg, cancel_event=cancel_event) + if prepared is None: + res = self.service.run( + bt_cfg, + cancel_event=cancel_event, + result_policy=trial_policy, + ) + else: + res = self.service.run( + bt_cfg, + cancel_event=cancel_event, + prepared=prepared, + result_policy=trial_policy, + ) except Exception as e: # 隔离单组失败, 记录后继续, 不拖垮整批 logger.warning("参数组 %s 回测异常: %r", combo, e) return {"params": combo, "error": repr(e), "objective_raw": None, "_sort": float("-inf")} if res.error: return {"params": combo, "error": res.error, "objective_raw": None, "_sort": float("-inf")} + if cfg.objective not in res.stats: + return { + "params": combo, + "error": f"回测结果缺少优化目标字段 '{cfg.objective}'", + "objective_raw": None, + "_sort": float("-inf"), + } # _sort: 内部排序键 (统一"越大越好"); objective_raw: 原始展示值 (不受方向取负污染)。 + objective_started = time.perf_counter() + sort_value = objective_value(res.stats, cfg.objective, direction) + objective_ms = round((time.perf_counter() - objective_started) * 1000, 3) return { "params": combo, "objective_raw": res.stats.get(cfg.objective), - "_sort": objective_value(res.stats, cfg.objective, direction), + "_sort": sort_value, "stats": res.stats, + "objective_evaluation_ms": objective_ms, } def _best_raw() -> float | None: @@ -257,39 +304,138 @@ class StrategyOptimizer: top = max(results, key=lambda x: x["_sort"]) return None if top["_sort"] == float("-inf") else top.get("objective_raw") - max_workers = max(1, min(int(cfg.max_workers), n_total)) - with ThreadPoolExecutor(max_workers=max_workers) as pool: - futures = {pool.submit(_run_one, i, c): i for i, c in enumerate(combos)} - for fut in as_completed(futures): - r = fut.result() # _run_one 内部已兜底, 不会 re-raise 业务异常 - with lock: - done += 1 - if r is not None: - results.append(r) - if progress_cb is not None: - br = _best_raw() - progress_cb({ - "type": "optimizer_progress", - "done": done, - "total": n_total, - "best_score": round(br, 4) if br is not None else None, - }) + try: + cancelled_before_prepare = cancel_event is not None and cancel_event.is_set() + if ( + getattr(s, "execution_backend", "polars_expr") == "matrix_native" + and not cancelled_before_prepare + ): + prepare_started = time.perf_counter() + prepare_kwargs = { + "matrix_cache_max_bytes": int(cfg.matrix_cache_max_mb) * 1024 * 1024, + } + if prepared_market_data is not None: + prepare_kwargs["market_data_override"] = prepared_market_data + prepared = self.service.prepare_matrix_optimization( + backtest_configs, + **prepare_kwargs, + ) + prepare_ms = round((time.perf_counter() - prepare_started) * 1000, 1) + if progress_cb is not None: + progress_cb({ + "type": "optimizer_prepare", + "done": 0, + "total": n_total, + "best_score": None, + "shared_matrix_bytes": prepared.market_data.nbytes, + "elapsed_ms": prepare_ms, + }) - # 排序: 内部 _sort 降序 (越大越好); -inf (失败/无效) 沉底。展示层用 objective_raw。 - ranked = sorted(results, key=lambda x: x["_sort"], reverse=True) - for i, r in enumerate(ranked): - r["rank"] = i + 1 - r.pop("_sort", None) # 不外露内部排序键, 避免展示层误用取负值 + trials_started = time.perf_counter() + if rss_sampler is not None: + rss_sampler.reset_phase() + for done, (combo, bt_cfg) in enumerate( + zip(combos, backtest_configs, strict=True), + start=1, + ): + r = _run_one(combo, bt_cfg) + if r is not None: + results.append(r) + if progress_cb is not None: + br = _best_raw() + progress_cb({ + "type": "optimizer_progress", + "done": done, + "total": n_total, + "best_score": round(br, 4) if br is not None else None, + }) + if cancel_event is not None and cancel_event.is_set(): + break + trials_ms = round((time.perf_counter() - trials_started) * 1000, 1) + if rss_sampler is not None: + trial_peak_rss_bytes = rss_sampler.phase_peak_rss_bytes() - best = ranked[0] if ranked and ranked[0].get("objective_raw") is not None else None - best_raw = best["objective_raw"] if best else None - return { - "objective": cfg.objective, - "direction": direction, - "n_combinations": n_total, - "n_completed": len(results), - "best_params": best["params"] if best else None, - "best_score": round(best_raw, 4) if best_raw is not None else None, - "results": ranked, - "elapsed_ms": round((time.perf_counter() - t0) * 1000, 1), - } + ranked = sorted(results, key=lambda x: x["_sort"], reverse=True) + for i, result_row in enumerate(ranked): + result_row["rank"] = i + 1 + result_row.pop("_sort", None) + + best = ranked[0] if ranked and ranked[0].get("objective_raw") is not None else None + best_raw = best["objective_raw"] if best else None + best_backtest = None + if best is not None and not (cancel_event is not None and cancel_event.is_set()): + if progress_cb is not None: + progress_cb({ + "type": "optimizer_finalize", + "done": len(results), + "total": n_total, + "best_score": round(best_raw, 4) if best_raw is not None else None, + }) + best_config = StrategyBacktestConfig( + strategy_id=cfg.strategy_id, + symbols=cfg.symbols, + start=cfg.start, + end=cfg.end, + params={**cfg.base_params, **best["params"]}, + overrides=cfg.overrides, + **cfg.backtest_kwargs, + ) + final_started = time.perf_counter() + if rss_sampler is not None: + rss_sampler.reset_phase() + final_result = self.service.run( + best_config, + cancel_event=cancel_event, + prepared=prepared, + ) + final_backtest_ms = round((time.perf_counter() - final_started) * 1000, 1) + if rss_sampler is not None: + final_backtest_peak_rss_bytes = rss_sampler.phase_peak_rss_bytes() + best_backtest = asdict(final_result) if is_dataclass(final_result) else dict(final_result) + + trials_per_second = ( + round(len(results) / (trials_ms / 1000.0), 4) + if trials_ms > 0 + else 0.0 + ) + output = { + "objective": cfg.objective, + "direction": direction, + "n_combinations": n_total, + "n_completed": len(results), + "best_params": best["params"] if best else None, + "best_score": round(best_raw, 4) if best_raw is not None else None, + "best_backtest": best_backtest, + "results": ranked, + "requested_max_workers": int(cfg.max_workers), + "effective_workers": 1, + "shared_market_data": prepared is not None, + "shared_market_data_bytes": prepared.market_data.nbytes if prepared is not None else 0, + "prepare_ms": prepare_ms, + "timing_ms": { + "prepare": prepare_ms, + "trials": trials_ms, + "best_backtest": final_backtest_ms, + }, + "performance": { + "mode": "serial", + "trials_per_second": trials_per_second, + "trial_peak_rss_bytes": trial_peak_rss_bytes, + "best_backtest_peak_rss_bytes": final_backtest_peak_rss_bytes, + "parallel_evaluated": False, + }, + } + finally: + if prepared is not None: + try: + cache_summary = prepared.compute_cache.snapshot() + finally: + prepared.compute_cache.close() + + if cache_summary is not None: + cache_summary["released"] = True + cache_summary["current_bytes_after_close"] = 0 + output["matrix_compute_cache"] = cache_summary + output["elapsed_ms"] = round((time.perf_counter() - t0) * 1000, 1) + output["timing_ms"]["total"] = output["elapsed_ms"] + return output diff --git a/backend/app/backtest/strategy.py b/backend/app/backtest/strategy.py index 1ec1ddc..9817f56 100644 --- a/backend/app/backtest/strategy.py +++ b/backend/app/backtest/strategy.py @@ -4,22 +4,446 @@ """ from __future__ import annotations +import hashlib +import json import logging +import threading import time import uuid +from collections.abc import Callable, Mapping from dataclasses import dataclass, field from datetime import date, timedelta -from typing import Callable, Literal +from typing import Literal import numpy as np import polars as pl -from app.backtest.engine import BacktestEngine, MatcherConfig, SimResult -from app.strategy.engine import StrategyEngine, StrategyDef +from app.backtest.engine import BacktestEngine, MatcherConfig, SimResult, SimulationOptions +from app.backtest.matrix import ( + MarketDataMatrix, + MatrixCacheProfile, + MatrixComputeCache, + MatrixPipelineConfig, + MatrixStrategyPipeline, + apply_time_masks, + build_market_matrix, + build_market_matrix_from_signals, + rolling_mean, + slice_market_data_matrix, + slice_signal_matrix, +) +from app.config import settings +from app.indicators.pipeline import ( + ENRICHED_STORAGE_COLS, + INDICATOR_COLUMNS, + LIMIT_SIGNAL_OUTPUTS, + get_signal_dependencies, +) +from app.strategy.engine import StrategyDataContext, StrategyDef, StrategyEngine logger = logging.getLogger(__name__) BENCHMARK_SYMBOL = "000001.SH" +_EXECUTION_COLUMNS = frozenset({ + "symbol", "date", "open", "high", "low", "close", "volume", + "name", "score", "signal_limit_up", "signal_limit_down", +}) +_LIMIT_BASE_COLUMNS = frozenset({"raw_close", "raw_high"}) +_INSTRUMENT_COLUMNS = frozenset({"name", "total_shares", "float_shares"}) + + +@dataclass(frozen=True) +class FeaturePlan: + required_features: frozenset[str] + required_signals: frozenset[str] + warmup_bars: int + + +@dataclass(frozen=True) +class ResolvedFeaturePlan: + base_columns: frozenset[str] + intermediate_columns: frozenset[str] + indicator_columns: frozenset[str] + signal_columns: frozenset[str] + matrix_columns: frozenset[str] + instrument_columns: frozenset[str] + warmup_bars: int + full_feature_fallback: bool = False + execution_backend: str = "polars_expr" + + +def _merge_resolved_feature_plans( + plans: list[ResolvedFeaturePlan], +) -> ResolvedFeaturePlan: + if not plans: + raise ValueError("cannot merge an empty feature plan list") + backends = {plan.execution_backend for plan in plans} + if backends != {"matrix_native"}: + raise ValueError("shared MarketDataMatrix preparation only supports matrix_native") + + def _union(field: str) -> frozenset[str]: + merged: set[str] = set() + for plan in plans: + merged.update(getattr(plan, field)) + return frozenset(merged) + + return ResolvedFeaturePlan( + base_columns=_union("base_columns"), + intermediate_columns=_union("intermediate_columns"), + indicator_columns=_union("indicator_columns"), + signal_columns=_union("signal_columns"), + matrix_columns=_union("matrix_columns"), + instrument_columns=_union("instrument_columns"), + warmup_bars=max(plan.warmup_bars for plan in plans), + full_feature_fallback=any(plan.full_feature_fallback for plan in plans), + execution_backend="matrix_native", + ) + + +class StrategyDependencyResolver: + """Resolve all backtest field dependencies once before loading market data.""" + + def resolve( + self, + strategy: StrategyDef, + *, + params: dict, + basic_filter: dict, + entry_signals: list[str], + exit_signals: list[str], + overrides: dict | None = None, + minute_fill: bool = False, + ) -> ResolvedFeaturePlan: + overrides = overrides or {} + if strategy.execution_backend == "matrix_native": + return self._resolve_matrix_native( + strategy, + params=params, + basic_filter=basic_filter, + overrides=overrides, + ) + + required_features = set(strategy.required_features) + required_signals = { + _normalize_signal_name(signal) + for signal in [*entry_signals, *exit_signals] + if signal + } + required_signals.update({"signal_limit_up", "signal_limit_down"}) + + scoring = dict(strategy.meta.get("scoring", {}) or {}) + scoring.update(overrides.get("scoring") or {}) + required_features.update(str(column) for column, weight in scoring.items() if weight) + order_by = strategy.meta.get("order_by") + if order_by and order_by != "score": + required_features.add(str(order_by)) + + required_features.update(_basic_filter_dependencies(basic_filter)) + filter_features, filter_resolved = _filter_dependencies(strategy, params) + required_features.update(filter_features) + embedded_signals = { + feature + for feature in required_features + if feature.startswith(("signal_", "csg_")) + } + required_signals.update(embedded_signals) + required_features.difference_update(embedded_signals) + + full_fallback = bool(strategy.filter_history_fn and not strategy.required_features) + full_fallback = full_fallback or not filter_resolved + signal_dependencies = get_signal_dependencies() + if full_fallback: + logger.warning( + "strategy %s has dynamic Python dependencies without REQUIRED_FEATURES; " + "backtest falls back to full feature computation", + strategy.meta.get("id", ""), + ) + required_features.update(INDICATOR_COLUMNS) + required_signals.update(signal_dependencies) + required_signals.update(LIMIT_SIGNAL_OUTPUTS) + + unknown_signals = required_signals - set(signal_dependencies) - set(LIMIT_SIGNAL_OUTPUTS) + if unknown_signals: + raise ValueError(f"策略引用了不存在的信号: {sorted(unknown_signals)}") + for signal in required_signals: + required_features.update(signal_dependencies.get(signal, ())) + + indicator_columns = frozenset(required_features & set(INDICATOR_COLUMNS)) + base_columns = _resolve_base_columns(required_features | set(_EXECUTION_COLUMNS)) + if required_signals & set(LIMIT_SIGNAL_OUTPUTS): + base_columns = frozenset(set(base_columns) | set(_LIMIT_BASE_COLUMNS)) + + instrument_columns = frozenset(required_features & set(_INSTRUMENT_COLUMNS)) + instrument_columns = frozenset(set(instrument_columns) | {"name"}) + matrix_columns = set(_EXECUTION_COLUMNS) | required_signals + if minute_fill: + indicator_columns = frozenset(set(indicator_columns) | {"ma5", "ma10", "ma20"}) + matrix_columns.update({"ma5", "ma10", "ma20"}) + base_columns = frozenset(set(base_columns) | {"close"}) + + plan = FeaturePlan( + required_features=frozenset(required_features), + required_signals=frozenset(required_signals), + warmup_bars=max(60, int(strategy.lookback_days or 1)), + ) + return ResolvedFeaturePlan( + base_columns=base_columns, + intermediate_columns=frozenset(), + indicator_columns=indicator_columns, + signal_columns=plan.required_signals, + matrix_columns=frozenset(matrix_columns), + instrument_columns=instrument_columns, + warmup_bars=plan.warmup_bars, + full_feature_fallback=full_fallback, + execution_backend=strategy.execution_backend, + ) + + @staticmethod + def _resolve_matrix_native( + strategy: StrategyDef, + *, + params: dict, + basic_filter: dict, + overrides: dict, + ) -> ResolvedFeaturePlan: + if strategy.matrix_strategy is None: + raise ValueError( + f"matrix_native strategy {strategy.meta.get('id', '')} " + "must declare MATRIX_STRATEGY" + ) + + required_features = set(strategy.required_features) + required_features.update(strategy.matrix_strategy.required_fields()) + required_features.update(_basic_filter_dependencies(basic_filter)) + scoring = dict(strategy.meta.get("scoring", {}) or {}) + scoring.update(overrides.get("scoring") or {}) + required_features.update(str(name) for name, weight in scoring.items() if weight) + order_by = strategy.meta.get("order_by") + if order_by and order_by != "score": + required_features.add(str(order_by)) + + base_columns = _resolve_base_columns(required_features | set(_EXECUTION_COLUMNS)) + base_columns = frozenset(set(base_columns) | set(_LIMIT_BASE_COLUMNS)) + instrument_columns = frozenset(required_features & set(_INSTRUMENT_COLUMNS)) + instrument_columns = frozenset(set(instrument_columns) | {"name"}) + warmup_bars = max(60, int(strategy.matrix_strategy.required_warmup_bars(params))) + matrix_columns = set(base_columns) | set(instrument_columns) | { + "signal_limit_up", + "signal_limit_down", + } + return ResolvedFeaturePlan( + base_columns=base_columns, + intermediate_columns=frozenset(), + indicator_columns=frozenset(), + signal_columns=frozenset({"signal_limit_up", "signal_limit_down"}), + matrix_columns=frozenset(matrix_columns), + instrument_columns=instrument_columns, + warmup_bars=warmup_bars, + full_feature_fallback=False, + execution_backend="matrix_native", + ) + + +def build_matrix_cache_profile( + strategy_engine: StrategyEngine, + asset_type: str, + *, + requested_plan: ResolvedFeaturePlan | None = None, + requested_forward_bars: int = 0, + max_disk_bytes: int = 512 * 1024 * 1024, +) -> MatrixCacheProfile: + """Merge registered matrix dependencies into one strategy-agnostic cache profile.""" + resolver = StrategyDependencyResolver() + plans: list[ResolvedFeaturePlan] = [] + if requested_plan is not None: + plans.append(requested_plan) + forward_bars = max(0, int(requested_forward_bars)) + common_filter = { + "enabled": True, + "amount_min": 0.0, + "turnover_min": 0.0, + "market_cap_min": 0.0, + "float_cap_min": 0.0, + "exclude_st": True, + } + definitions = ( + strategy_engine.strategy_definitions() + if hasattr(strategy_engine, "strategy_definitions") + else () + ) + for strategy in definitions: + if strategy.execution_backend != "matrix_native": + continue + if asset_type not in strategy.meta.get("asset_types", ["stock"]): + continue + if "1d" not in strategy.meta.get("timeframes", ["1d"]): + continue + params = StrategyEngine.resolve_params(strategy) + for item in strategy.meta.get("params", []): + if not isinstance(item, dict) or not item.get("id"): + continue + if item.get("type") in {"int", "float"} and item.get("max") is not None: + params[str(item["id"])] = item["max"] + plans.append(resolver.resolve( + strategy, + params=params, + basic_filter={**dict(strategy.basic_filter or {}), **common_filter}, + entry_signals=strategy.entry_signals, + exit_signals=strategy.exit_signals, + overrides={}, + minute_fill=False, + )) + forward_bars = max(forward_bars, int(strategy.max_hold_days or 0)) + + if not plans: + raise ValueError(f"no matrix-native cache profile available for asset_type={asset_type!r}") + merged = _merge_resolved_feature_plans(plans) + fields = frozenset( + set(merged.base_columns) + | set(merged.instrument_columns) + | set(merged.matrix_columns) + ) + generation_payload = json.dumps( + { + "asset_type": asset_type, + "fields": sorted(fields), + "warmup_bars": merged.warmup_bars, + "forward_bars": forward_bars, + }, + sort_keys=True, + separators=(",", ":"), + ) + generation = hashlib.blake2b( + generation_payload.encode("utf-8"), + digest_size=12, + ).hexdigest() + return MatrixCacheProfile( + field_columns=fields, + warmup_bars=merged.warmup_bars, + forward_bars=forward_bars, + max_disk_bytes=int(max_disk_bytes), + generation=generation, + ) + + +def prewarm_matrix_cache( + engine: BacktestEngine, + strategy_engine: StrategyEngine, + *, + asset_type: str, + latest_date: date, + years: int = 5, +) -> dict[str, object]: + """Build the shared full-universe mmap outside a user backtest request.""" + if years <= 0: + raise ValueError("matrix cache prewarm years must be positive") + profile = build_matrix_cache_profile( + strategy_engine, + asset_type, + max_disk_bytes=settings.backtest_matrix_cache_max_mb * 1024 * 1024, + ) + formal_start = date(max(1, latest_date.year - years + 1), 1, 1) + warmup_days = max(120, int(max(profile.warmup_bars, 1) * 1.6)) + coverage_start = formal_start - timedelta(days=warmup_days) + prewarm_columns = frozenset({ + "symbol", + "date", + "open", + "high", + "low", + "close", + "volume", + "raw_close", + "raw_high", + }) + plan = ResolvedFeaturePlan( + base_columns=prewarm_columns, + intermediate_columns=frozenset(), + indicator_columns=frozenset(), + signal_columns=frozenset(), + matrix_columns=prewarm_columns, + instrument_columns=frozenset({"name"}), + warmup_bars=profile.warmup_bars, + full_feature_fallback=False, + execution_backend="matrix_native", + ) + started = time.perf_counter() + market = engine.load_market_data_matrix_for_backtest( + None, + coverage_start, + latest_date, + plan, + asset_type=asset_type, + cache_profile=profile, + coverage_start=coverage_start, + coverage_end=latest_date, + ) + result = { + "asset_type": asset_type, + "start": coverage_start.isoformat(), + "end": latest_date.isoformat(), + "cache_status": market.cache_status, + "cache_path": market.cache_path, + "bytes": market.nbytes, + "elapsed_ms": round((time.perf_counter() - started) * 1000, 1), + } + del market + return result + + +def _normalize_signal_name(signal: str) -> str: + if signal.startswith(("signal_", "csg_")): + return signal + return f"signal_{signal}" + + +def _filter_dependencies(strategy: StrategyDef, params: dict) -> tuple[set[str], bool]: + if strategy.filter_history_fn: + return set(strategy.required_features), bool(strategy.required_features) + if not strategy.filter_fn: + return set(), True + try: + expr = strategy.filter_fn(pl.DataFrame(), params) + if expr is None: + return set(), True + return set(expr.meta.root_names()), True + except Exception as exc: + logger.warning("strategy filter dependency resolution failed: %s", exc) + return set(strategy.required_features), bool(strategy.required_features) + + +def _basic_filter_dependencies(config: dict) -> set[str]: + if not config or not config.get("enabled", True): + return set() + dependencies = {"symbol", "close"} + if any(config.get(key) is not None for key in ("amount_min", "amount_max")): + dependencies.add("amount") + if any(config.get(key) is not None for key in ("turnover_min", "turnover_max")): + dependencies.add("turnover_rate") + if any(config.get(key) is not None for key in ("market_cap_min", "market_cap_max")): + dependencies.add("total_shares") + if any(config.get(key) is not None for key in ("float_cap_min", "float_cap_max")): + dependencies.add("float_shares") + if config.get("exclude_st"): + dependencies.add("name") + return dependencies + + +def _resolve_base_columns(features: set[str]) -> frozenset[str]: + storage = set(ENRICHED_STORAGE_COLS) + base = {"symbol", "date"} | (features & storage) + close_indicators = set(INDICATOR_COLUMNS) - { + "atr_14", "amplitude", "kdj_k", "kdj_d", "kdj_j", + "vol_ma5", "vol_ma10", "vol_ratio_5d", + } + if features & close_indicators: + base.add("close") + if features & {"atr_14", "amplitude", "kdj_k", "kdj_d", "kdj_j"}: + base.update({"high", "low", "close"}) + if features & {"vol_ma5", "vol_ma10", "vol_ratio_5d"}: + base.add("volume") + base.update({"open", "high", "low", "close", "volume"}) + return frozenset(base & storage) @dataclass @@ -70,6 +494,84 @@ class StrategyBacktestResult: error: str | None = None +@dataclass(frozen=True) +class BacktestResultPolicy: + """Explicit result contract for full backtests and lightweight optimizer trials.""" + + required_stats: frozenset[str] | None = None + include_monte_carlo: bool = True + include_curves: bool = True + include_trades: bool = True + include_per_symbol_stats: bool = True + include_return_distribution: bool = True + include_benchmark: bool = True + include_strategy_info: bool = True + + @classmethod + def optimizer_trial(cls, objective: str) -> BacktestResultPolicy: + return cls( + required_stats=frozenset({str(objective)}), + include_monte_carlo=str(objective).startswith("mc_maxdd_"), + include_curves=False, + include_trades=False, + include_per_symbol_stats=False, + include_return_distribution=False, + include_benchmark=False, + include_strategy_info=False, + ) + + def simulation_options(self) -> SimulationOptions: + return SimulationOptions( + include_monte_carlo=self.include_monte_carlo, + include_curves=self.include_curves, + include_trades=self.include_trades, + include_per_symbol_stats=self.include_per_symbol_stats, + include_return_distribution=self.include_return_distribution, + ) + + def select_stats(self, stats: dict) -> dict: + if self.required_stats is None: + return stats + diagnostic = { + "error", + "timing_ms", + "execution", + "execution_backend", + "shared_market_data", + "shared_market_data_bytes", + "shared_prepare_timing_ms", + "matrix_data_cache_hit", + "matrix_compute_cache", + "market_matrix_shape", + "market_matrix_bytes", + "panel_rows", + "panel_columns", + "feature_columns", + "full_feature_fallback", + } + keep = set(self.required_stats) | diagnostic + return {key: value for key, value in stats.items() if key in keep} + + +@dataclass(frozen=True) +class PreparedMatrixBacktest: + """Job-scoped immutable market data reused by every optimizer trial.""" + + signature: tuple + market_data: MarketDataMatrix + feature_width: int + load_start: date + load_end: date + sim_end: date + entry_time_mask: np.ndarray + exit_time_mask: np.ndarray + start_id: int + stop_id: int + reference_price: np.ndarray | None + prepare_timing_ms: dict[str, float] + compute_cache: MatrixComputeCache + + class StrategyBacktestService: def __init__( self, @@ -79,14 +581,195 @@ class StrategyBacktestService: self.engine = engine self.strategy_engine = strategy_engine + @staticmethod + def _matrix_prepare_signature(config: StrategyBacktestConfig) -> tuple: + return ( + config.strategy_id, + None if config.symbols is None else tuple(config.symbols), + config.start, + config.end, + config.mode, + config.asset_type, + config.holding_days, + config.minute_fill, + json.dumps(config.overrides or {}, sort_keys=True, ensure_ascii=False, default=str), + ) + + def prepare_matrix_optimization( + self, + configs: list[StrategyBacktestConfig], + *, + matrix_cache_max_bytes: int = 512 * 1024 * 1024, + market_data_override: MarketDataMatrix | None = None, + ) -> PreparedMatrixBacktest: + """Load and encode one immutable base matrix for all matrix-native trials. + + ``market_data_override`` is an optional shared WF matrix. Fold-local + prepared objects take a read-only time view of it, so the base mmap and + its arrays are not copied while each fold still receives a bounded + history window for strict out-of-sample evaluation. + """ + if not configs: + raise ValueError("optimizer preparation requires at least one backtest config") + + signature = self._matrix_prepare_signature(configs[0]) + if any(self._matrix_prepare_signature(config) != signature for config in configs[1:]): + raise ValueError("optimizer trials must share strategy, universe, range and overrides") + + first = configs[0] + strategy = self.strategy_engine.get(first.strategy_id) + if strategy.execution_backend != "matrix_native": + raise ValueError("shared MarketDataMatrix preparation requires matrix_native strategy") + StrategyEngine.validate_context( + strategy, + StrategyDataContext( + asset_type=first.asset_type, + timeframe="1d", + as_of=first.end, + ), + ) + + overrides = first.overrides or {} + basic_filter = self._effective_basic_filter(strategy, overrides) + entry_signals = self._effective_signals(overrides, "entry_signals", strategy.entry_signals) + exit_signals = self._effective_signals(overrides, "exit_signals", strategy.exit_signals) + resolver = StrategyDependencyResolver() + plans: list[ResolvedFeaturePlan] = [] + for config in configs: + params = self._normalize_params(config.params or {}, strategy) + plans.append(resolver.resolve( + strategy, + params=params, + basic_filter=basic_filter, + entry_signals=entry_signals, + exit_signals=exit_signals, + overrides=overrides, + minute_fill=config.minute_fill, + )) + feature_plan = _merge_resolved_feature_plans(plans) + + max_hold_days = self._override_value(overrides, "max_hold_days", strategy.max_hold_days) + full_horizon_days = max(int(max_hold_days or first.holding_days or 5), 1) + cache_profile = build_matrix_cache_profile( + self.strategy_engine, + first.asset_type, + requested_plan=feature_plan, + requested_forward_bars=full_horizon_days, + max_disk_bytes=settings.backtest_matrix_cache_max_mb * 1024 * 1024, + ) + warmup_days = max(120, int(max(feature_plan.warmup_bars, 1) * 1.6)) + load_start = first.start - timedelta(days=warmup_days) + cache_warmup_days = max(120, int(max(cache_profile.warmup_bars, 1) * 1.6)) + coverage_start = first.start - timedelta(days=cache_warmup_days) + load_end = first.end + coverage_end = first.end + if first.mode == "full": + load_end = first.end + timedelta(days=(full_horizon_days + 5) * 2) + coverage_end = first.end + timedelta(days=(cache_profile.forward_bars + 5) * 2) + sim_end = load_end if first.mode == "full" else first.end + + timing_ms: dict[str, float] = {} + prepare_started = time.perf_counter() + if market_data_override is None: + started = time.perf_counter() + market_data = self.engine.load_market_data_matrix_for_backtest( + first.symbols, + load_start, + load_end, + feature_plan, + asset_type=first.asset_type, + cache_profile=cache_profile, + coverage_start=coverage_start, + coverage_end=coverage_end, + ) + direct_load_ms = round((time.perf_counter() - started) * 1000, 1) + else: + labels = market_data_override.timestamp_labels + visible_ids = np.flatnonzero( + np.fromiter( + ( + str(load_start) <= label[:10] <= str(load_end) + for label in labels + ), + dtype=bool, + count=len(labels), + ) + ) + if visible_ids.size == 0: + raise ValueError("shared WF matrix does not cover the fold window") + market_data = slice_market_data_matrix( + market_data_override, + int(visible_ids[0]), + int(visible_ids[-1]) + 1, + ) + direct_load_ms = 0.0 + timing_ms["load_panel"] = direct_load_ms + timing_ms["market_data_matrix_build"] = 0.0 + timing_ms["market_data_direct_load"] = direct_load_ms + formal_range = self._matrix_date_range_mask( + market_data.timestamp_labels, + first.start, + first.end, + ) + if not formal_range.any(): + raise ValueError("正式回测区间内无数据") + + feature_width = len(feature_plan.matrix_columns) + + entry_time_mask = self._matrix_date_range_mask( + market_data.timestamp_labels, + first.start, + first.end, + ) + exit_time_mask = self._matrix_date_range_mask( + market_data.timestamp_labels, + first.start, + load_end if first.mode == "full" else first.end, + ) + sim_time_mask = self._matrix_date_range_mask( + market_data.timestamp_labels, + first.start, + sim_end, + ) + time_ids = np.flatnonzero(sim_time_mask) + if time_ids.size == 0: + raise ValueError("正式回测区间内无数据") + start_id = int(time_ids[0]) + stop_id = int(time_ids[-1]) + 1 + reference_price = ( + rolling_mean(market_data.close, 5)[start_id:stop_id] + if first.minute_fill + else None + ) + timing_ms["total"] = round((time.perf_counter() - prepare_started) * 1000, 1) + compute_cache = MatrixComputeCache(max_bytes=matrix_cache_max_bytes) + return PreparedMatrixBacktest( + signature=signature, + market_data=market_data, + feature_width=feature_width, + load_start=load_start, + load_end=load_end, + sim_end=sim_end, + entry_time_mask=entry_time_mask, + exit_time_mask=exit_time_mask, + start_id=start_id, + stop_id=stop_id, + reference_price=reference_price, + prepare_timing_ms=timing_ms, + compute_cache=compute_cache, + ) + def run( self, config: StrategyBacktestConfig, - progress_cb: "Callable[[dict], None] | None" = None, - cancel_event: "threading.Event | None" = None, + progress_cb: Callable[[dict], None] | None = None, + cancel_event: threading.Event | None = None, + prepared: PreparedMatrixBacktest | None = None, + result_policy: BacktestResultPolicy | None = None, ) -> StrategyBacktestResult: t0 = time.perf_counter() run_id = uuid.uuid4().hex[:10] + result_policy = result_policy or BacktestResultPolicy() def _err(msg: str) -> StrategyBacktestResult: return StrategyBacktestResult( @@ -99,6 +782,14 @@ class StrategyBacktestService: # 获取策略定义 try: s = self.strategy_engine.get(config.strategy_id) + StrategyEngine.validate_context( + s, + StrategyDataContext( + asset_type=config.asset_type, + timeframe="1d", + as_of=config.end, + ), + ) except ValueError as e: return _err(str(e)) @@ -136,10 +827,26 @@ class StrategyBacktestService: overrides.get("score_max"), ) - timing_ms: dict[str, float] = {} + try: + feature_plan = StrategyDependencyResolver().resolve( + s, + params=params, + basic_filter=basic_filter, + entry_signals=entry_signals, + exit_signals=exit_signals, + overrides=overrides, + minute_fill=config.minute_fill, + ) + except ValueError as e: + return _err(str(e)) - # 加载面板 (含 warmup + 全量指标 + 信号)。warmup 只用于指标/形态计算, 不参与正式交易。 - warmup_days = max(120, int(max(s.lookback_days or 1, 1) * 1.5)) + timing_ms: dict[str, float] = {} + matrix_data_cache_hit = False + matrix_data_cache_status = "none" + matrix_data_cache_timing_ms: Mapping[str, float] = {} + + # 加载 warmup + 正式区间。矩阵策略的 warmup 由协议解析，不再依赖策略名称。 + warmup_days = max(120, int(max(feature_plan.warmup_bars, 1) * 1.6)) load_start = config.start - timedelta(days=warmup_days) # 全量模式: entries 只在正式区间触发, exits 需要 end 之后的尾部数据继续执行策略卖点。 @@ -151,53 +858,97 @@ class StrategyBacktestService: fwd_buffer = full_horizon_days + 5 # 多取几天, 容错停牌缺口/open_t+1 load_end = config.end + timedelta(days=fwd_buffer * 2) # 日历日放宽, 确保覆盖 N 个交易日 - t_load = time.perf_counter() - panel = self.engine.load_panel(config.symbols, load_start, load_end, asset_type=config.asset_type) - timing_ms["load_panel"] = round((time.perf_counter() - t_load) * 1000, 1) - if panel.is_empty(): - return _err("无数据，请检查日期范围或先运行盘后管道") - - formal_range = self._date_range_mask(panel, config.start, config.end) - if not formal_range.any(): - return _err("正式回测区间内无数据") - - t_signal = time.perf_counter() - - # basic_filter 只影响买入候选, 不能删除行情 panel, 否则持仓 mark / 卖出 / full forward return 都会失真。 - basic_mask = pl.Series("_basic", [True] * len(panel), dtype=pl.Boolean) - if basic_filter and basic_filter.get("enabled", True): - expr = StrategyEngine._basic_filter_expr(panel, basic_filter) - if expr is not None: - try: - basic_mask = panel.select(expr.alias("_basic"))["_basic"].fill_null(False).cast(pl.Boolean) - except Exception as e: # noqa: BLE001 - logger.warning("basic_filter mask failed: %s", e) - return _err(f"基础过滤计算失败: {e}") - - # 策略候选层用于评分归一化；entry_signals 只是买点层, 不参与 score universe。 - candidate_filter_mask = self._build_candidate_filter_mask(panel, s, params) - candidate_mask = basic_mask & candidate_filter_mask - panel = self._apply_score(panel, s, overrides, universe_mask=candidate_mask) - - entry_mask = self._build_entry_mask_from_candidate(panel, candidate_mask, s, entry_signals) - entry_mask = entry_mask & formal_range - raw_exit_mask = self._build_signal_mask(panel, exit_signals, "_exit") - exit_mask = raw_exit_mask & (self._date_range_mask(panel, config.start, load_end) if config.mode == "full" else formal_range) - timing_ms["signals_score"] = round((time.perf_counter() - t_signal) * 1000, 1) - - if not entry_mask.any(): - return _err("在指定区间内未产生买入信号") - - # warmup 之后才交给撮合；full mode 保留 end 之后前瞻段用于 shift(-N)。 sim_end = load_end if config.mode == "full" else config.end - sim_range = self._date_range_mask(panel, config.start, sim_end) - sim_panel = panel.filter(sim_range) - sim_entry_mask = entry_mask.filter(sim_range) - sim_exit_mask = exit_mask.filter(sim_range) - if sim_panel.is_empty(): - return _err("正式回测区间内无数据") + panel: pl.DataFrame | None = None + formal_range: pl.Series | None = None + market_data: MarketDataMatrix | None = None + if prepared is not None: + if s.execution_backend != "matrix_native": + return _err("共享基础矩阵只能用于 matrix_native 策略") + if prepared.signature != self._matrix_prepare_signature(config): + return _err("共享基础矩阵与当前回测配置不匹配") + load_start = prepared.load_start + load_end = prepared.load_end + sim_end = prepared.sim_end + feature_width = prepared.feature_width + timing_ms["load_panel"] = 0.0 + timing_ms["market_data_matrix_build"] = 0.0 + matrix_data_cache_status = prepared.market_data.cache_status + matrix_data_cache_hit = matrix_data_cache_status in {"exact", "covering"} + matrix_data_cache_timing_ms = prepared.market_data.cache_timing_ms + elif s.execution_backend == "matrix_native": + t_load = time.perf_counter() + max_hold_for_profile = self._override_value( + overrides, + "max_hold_days", + s.max_hold_days, + ) + profile_forward = max(int(max_hold_for_profile or config.holding_days or 5), 1) + cache_profile = build_matrix_cache_profile( + self.strategy_engine, + config.asset_type, + requested_plan=feature_plan, + requested_forward_bars=profile_forward, + max_disk_bytes=settings.backtest_matrix_cache_max_mb * 1024 * 1024, + ) + cache_warmup_days = max( + 120, + int(max(cache_profile.warmup_bars, 1) * 1.6), + ) + coverage_start = config.start - timedelta(days=cache_warmup_days) + coverage_end = config.end + if config.mode == "full": + coverage_end = config.end + timedelta( + days=(cache_profile.forward_bars + 5) * 2 + ) + try: + market_data = self.engine.load_market_data_matrix_for_backtest( + config.symbols, + load_start, + load_end, + feature_plan, + asset_type=config.asset_type, + cache_profile=cache_profile, + coverage_start=coverage_start, + coverage_end=coverage_end, + ) + except (ValueError, OSError) as e: + return _err(f"回测矩阵准备失败: {e}") + direct_load_ms = round((time.perf_counter() - t_load) * 1000, 1) + timing_ms["load_panel"] = direct_load_ms + timing_ms["market_data_matrix_build"] = 0.0 + timing_ms["market_data_direct_load"] = direct_load_ms + matrix_data_cache_status = market_data.cache_status + matrix_data_cache_hit = matrix_data_cache_status in {"exact", "covering"} + matrix_data_cache_timing_ms = market_data.cache_timing_ms + formal_time_mask = self._matrix_date_range_mask( + market_data.timestamp_labels, + config.start, + config.end, + ) + if not formal_time_mask.any(): + return _err("正式回测区间内无数据") + feature_width = len(feature_plan.matrix_columns) + else: + t_load = time.perf_counter() + try: + panel = self.engine.load_panel_for_backtest( + config.symbols, + load_start, + load_end, + feature_plan, + asset_type=config.asset_type, + ) + except (ValueError, pl.exceptions.PolarsError) as e: + return _err(f"回测特征准备失败: {e}") + timing_ms["load_panel"] = round((time.perf_counter() - t_load) * 1000, 1) + if panel.is_empty(): + return _err("无数据，请检查日期范围或先运行盘后管道") + formal_range = self._date_range_mask(panel, config.start, config.end) + if not formal_range.any(): + return _err("正式回测区间内无数据") + feature_width = int(panel.width) - t_sim = time.perf_counter() matcher_config = MatcherConfig( matching=config.matching, entry_fill=config.entry_fill, @@ -220,25 +971,180 @@ class StrategyBacktestService: position_sizing=config.position_sizing, minute_fill=config.minute_fill, ) - # 撮合 — full 为全候选独立执行；position 为账户级仓位模拟。 - if config.mode == "full": - result = self.engine.simulate_independent_candidates( + t_signal = time.perf_counter() + + if s.execution_backend == "matrix_native": + if s.matrix_strategy is None: + return _err("矩阵策略未注册") + if self._has_matrix_signal_override(s, overrides): + return _err("matrix_native 策略的进出场信号由策略协议生成，不支持列信号覆盖") + + if prepared is not None: + market_data = prepared.market_data + entry_time_mask = prepared.entry_time_mask + exit_time_mask = prepared.exit_time_mask + start_id = prepared.start_id + stop_id = prepared.stop_id + reference_price = prepared.reference_price + panel_rows = int(np.isfinite(market_data.close[start_id:stop_id]).sum()) + panel_columns = len(feature_plan.matrix_columns) + else: + if market_data is None: + return _err("矩阵回测缺少基础行情矩阵") + entry_time_mask = self._matrix_date_range_mask( + market_data.timestamp_labels, + config.start, + config.end, + ) + exit_time_mask = self._matrix_date_range_mask( + market_data.timestamp_labels, + config.start, + load_end if config.mode == "full" else config.end, + ) + sim_time_mask = self._matrix_date_range_mask( + market_data.timestamp_labels, + config.start, + sim_end, + ) + time_ids = np.flatnonzero(sim_time_mask) + if time_ids.size == 0: + return _err("正式回测区间内无数据") + start_id = int(time_ids[0]) + stop_id = int(time_ids[-1]) + 1 + panel_rows = int(np.isfinite(market_data.close[start_id:stop_id]).sum()) + panel_columns = len(feature_plan.matrix_columns) + reference_price = ( + rolling_mean(market_data.close, 5)[start_id:stop_id] + if matcher_config.minute_fill + else None + ) + + scoring = dict(s.meta.get("scoring", {}) or {}) + scoring.update(overrides.get("scoring") or {}) + try: + pipeline_config = MatrixPipelineConfig( + basic_filter=basic_filter, + scoring=scoring, + order_by=s.meta.get("order_by"), + descending=bool(s.meta.get("descending", True)), + protect_strategy_cache=prepared is not None, + ) + if prepared is None: + signal_matrix = MatrixStrategyPipeline().run( + s.matrix_strategy, + market_data, + params, + pipeline_config, + timing_ms, + ) + else: + with prepared.compute_cache.activate(market_data): + signal_matrix = MatrixStrategyPipeline().run( + s.matrix_strategy, + market_data, + params, + pipeline_config, + timing_ms, + ) + except (TypeError, ValueError) as e: + return _err(f"矩阵策略信号计算失败: {e}") + + sim_market_data = slice_market_data_matrix(market_data, start_id, stop_id) + sim_signal_matrix = slice_signal_matrix(signal_matrix, start_id, stop_id) + sim_signal_matrix = apply_time_masks( + sim_signal_matrix, + entry_time_mask[start_id:stop_id], + exit_time_mask[start_id:stop_id], + ) + timing_ms["signals_score"] = round((time.perf_counter() - t_signal) * 1000, 1) + if not sim_signal_matrix.entry.any(): + return _err("在指定区间内未产生买入信号") + + raw_candidates = int(sim_signal_matrix.entry.sum()) + del market_data, signal_matrix + + t_matrix = time.perf_counter() + market_matrix = build_market_matrix_from_signals( + sim_market_data, + sim_signal_matrix, + entry_delay_bars=1 if matcher_config.entry_fill == "open_t+1" else 0, + exit_delay_bars=1 if matcher_config.exit_fill == "open_t+1" else 0, + reference_price=reference_price, + ) + timing_ms["matrix_build"] = round((time.perf_counter() - t_matrix) * 1000, 1) + del sim_market_data, sim_signal_matrix + else: + if panel is None or formal_range is None: + return _err("非矩阵策略不能使用共享基础矩阵") + # basic_filter 只影响买入候选，不能删除持仓估值和卖出所需行情。 + basic_mask = pl.Series("_basic", [True] * len(panel), dtype=pl.Boolean) + if basic_filter and basic_filter.get("enabled", True): + expr = StrategyEngine._basic_filter_expr(panel, basic_filter) + if expr is not None: + try: + basic_mask = panel.select(expr.alias("_basic"))["_basic"].fill_null(False).cast(pl.Boolean) + except Exception as e: # noqa: BLE001 + logger.warning("basic_filter mask failed: %s", e) + return _err(f"基础过滤计算失败: {e}") + + candidate_filter_mask = self._build_candidate_filter_mask(panel, s, params) + candidate_mask = basic_mask & candidate_filter_mask + panel = self._apply_score(panel, s, overrides, universe_mask=candidate_mask) + entry_mask = self._build_entry_mask_from_candidate(panel, candidate_mask, s, entry_signals) + entry_mask = entry_mask & formal_range + raw_exit_mask = self._build_signal_mask(panel, exit_signals, "_exit") + exit_range = self._date_range_mask(panel, config.start, load_end) if config.mode == "full" else formal_range + exit_mask = raw_exit_mask & exit_range + timing_ms["signals_score"] = round((time.perf_counter() - t_signal) * 1000, 1) + if not entry_mask.any(): + return _err("在指定区间内未产生买入信号") + + sim_range = self._date_range_mask(panel, config.start, sim_end) + sim_columns = [column for column in feature_plan.matrix_columns if column in panel.columns] + sim_panel = panel.filter(sim_range).select(sorted(sim_columns)) + sim_entry_mask = entry_mask.filter(sim_range) + sim_exit_mask = exit_mask.filter(sim_range) + if sim_panel.is_empty(): + return _err("正式回测区间内无数据") + panel_rows = int(sim_panel.height) + panel_columns = int(sim_panel.width) + raw_candidates = int(sim_entry_mask.sum()) + + t_matrix = time.perf_counter() + market_matrix = build_market_matrix( sim_panel, sim_entry_mask, sim_exit_mask, - matcher_config, - progress_cb, - cancel_event, + entry_delay_bars=1 if matcher_config.entry_fill == "open_t+1" else 0, + exit_delay_bars=1 if matcher_config.exit_fill == "open_t+1" else 0, entry_signal_ids=entry_signals, exit_signal_ids=exit_signals, ) + timing_ms["matrix_build"] = round((time.perf_counter() - t_matrix) * 1000, 1) + del panel, sim_panel, sim_entry_mask, sim_exit_mask + + t_sim = time.perf_counter() + + # 撮合 — 两条生产路径共享同一只读 MarketMatrix。 + if config.mode == "full": + result = self.engine.simulate_independent_market_matrix( + market_matrix, + raw_candidates, + matcher_config, + progress_cb, + cancel_event, + result_policy.simulation_options(), + ) else: - result = self.engine.simulate_portfolio( - sim_panel, sim_entry_mask, sim_exit_mask, matcher_config, - progress_cb, cancel_event, - entry_signal_ids=entry_signals, exit_signal_ids=exit_signals, + result = self.engine.simulate_market_matrix( + market_matrix, + matcher_config, + progress_cb, + cancel_event, + result_policy.simulation_options(), ) timing_ms["simulate"] = round((time.perf_counter() - t_sim) * 1000, 1) + timing_ms["statistics"] = float(result.stats.pop("statistics_ms", 0.0)) # 检查是否被取消 if cancel_event is not None and cancel_event.is_set(): @@ -254,9 +1160,25 @@ class StrategyBacktestService: timing_ms["total"] = round((time.perf_counter() - t0) * 1000, 1) result.stats["timing_ms"] = timing_ms - result.stats["panel_rows"] = int(sim_panel.height) + result.stats["panel_rows"] = panel_rows + result.stats["panel_columns"] = panel_columns + result.stats["feature_columns"] = feature_width + result.stats["full_feature_fallback"] = feature_plan.full_feature_fallback + result.stats["execution_backend"] = s.execution_backend + result.stats["shared_market_data"] = prepared is not None + result.stats["matrix_data_cache_hit"] = matrix_data_cache_hit + result.stats["matrix_data_cache_status"] = matrix_data_cache_status + result.stats["matrix_data_cache_timing_ms"] = dict(matrix_data_cache_timing_ms) + if prepared is not None: + result.stats["shared_market_data_bytes"] = prepared.market_data.nbytes + result.stats["shared_prepare_timing_ms"] = prepared.prepare_timing_ms + result.stats["matrix_compute_cache"] = prepared.compute_cache.snapshot() - benchmark_curve = self._build_benchmark_curve(config.start, config.end) + benchmark_curve = ( + self._build_benchmark_curve(config.start, config.end) + if result_policy.include_benchmark + else [] + ) # 构建策略信息 strategy_info = { @@ -275,19 +1197,30 @@ class StrategyBacktestService: "score_min": score_min, "score_max": score_max, "source": s.source, - } + "execution_backend": s.execution_backend, + } if result_policy.include_strategy_info else {} + + selected_stats = result_policy.select_stats(result.stats) elapsed = (time.perf_counter() - t0) * 1000 return StrategyBacktestResult( run_id=run_id, config=self._config_to_dict(config), - stats=result.stats, - equity_curve=result.equity_curve, - drawdown_curve=result.drawdown_curve, + stats=selected_stats, + equity_curve=result.equity_curve if result_policy.include_curves else [], + drawdown_curve=result.drawdown_curve if result_policy.include_curves else [], benchmark_curve=benchmark_curve, - trades=[self._trade_to_dict(t) for t in result.trades], - per_symbol_stats=result.per_symbol_stats, + trades=( + [self._trade_to_dict(t) for t in result.trades] + if result_policy.include_trades + else [] + ), + per_symbol_stats=( + result.per_symbol_stats + if result_policy.include_per_symbol_stats + else [] + ), strategy_info=strategy_info, elapsed_ms=round(elapsed, 1), ) @@ -419,6 +1352,20 @@ class StrategyBacktestService: ((pl.col("date") >= start) & (pl.col("date") <= end)).alias("_range") )["_range"].fill_null(False).cast(pl.Boolean) + @staticmethod + def _matrix_date_range_mask( + timestamp_labels: tuple[str, ...], + start: date, + end: date, + ) -> np.ndarray: + start_text = str(start) + end_text = str(end) + return np.fromiter( + (start_text <= label[:10] <= end_text for label in timestamp_labels), + dtype=bool, + count=len(timestamp_labels), + ) + def _build_candidate_filter_mask( self, panel: pl.DataFrame, @@ -557,6 +1504,22 @@ class StrategyBacktestService: return [str(v) for v in value if v] return list(default or []) + @classmethod + def _has_matrix_signal_override(cls, strategy: StrategyDef, overrides: dict) -> bool: + """Allow legacy persisted defaults, but reject a real Matrix signal replacement.""" + for key, default in ( + ("entry_signals", strategy.entry_signals), + ("exit_signals", strategy.exit_signals), + ): + if key not in overrides: + continue + actual = cls._effective_signals(overrides, key, default) + expected = [_normalize_signal_name(str(signal)) for signal in (default or [])] + normalized_actual = [_normalize_signal_name(signal) for signal in actual] + if normalized_actual != expected: + return True + return False + @staticmethod def _override_value(overrides: dict, key: str, default): if key in overrides: diff --git a/backend/app/backtest/walkforward.py b/backend/app/backtest/walkforward.py index cd7361d..ca2b1a1 100644 --- a/backend/app/backtest/walkforward.py +++ b/backend/app/backtest/walkforward.py @@ -136,6 +136,7 @@ class WalkForwardConfig: base_params: dict = field(default_factory=dict) overrides: dict | None = None backtest_kwargs: dict = field(default_factory=dict) + matrix_cache_max_mb: int = 512 class WalkForwardService: @@ -146,6 +147,37 @@ class WalkForwardService: self.service = service self.strategy_engine = strategy_engine + def _prepare_shared_matrix(self, cfg: WalkForwardConfig, folds: list[Fold]): + """Build one immutable superset matrix for every matrix-native fold.""" + if self.strategy_engine is None or not folds: + return None + strategy = self.strategy_engine.get(cfg.strategy_id) + if strategy.execution_backend != "matrix_native": + return None + + from app.backtest.optimizer import expand_param_grid + from app.backtest.strategy import StrategyBacktestConfig + + combos = expand_param_grid(strategy.meta.get("params", []), cfg.param_grid) + shared_start = min(fold.train_start for fold in folds) + shared_end = max(fold.test_end for fold in folds) + configs = [ + StrategyBacktestConfig( + strategy_id=cfg.strategy_id, + symbols=cfg.symbols, + start=shared_start, + end=shared_end, + params={**cfg.base_params, **combo}, + overrides=cfg.overrides, + **cfg.backtest_kwargs, + ) + for combo in combos + ] + return self.service.prepare_matrix_optimization( + configs, + matrix_cache_max_bytes=int(cfg.matrix_cache_max_mb) * 1024 * 1024, + ) + def run( self, cfg: WalkForwardConfig, @@ -160,6 +192,11 @@ class WalkForwardService: folds = generate_folds(cfg.start, cfg.end, cfg.train_days, cfg.test_days, cfg.step_days) n_total = len(folds) + shared_prepared = self._prepare_shared_matrix(cfg, folds) + shared_market_data = ( + shared_prepared.market_data if shared_prepared is not None else None + ) + # 遥测: 首尾快照 PanelCache, 量化跨折重叠区间重复扫盘的 IO 占比 (是否值得进一步优化)。 cache_before = self.service.engine.cache_stats() @@ -190,7 +227,14 @@ class WalkForwardService: overrides=cfg.overrides, backtest_kwargs=is_backtest_kwargs, # IS 强制 position, 堵前视泄漏 ) - opt_res = self.optimizer.optimize(opt_cfg, cancel_event=cancel_event) + if shared_market_data is None: + opt_res = self.optimizer.optimize(opt_cfg, cancel_event=cancel_event) + else: + opt_res = self.optimizer.optimize( + opt_cfg, + cancel_event=cancel_event, + prepared_market_data=shared_market_data, + ) best_params = opt_res.get("best_params") is_score = opt_res.get("best_score") done += 1 @@ -221,7 +265,28 @@ class WalkForwardService: overrides=cfg.overrides, **cfg.backtest_kwargs, ) - oos_res = self.service.run(oos_cfg, cancel_event=cancel_event) + oos_prepared = None + try: + if shared_market_data is not None: + oos_prepared = self.service.prepare_matrix_optimization( + [oos_cfg], + matrix_cache_max_bytes=int(cfg.matrix_cache_max_mb) * 1024 * 1024, + market_data_override=shared_market_data, + ) + if oos_prepared is None: + oos_res = self.service.run( + oos_cfg, + cancel_event=cancel_event, + ) + else: + oos_res = self.service.run( + oos_cfg, + cancel_event=cancel_event, + prepared=oos_prepared, + ) + finally: + if oos_prepared is not None: + oos_prepared.compute_cache.close() # OOS 失败 (含 cancelled) -> 跳过, 不把空/0 收益混入复利曲线 if oos_res.error: @@ -250,6 +315,15 @@ class WalkForwardService: summary = aggregate_oos(valid_records, cfg.objective, direction) + shared_matrix_bytes = ( + shared_prepared.market_data.nbytes if shared_prepared is not None else 0 + ) + shared_matrix_status = ( + shared_prepared.market_data.cache_status if shared_prepared is not None else "none" + ) + if shared_prepared is not None: + shared_prepared.compute_cache.close() + # 遥测收尾: 本次 WF 累计扫盘耗时 / 命中 / 复用, 与总耗时对比得出 load_panel 占比。 cache_after = self.service.engine.cache_stats() elapsed_ms = round((time.perf_counter() - t0) * 1000, 1) @@ -278,5 +352,8 @@ class WalkForwardService: "skipped": skipped, "summary": summary, "cache_telemetry": cache_telemetry, + "shared_market_data": shared_prepared is not None, + "shared_market_data_bytes": shared_matrix_bytes, + "shared_market_data_status": shared_matrix_status, "elapsed_ms": elapsed_ms, } diff --git a/backend/app/backtest/worker.py b/backend/app/backtest/worker.py new file mode 100644 index 0000000..ba35c48 --- /dev/null +++ b/backend/app/backtest/worker.py @@ -0,0 +1,311 @@ +"""Spawn-isolated strategy backtest and optimizer task runner.""" +from __future__ import annotations + +import json +import multiprocessing as mp +import os +import queue +import threading +import time +import traceback +from collections.abc import Callable +from contextlib import suppress +from dataclasses import asdict +from datetime import date +from pathlib import Path +from typing import Any + +import psutil + + +class BacktestWorkerError(RuntimeError): + """Raised when a spawned worker fails before returning a task result.""" + + +class _PeakRssSampler: + """Track whole-task and resettable phase RSS peaks with one sampling thread.""" + + def __init__(self, interval_seconds: float = 0.05) -> None: + if interval_seconds <= 0: + raise ValueError("RSS sample interval must be positive") + self._process = psutil.Process(os.getpid()) + self._interval_seconds = float(interval_seconds) + self._stop = threading.Event() + self._thread = threading.Thread(target=self._sample, daemon=True) + self._lock = threading.Lock() + self._started = False + current = int(self._process.memory_info().rss) + self.peak_rss_bytes = current + self._phase_peak_rss_bytes = current + + def start(self) -> None: + if self._started: + raise RuntimeError("RSS sampler has already started") + self._started = True + self._thread.start() + + def stop(self) -> int: + if self._started: + self._stop.set() + self._thread.join(timeout=1.0) + self._record_current() + return self.peak_rss_bytes + + def reset_phase(self) -> None: + current = int(self._process.memory_info().rss) + with self._lock: + self._phase_peak_rss_bytes = current + + def phase_peak_rss_bytes(self) -> int: + self._record_current() + with self._lock: + return self._phase_peak_rss_bytes + + def _record_current(self) -> None: + current = int(self._process.memory_info().rss) + with self._lock: + self.peak_rss_bytes = max(self.peak_rss_bytes, current) + self._phase_peak_rss_bytes = max(self._phase_peak_rss_bytes, current) + + def _sample(self) -> None: + while not self._stop.wait(self._interval_seconds): + self._record_current() + + +def _rss_bytes() -> int: + return int(psutil.Process(os.getpid()).memory_info().rss) + + +def _strategy_dirs(data_dir: Path) -> list[Path]: + app_dir = Path(__file__).resolve().parents[1] + return [ + app_dir / "strategy" / "builtin", + data_dir / "strategies" / "custom", + data_dir / "strategies" / "ai", + ] + + +def _decode_backtest_config(payload: dict[str, Any]): + from app.backtest.strategy import StrategyBacktestConfig + + values = dict(payload) + values["start"] = date.fromisoformat(values["start"]) + values["end"] = date.fromisoformat(values["end"]) + return StrategyBacktestConfig(**values) + + +def _decode_optimize_config(payload: dict[str, Any]): + from app.backtest.optimizer import OptimizeConfig + + values = dict(payload) + values["start"] = date.fromisoformat(values["start"]) + values["end"] = date.fromisoformat(values["end"]) + return OptimizeConfig(**values) + + +def _decode_walkforward_config(payload: dict[str, Any]): + from app.backtest.walkforward import WalkForwardConfig + + values = dict(payload) + values["start"] = date.fromisoformat(values["start"]) + values["end"] = date.fromisoformat(values["end"]) + return WalkForwardConfig(**values) + + +def encode_backtest_config(config) -> dict[str, Any]: + payload = asdict(config) + payload["start"] = config.start.isoformat() + payload["end"] = config.end.isoformat() + return payload + + +def encode_optimize_config(config) -> dict[str, Any]: + payload = asdict(config) + payload["start"] = config.start.isoformat() + payload["end"] = config.end.isoformat() + return payload + + +def make_worker_task(kind: str, data_dir: Path, config) -> dict[str, Any]: + if kind == "backtest": + encoded = encode_backtest_config(config) + elif kind == "optimize": + encoded = encode_optimize_config(config) + elif kind == "walkforward": + encoded = asdict(config) + encoded["start"] = config.start.isoformat() + encoded["end"] = config.end.isoformat() + else: + raise ValueError(f"unsupported worker task kind: {kind}") + return { + "kind": kind, + "data_dir": str(data_dir.resolve()), + "config": encoded, + } + + +def _attach_worker_metrics( + kind: str, + result: dict[str, Any], + metrics: dict[str, Any], +) -> None: + if kind == "backtest": + result.setdefault("stats", {})["worker"] = metrics + else: + result["worker"] = metrics + + +def _worker_entry(task: dict[str, Any], event_queue, cancel_event) -> None: + sampler = _PeakRssSampler() + sampler.start() + started = time.perf_counter() + store = None + try: + from app.backtest.engine import BacktestEngine + from app.backtest.optimizer import StrategyOptimizer + from app.backtest.strategy import StrategyBacktestService + from app.strategy.engine import StrategyEngine + from app.tickflow.repository import DataStore, KlineRepository + + data_dir = Path(task["data_dir"]) + store = DataStore(data_dir) + repo = KlineRepository(store) + strategy_engine = StrategyEngine(strategy_dirs=_strategy_dirs(data_dir)) + service = StrategyBacktestService(BacktestEngine(repo), strategy_engine) + + def _progress(message: dict) -> None: + event_queue.put({"type": "progress", "payload": message}) + + kind = task["kind"] + if kind == "backtest": + config = _decode_backtest_config(task["config"]) + result = asdict(service.run(config, _progress, cancel_event)) + elif kind == "optimize": + config = _decode_optimize_config(task["config"]) + optimizer = StrategyOptimizer(service, strategy_engine) + result = optimizer.optimize( + config, + _progress, + cancel_event, + rss_sampler=sampler, + ) + elif kind == "walkforward": + from app.backtest.walkforward import WalkForwardService + + config = _decode_walkforward_config(task["config"]) + optimizer = StrategyOptimizer(service, strategy_engine) + walkforward = WalkForwardService(optimizer, service, strategy_engine) + result = walkforward.run(config, _progress, cancel_event) + else: + raise ValueError(f"unsupported worker task kind: {kind}") + + serialization_started = time.perf_counter() + serialized_bytes = len( + json.dumps(result, ensure_ascii=False, default=str).encode("utf-8") + ) + serialization_ms = round( + (time.perf_counter() - serialization_started) * 1000, + 1, + ) + peak_rss = sampler.stop() + metrics = { + "pid": os.getpid(), + "peak_rss_bytes": peak_rss, + "final_rss_bytes": _rss_bytes(), + "serialization_ms": serialization_ms, + "serialized_result_bytes": serialized_bytes, + "task_elapsed_ms": round((time.perf_counter() - started) * 1000, 1), + } + _attach_worker_metrics(kind, result, metrics) + event_queue.put({"type": "result", "payload": result}) + except BaseException as exc: + with suppress(Exception): + sampler.stop() + event_queue.put({ + "type": "error", + "message": str(exc), + "traceback": traceback.format_exc(), + }) + finally: + if store is not None: + with suppress(Exception): + store.db.close() + + +def run_worker_task( + task: dict[str, Any], + progress_cb: Callable[[dict], None] | None = None, + cancel_event: threading.Event | None = None, +) -> dict[str, Any]: + """Run one complete task in a spawned process and wait for deterministic exit.""" + context = mp.get_context("spawn") + events = context.Queue() + process_cancel = context.Event() + process = context.Process( + target=_worker_entry, + args=(task, events, process_cancel), + daemon=False, + ) + parent_rss_before = _rss_bytes() + try: + process.start() + except BaseException: + events.close() + events.join_thread() + raise + result: dict[str, Any] | None = None + failure: dict[str, Any] | None = None + ipc_started = time.perf_counter() + + try: + while result is None and failure is None: + if cancel_event is not None and cancel_event.is_set(): + process_cancel.set() + try: + message = events.get(timeout=0.1) + except queue.Empty: + if not process.is_alive(): + break + continue + + message_type = message.get("type") + if message_type == "progress": + if progress_cb is not None: + progress_cb(message["payload"]) + elif message_type == "result": + result = message["payload"] + elif message_type == "error": + failure = message + + process.join(timeout=10.0) + if process.is_alive(): + process.terminate() + process.join(timeout=5.0) + raise BacktestWorkerError("backtest worker returned but did not exit within 10 seconds") + if failure is not None: + raise BacktestWorkerError( + f"{failure.get('message', 'worker failed')}\n{failure.get('traceback', '')}".rstrip() + ) + if result is None: + raise BacktestWorkerError( + f"backtest worker exited without result (exitcode={process.exitcode})" + ) + + parent_metrics = { + "ipc_elapsed_ms": round((time.perf_counter() - ipc_started) * 1000, 1), + "parent_rss_before_bytes": parent_rss_before, + "parent_rss_after_worker_exit_bytes": _rss_bytes(), + "worker_exitcode": process.exitcode, + } + kind = task["kind"] + if kind == "backtest": + result.setdefault("stats", {}).setdefault("worker", {}).update(parent_metrics) + else: + result.setdefault("worker", {}).update(parent_metrics) + return result + finally: + if process.is_alive(): + process.terminate() + process.join(timeout=5.0) + events.close() + events.join_thread() diff --git a/backend/app/config.py b/backend/app/config.py index 3ac9e0c..b57f305 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -95,6 +95,10 @@ class Settings(BaseSettings): port: int = 3018 log_level: str = "INFO" backtest_range_guard: bool = False + backtest_matrix_disk_cache_enabled: bool = True + backtest_matrix_cache_max_mb: int = 512 + backtest_matrix_cache_prewarm: bool = True + backtest_matrix_cache_prewarm_years: int = 5 # Auth — 首次启动时预置访问密码(明文, 仅用于初始化, 详见 services/auth.bootstrap_from_env) # 公网服务器部署时免去 SSH 端口转发设密码的麻烦。写入 auth.json(哈希)后即不再读取。 @@ -116,6 +120,10 @@ class Settings(BaseSettings): if not self.data_dir.is_absolute(): # 相对路径基于项目根目录解析，而非 CWD self.data_dir = (_PROJECT_ROOT / self.data_dir).resolve() + if self.backtest_matrix_cache_max_mb <= 0: + raise ValueError("backtest_matrix_cache_max_mb must be positive") + if self.backtest_matrix_cache_prewarm_years <= 0: + raise ValueError("backtest_matrix_cache_prewarm_years must be positive") return self @property diff --git a/backend/app/desktop.py b/backend/app/desktop.py index ad6f91b..bc575b7 100644 --- a/backend/app/desktop.py +++ b/backend/app/desktop.py @@ -346,4 +346,7 @@ def main() -> int: if __name__ == "__main__": + import multiprocessing + + multiprocessing.freeze_support() sys.exit(main()) diff --git a/backend/app/indicators/pipeline.py b/backend/app/indicators/pipeline.py index c1606d0..f03b501 100644 --- a/backend/app/indicators/pipeline.py +++ b/backend/app/indicators/pipeline.py @@ -541,7 +541,50 @@ def compute_indicators(df: pl.DataFrame, needed: set[str] | None = None) -> pl.D return df -def compute_signals(df: pl.DataFrame) -> pl.DataFrame: +SIGNAL_DEPENDENCIES: dict[str, frozenset[str]] = { + "signal_ma_golden_5_20": frozenset({"ma5", "ma20"}), + "signal_ma_dead_5_20": frozenset({"ma5", "ma20"}), + "signal_ma_golden_20_60": frozenset({"ma20", "ma60"}), + "signal_macd_golden": frozenset({"macd_dif", "macd_dea"}), + "signal_macd_dead": frozenset({"macd_dif", "macd_dea"}), + "signal_ma20_breakout": frozenset({"close", "ma20"}), + "signal_ma20_breakdown": frozenset({"close", "ma20"}), + "signal_ma5_breakout": frozenset({"close", "ma5"}), + "signal_ma5_breakdown": frozenset({"close", "ma5"}), + "signal_ma10_breakout": frozenset({"close", "ma10"}), + "signal_ma10_breakdown": frozenset({"close", "ma10"}), + "signal_n_day_high": frozenset({"close", "high_60d"}), + "signal_n_day_low": frozenset({"close", "low_60d"}), + "signal_boll_breakout_upper": frozenset({"close", "boll_upper"}), + "signal_boll_breakdown_lower": frozenset({"close", "boll_lower"}), + "signal_volume_surge": frozenset({"vol_ratio_5d"}), +} + +LIMIT_SIGNAL_OUTPUTS: frozenset[str] = frozenset({ + "signal_limit_up", + "signal_limit_down", + "signal_limit_down_recovery", + "signal_broken_limit_up", + "consecutive_limit_ups", + "consecutive_limit_downs", + "turnover_rate", +}) + +INDICATOR_COLUMNS: frozenset[str] = frozenset( + col for col in _ALL_INDICATOR_COLS if not col.startswith("_") +) + + +def get_signal_dependencies() -> dict[str, frozenset[str]]: + """返回内置与 JSON 自定义信号的唯一依赖映射。""" + from app.strategy import custom_signals + + return { + **SIGNAL_DEPENDENCIES, + **custom_signals.expression_dependencies(_get_custom_signal_exprs()), + } + +def compute_signals(df: pl.DataFrame, needed: set[str] | None = None) -> pl.DataFrame: """从已有指标列计算原子信号布尔列。 输入必须包含 compute_indicators() 产出的指标列。 @@ -549,55 +592,62 @@ def compute_signals(df: pl.DataFrame) -> pl.DataFrame: if df.is_empty(): return df - df = df.with_columns([ - ((pl.col("ma5") > pl.col("ma20")) & + want = set(SIGNAL_DEPENDENCIES) if needed is None else set(needed) & set(SIGNAL_DEPENDENCIES) + expressions: dict[str, pl.Expr] = { + "signal_ma_golden_5_20": ((pl.col("ma5") > pl.col("ma20")) & (pl.col("ma5").shift(1).over("symbol") <= pl.col("ma20").shift(1).over("symbol"))) .alias("signal_ma_golden_5_20"), - ((pl.col("ma5") < pl.col("ma20")) & + "signal_ma_dead_5_20": ((pl.col("ma5") < pl.col("ma20")) & (pl.col("ma5").shift(1).over("symbol") >= pl.col("ma20").shift(1).over("symbol"))) .alias("signal_ma_dead_5_20"), - ((pl.col("ma20") > pl.col("ma60")) & + "signal_ma_golden_20_60": ((pl.col("ma20") > pl.col("ma60")) & (pl.col("ma20").shift(1).over("symbol") <= pl.col("ma60").shift(1).over("symbol"))) .alias("signal_ma_golden_20_60"), - ((pl.col("macd_dif") > pl.col("macd_dea")) & + "signal_macd_golden": ((pl.col("macd_dif") > pl.col("macd_dea")) & (pl.col("macd_dif").shift(1).over("symbol") <= pl.col("macd_dea").shift(1).over("symbol"))) .alias("signal_macd_golden"), - ((pl.col("macd_dif") < pl.col("macd_dea")) & + "signal_macd_dead": ((pl.col("macd_dif") < pl.col("macd_dea")) & (pl.col("macd_dif").shift(1).over("symbol") >= pl.col("macd_dea").shift(1).over("symbol"))) .alias("signal_macd_dead"), - ((pl.col("close") > pl.col("ma20")) & + "signal_ma20_breakout": ((pl.col("close") > pl.col("ma20")) & (pl.col("close").shift(1).over("symbol") <= pl.col("ma20").shift(1).over("symbol"))) .alias("signal_ma20_breakout"), - ((pl.col("close") < pl.col("ma20")) & + "signal_ma20_breakdown": ((pl.col("close") < pl.col("ma20")) & (pl.col("close").shift(1).over("symbol") >= pl.col("ma20").shift(1).over("symbol"))) .alias("signal_ma20_breakdown"), - ((pl.col("close") > pl.col("ma5")) & + "signal_ma5_breakout": ((pl.col("close") > pl.col("ma5")) & (pl.col("close").shift(1).over("symbol") <= pl.col("ma5").shift(1).over("symbol"))) .alias("signal_ma5_breakout"), - ((pl.col("close") < pl.col("ma5")) & + "signal_ma5_breakdown": ((pl.col("close") < pl.col("ma5")) & (pl.col("close").shift(1).over("symbol") >= pl.col("ma5").shift(1).over("symbol"))) .alias("signal_ma5_breakdown"), - ((pl.col("close") > pl.col("ma10")) & + "signal_ma10_breakout": ((pl.col("close") > pl.col("ma10")) & (pl.col("close").shift(1).over("symbol") <= pl.col("ma10").shift(1).over("symbol"))) .alias("signal_ma10_breakout"), - ((pl.col("close") < pl.col("ma10")) & + "signal_ma10_breakdown": ((pl.col("close") < pl.col("ma10")) & (pl.col("close").shift(1).over("symbol") >= pl.col("ma10").shift(1).over("symbol"))) .alias("signal_ma10_breakdown"), - (pl.col("close") >= pl.col("high_60d")).alias("signal_n_day_high"), - (pl.col("close") <= pl.col("low_60d")).alias("signal_n_day_low"), - (pl.col("close") > pl.col("boll_upper")).alias("signal_boll_breakout_upper"), - (pl.col("close") < pl.col("boll_lower")).alias("signal_boll_breakdown_lower"), - (pl.col("vol_ratio_5d") >= 2.0).alias("signal_volume_surge"), - ]) + "signal_n_day_high": (pl.col("close") >= pl.col("high_60d")).alias("signal_n_day_high"), + "signal_n_day_low": (pl.col("close") <= pl.col("low_60d")).alias("signal_n_day_low"), + "signal_boll_breakout_upper": (pl.col("close") > pl.col("boll_upper")).alias("signal_boll_breakout_upper"), + "signal_boll_breakdown_lower": (pl.col("close") < pl.col("boll_lower")).alias("signal_boll_breakdown_lower"), + "signal_volume_surge": (pl.col("vol_ratio_5d") >= 2.0).alias("signal_volume_surge"), + } + if want: + df = df.with_columns([expressions[name] for name in SIGNAL_DEPENDENCIES if name in want]) # 自定义信号（用户配置的字段+运算符+值组合，编译为布尔列） from app.strategy import custom_signals - df = custom_signals.inject(df, _get_custom_signal_exprs()) + df = custom_signals.inject(df, _get_custom_signal_exprs(), needed=needed) return df -def compute_limit_signals(df: pl.DataFrame, instruments: pl.DataFrame) -> pl.DataFrame: +def compute_limit_signals( + df: pl.DataFrame, + instruments: pl.DataFrame, + needed: set[str] | None = None, +) -> pl.DataFrame: """计算涨跌停相关信号。 产出: @@ -612,14 +662,33 @@ def compute_limit_signals(df: pl.DataFrame, instruments: pl.DataFrame) -> pl.Dat if df.is_empty(): return df + want = set(LIMIT_SIGNAL_OUTPUTS) if needed is None else set(needed) & set(LIMIT_SIGNAL_OUTPUTS) + if not want: + return df + + need_up = bool(want & {"signal_limit_up", "consecutive_limit_ups", "signal_broken_limit_up"}) + need_down = bool(want & {"signal_limit_down", "consecutive_limit_downs", "signal_limit_down_recovery"}) + need_price_limits = need_up or need_down + # 从 instruments 取 ST 标记、流通股本(换手率用)以及最新日涨跌停价 inst_cols = ["symbol"] + instrument_needs = set() + if need_price_limits: + instrument_needs.add("name") + if "turnover_rate" in want: + instrument_needs.add("float_shares") + if need_up: + instrument_needs.add("limit_up") + if need_down: + instrument_needs.add("limit_down") for c in ["name", "float_shares", "limit_up", "limit_down"]: + if c not in instrument_needs: + continue if c in instruments.columns: inst_cols.append(c) inst_subset = instruments.select(inst_cols).unique(subset=["symbol"]) - if "name" in instruments.columns: + if need_price_limits and "name" in instruments.columns: st_flag = ( instruments .select("symbol", pl.col("name").str.contains("ST").alias("_is_st")) @@ -630,19 +699,23 @@ def compute_limit_signals(df: pl.DataFrame, instruments: pl.DataFrame) -> pl.Dat df = df.join(inst_subset, on="symbol", how="left", suffix="_inst") # 计算换手率(%) = volume(手) * 10000 / float_shares(股) - if "float_shares" in df.columns and "volume" in df.columns: + if "turnover_rate" in want and "float_shares" in df.columns and "volume" in df.columns: df = df.with_columns( pl.when(pl.col("float_shares") > 0) .then(pl.col("volume") * 10000.0 / pl.col("float_shares")) .otherwise(None) .alias("turnover_rate") ) - elif "turnover_rate" not in df.columns: + elif "turnover_rate" in want and "turnover_rate" not in df.columns: df = df.with_columns(pl.lit(None).cast(pl.Float64).alias("turnover_rate")) # 前一日参考收盘价（交易所涨跌停基准价） # 仅在 adj_factor 发生变化（除权除息 XD/DR）时使用前复权昨收作为交易所参考价; # 否则使用原始 raw_close.shift(1) 以避免浮点精度误差。 + if not need_price_limits: + cleanup = [c for c in ("name", "float_shares", "limit_up", "limit_down") if c in df.columns] + return df.drop(cleanup) + _adj_today = pl.col("close") / pl.col("raw_close") _adj_yesterday = pl.col("close").shift(1).over("symbol") / pl.col("raw_close").shift(1).over("symbol") _adj_changed = (_adj_today - _adj_yesterday).abs() > 1e-6 @@ -706,13 +779,16 @@ def compute_limit_signals(df: pl.DataFrame, instruments: pl.DataFrame) -> pl.Dat ).then(pl.col("limit_down")).otherwise(pl.col("_theoretical_limit_down")) else: effective_limit_down = pl.col("_theoretical_limit_down") - df = df.with_columns([ - effective_limit_up.alias("_effective_limit_up"), - effective_limit_down.alias("_effective_limit_down"), - ]) + effective_exprs: list[pl.Expr] = [] + if need_up: + effective_exprs.append(effective_limit_up.alias("_effective_limit_up")) + if need_down: + effective_exprs.append(effective_limit_down.alias("_effective_limit_down")) + df = df.with_columns(effective_exprs) # ── signal_limit_up ── - df = df.with_columns( + if need_up: + df = df.with_columns( pl.when( pl.col("_prev_raw_close").is_not_null() & (pl.col("_prev_raw_close") > 0) @@ -721,32 +797,34 @@ def compute_limit_signals(df: pl.DataFrame, instruments: pl.DataFrame) -> pl.Dat pl.col("raw_close") >= (pl.col("_effective_limit_up") - 0.005) ).otherwise(None).cast(pl.Boolean) .alias("signal_limit_up") - ) + ) # ── consecutive_limit_ups ── - df = df.with_columns( + if "consecutive_limit_ups" in want: + df = df.with_columns( (~pl.col("signal_limit_up").fill_null(False)) .cast(pl.UInt32) .cum_sum() .over("symbol") .alias("_grp_up") - ).with_columns( + ).with_columns( pl.col("signal_limit_up") .cast(pl.UInt32) .cum_sum() .over("symbol", "_grp_up") .cast(pl.UInt32) .alias("consecutive_limit_ups") - ).with_columns( + ).with_columns( pl.when(pl.col("signal_limit_up").fill_null(False)) .then(pl.col("consecutive_limit_ups")) .otherwise(0) .cast(pl.UInt32) .alias("consecutive_limit_ups") - ) + ) # ── signal_limit_down ── - df = df.with_columns( + if need_down: + df = df.with_columns( pl.when( pl.col("_prev_raw_close").is_not_null() & (pl.col("_prev_raw_close") > 0) @@ -755,33 +833,35 @@ def compute_limit_signals(df: pl.DataFrame, instruments: pl.DataFrame) -> pl.Dat pl.col("raw_close") <= (pl.col("_effective_limit_down") + 0.005) ).otherwise(None).cast(pl.Boolean) .alias("signal_limit_down") - ) + ) # ── consecutive_limit_downs ── - df = df.with_columns( + if "consecutive_limit_downs" in want: + df = df.with_columns( (~pl.col("signal_limit_down").fill_null(False)) .cast(pl.UInt32) .cum_sum() .over("symbol") .alias("_grp_down") - ).with_columns( + ).with_columns( pl.col("signal_limit_down") .cast(pl.UInt32) .cum_sum() .over("symbol", "_grp_down") .cast(pl.UInt32) .alias("consecutive_limit_downs") - ).with_columns( + ).with_columns( pl.when(pl.col("signal_limit_down").fill_null(False)) .then(pl.col("consecutive_limit_downs")) .otherwise(0) .cast(pl.UInt32) .alias("consecutive_limit_downs") - ) + ) # ── signal_limit_down_recovery (跌停翘板) ── # 条件: 当日最低价曾触及跌停价 + 最终没有跌停 + 收阳 - df = df.with_columns( + if "signal_limit_down_recovery" in want: + df = df.with_columns( pl.when( pl.col("_prev_raw_close").is_not_null() & (pl.col("_prev_raw_close") > 0) @@ -791,11 +871,12 @@ def compute_limit_signals(df: pl.DataFrame, instruments: pl.DataFrame) -> pl.Dat & (pl.col("close") > pl.col("open")) # 收阳 ).otherwise(None).cast(pl.Boolean) .alias("signal_limit_down_recovery") - ) + ) # ── signal_broken_limit_up (炸板) ── # 条件: 最高价曾触及涨停价 + 最终没有封住涨停 - df = df.with_columns( + if "signal_broken_limit_up" in want: + df = df.with_columns( pl.when( pl.col("_prev_raw_close").is_not_null() & (pl.col("_prev_raw_close") > 0) @@ -805,7 +886,7 @@ def compute_limit_signals(df: pl.DataFrame, instruments: pl.DataFrame) -> pl.Dat & (pl.col("raw_high") >= pl.col("_effective_limit_up") - 0.005) # 曾触及涨停价 ).otherwise(None).cast(pl.Boolean) .alias("signal_broken_limit_up") - ) + ) # 清理临时列 + JOIN 引入的 instruments 列 (不存入 enriched) cleanup = ["_prev_raw_close", "_board_pct", "_limit_pct", @@ -822,6 +903,8 @@ def compute_limit_signals(df: pl.DataFrame, instruments: pl.DataFrame) -> pl.Dat for c in ["name", "float_shares", "limit_up", "limit_down"]: if c in df.columns and c != "turnover_rate": cleanup.append(c) + internal_outputs = {"signal_limit_up", "signal_limit_down"} - want + cleanup.extend(c for c in internal_outputs if c in df.columns) df = df.drop([c for c in cleanup if c in df.columns]) return df diff --git a/backend/app/main.py b/backend/app/main.py index 0da2f49..1f801ab 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging +import threading from contextlib import asynccontextmanager from pathlib import Path @@ -47,6 +48,9 @@ async def lifespan(app: FastAPI): repo = KlineRepository(store) app.state.datastore = store app.state.repo = repo + # 在接受回测请求前固定 managed generation，避免首批并发 worker 各自创建版本。 + if settings.backtest_matrix_disk_cache_enabled: + repo.get_matrix_data_generation("stock") # 指标异步预热标志: enriched 缓存在后台线程构建, 完成后置 True app.state.indicators_ready = False repo._on_warmup_done = lambda: setattr(app.state, "indicators_ready", True) # noqa: SLF001 @@ -148,13 +152,61 @@ async def lifespan(app: FastAPI): store.data_dir / "strategies" / "ai", ] strategy_engine = StrategyEngine( - enriched_loader=_screener_svc._load_enriched_for_date, - enriched_history_loader=_screener_svc._load_enriched_history, strategy_dirs=strategy_dirs, ) app.state.strategy_engine = strategy_engine logger.info("strategy engine loaded: %d strategies", len(strategy_engine.list_strategies())) + matrix_prewarm_lock = threading.Lock() + matrix_prewarm_running = False + + def _schedule_matrix_cache_prewarm() -> None: + nonlocal matrix_prewarm_running + if ( + not settings.backtest_matrix_disk_cache_enabled + or not settings.backtest_matrix_cache_prewarm + ): + return + with matrix_prewarm_lock: + if matrix_prewarm_running: + logger.info("matrix cache prewarm already in progress, skip") + return + matrix_prewarm_running = True + + def _prewarm() -> None: + nonlocal matrix_prewarm_running + try: + latest = repo.latest_enriched_date("stock") + if latest is None: + logger.info("matrix cache prewarm skipped: no stock enriched data") + return + from app.backtest.engine import BacktestEngine + from app.backtest.strategy import prewarm_matrix_cache + + result = prewarm_matrix_cache( + BacktestEngine(repo), + strategy_engine, + asset_type="stock", + latest_date=latest, + years=settings.backtest_matrix_cache_prewarm_years, + ) + logger.info("matrix cache prewarm done: %s", result) + except Exception: # noqa: BLE001 + logger.exception("matrix cache prewarm failed") + finally: + with matrix_prewarm_lock: + matrix_prewarm_running = False + + threading.Thread( + target=_prewarm, + name="matrix-cache-prewarm", + daemon=True, + ).start() + + repo._on_refresh_done = _schedule_matrix_cache_prewarm # noqa: SLF001 + if repo.enriched_ready: + _schedule_matrix_cache_prewarm() + # 通用监控规则引擎: 启动时 reload 规则到内存态 (修复重启后告警失效) from app.strategy.monitor import MonitorRuleEngine from app.strategy import monitor_rules as mr_store @@ -173,7 +225,7 @@ async def lifespan(app: FastAPI): if preferences.get_strategy_monitor_enabled(): ids = preferences.get_strategy_monitor_ids() if ids: - names = {s.id: s.name for s in strategy_engine.list_strategies()} + names = {s["id"]: s["name"] for s in strategy_engine.list_strategies()} mr_store.migrate_strategy_monitors(store.data_dir, ids, names) logger.info("strategy monitor migrated: %d strategies", len(ids)) except Exception as e: # noqa: BLE001 diff --git a/backend/app/services/screener.py b/backend/app/services/screener.py index e8f71c8..608ca07 100644 --- a/backend/app/services/screener.py +++ b/backend/app/services/screener.py @@ -9,7 +9,6 @@ from __future__ import annotations import logging import time -from collections.abc import Callable from dataclasses import dataclass, field from datetime import date, timedelta @@ -25,172 +24,6 @@ _history_cache: dict[tuple[str, date, int], tuple[float, pl.DataFrame]] = {} _HISTORY_CACHE_TTL = 120.0 # 秒 -# 内置预设策略 — Polars 表达式方式 -PRESET_STRATEGIES: dict[str, dict] = { - "trend_breakout": { - "name": "趋势突破", - "description": "MA60 上方 + 60 日新高 + 量能 ≥ 2 倍均量", - "filter": ( - (pl.col("close") > pl.col("ma60")) - & pl.col("signal_n_day_high").fill_null(False) - & (pl.col("vol_ratio_5d") >= 2.0) - ), - "order_by": "momentum_60d", - "descending": True, - "limit": 100, - "asset_types": ["stock", "etf"], - }, - "ma_golden_cross": { - "name": "MA 金叉", - "description": "MA5 上穿 MA20 当日触发,量能配合", - "filter": ( - pl.col("signal_ma_golden_5_20").fill_null(False) - & (pl.col("vol_ratio_5d") >= 1.2) - & (pl.col("close") > pl.col("ma60")) - ), - "order_by": "momentum_20d", - "descending": True, - "limit": 100, - "asset_types": ["stock", "etf"], - }, - "macd_golden": { - "name": "MACD 金叉放量", - "description": "MACD 金叉当日 + 量能放大", - "filter": ( - pl.col("signal_macd_golden").fill_null(False) - & (pl.col("vol_ratio_5d") >= 1.5) - ), - "order_by": "momentum_60d", - "descending": True, - "limit": 100, - "asset_types": ["stock", "etf"], - }, - "volume_price_surge": { - "name": "量价齐升", - "description": "突破 MA20 + 放量 + 收阳", - "filter": ( - pl.col("signal_ma20_breakout").fill_null(False) - & (pl.col("vol_ratio_5d") >= 2.0) - & (pl.col("close") > pl.col("open")) - ), - "order_by": "vol_ratio_5d", - "descending": True, - "limit": 100, - "asset_types": ["stock", "etf"], - }, - "low_volatility_leader": { - "name": "低波动龙头", - "description": "20 日动量为正 + 年化波动 < 30% + MA20 上方", - "filter": ( - (pl.col("momentum_20d") > 0) - & (pl.col("annual_vol_20d") < 0.30) - & (pl.col("close") > pl.col("ma20")) - ), - "order_by": "momentum_60d", - "descending": True, - "limit": 100, - "asset_types": ["stock", "etf"], - }, - "broken_board_recovery": { - "name": "断板反包", - "description": "连板 ≥2 后断板 1-2 天，出现放量反包信号", - "filter": ( - pl.col("signal_limit_up").fill_null(False) - & (pl.col("vol_ratio_5d") >= 1.5) - & (pl.col("change_pct") > 0.03) - ), - "order_by": "change_pct", - "descending": True, - "limit": 100, - "asset_types": ["stock"], - }, - "oversold_bounce": { - "name": "超跌反弹", - "description": "RSI14 < 30 超卖区 + 当日收阳 + 放量，抄底信号", - "filter": ( - (pl.col("rsi_14") < 30) - & (pl.col("close") > pl.col("open")) - & (pl.col("vol_ratio_5d") >= 1.2) - ), - "order_by": "rsi_14", - "descending": False, - "limit": 100, - "asset_types": ["stock", "etf"], - }, - "boll_breakout": { - "name": "布林突破", - "description": "突破布林上轨 + 放量，强势加速信号", - "filter": ( - pl.col("signal_boll_breakout_upper").fill_null(False) - & (pl.col("vol_ratio_5d") >= 1.5) - ), - "order_by": "vol_ratio_5d", - "descending": True, - "limit": 100, - "asset_types": ["stock", "etf"], - }, - "bullish_alignment": { - "name": "均线多头", - "description": "MA5 > MA10 > MA20 > MA60 多头排列 + 短期动量为正", - "filter": ( - (pl.col("ma5") > pl.col("ma10")) - & (pl.col("ma10") > pl.col("ma20")) - & (pl.col("ma20") > pl.col("ma60")) - & (pl.col("momentum_20d") > 0) - ), - "order_by": "momentum_60d", - "descending": True, - "limit": 100, - "asset_types": ["stock", "etf"], - }, - "consecutive_limit_ups": { - "name": "连板股", - "description": "当日涨停且连续涨停 ≥ 2 天，强势追涨", - "filter": ( - pl.col("signal_limit_up").fill_null(False) - & (pl.col("consecutive_limit_ups") >= 2) - ), - "order_by": "consecutive_limit_ups", - "descending": True, - "limit": 100, - "asset_types": ["stock"], - }, - "pullback_to_support": { - "name": "缩量回踩", - "description": "回踩 MA20 附近 + 缩量 + 中期趋势向上", - "filter": ( - (pl.col("close") > pl.col("ma20") * 0.98) - & (pl.col("close") < pl.col("ma20") * 1.02) - & (pl.col("vol_ratio_5d") < 0.8) - & (pl.col("close") > pl.col("ma60")) - & (pl.col("momentum_20d") > 0) - ), - "order_by": "momentum_60d", - "descending": True, - "limit": 100, - "asset_types": ["stock", "etf"], - }, - "n_day_low_reversal": { - "name": "新低反转", - "description": "触及 60 日新低后当日收阳放量，反转信号", - "filter": ( - pl.col("signal_n_day_low").fill_null(False) - & (pl.col("close") > pl.col("open")) - & (pl.col("vol_ratio_5d") >= 1.5) - ), - "order_by": "change_pct", - "descending": True, - "limit": 100, - "asset_types": ["stock", "etf"], - }, -} - - -def strategy_supports_asset(strat: dict, asset_type: str) -> bool: - """策略是否支持该资产类型。默认仅 stock（未标注 asset_types 的自定义/AI 策略保守视为股票专用）。""" - return asset_type in strat.get("asset_types", ["stock"]) - - @dataclass class ScreenerResult: as_of: date @@ -312,7 +145,11 @@ class ScreenerService: 读取历史数据作为指标计算的 warmup, 计算完成后只返回目标日期的行。 """ - from app.indicators.pipeline import compute_indicators, compute_signals, compute_limit_signals + from app.indicators.pipeline import ( + compute_indicators, + compute_limit_signals, + compute_signals, + ) # 加载 warmup 历史 (目标日期前 ~120 天) enriched_dir = self.repo.store.data_dir / self._enriched_dirname @@ -394,7 +231,11 @@ class ScreenerService: # 优先级 3: scan_parquet + compute_indicators (慢路径, ~5s) logger.warning("_load_enriched_history cache miss, computing indicators (%s, %d)...", target_date, lookback_days) - from app.indicators.pipeline import compute_indicators, compute_signals, compute_limit_signals + from app.indicators.pipeline import ( + compute_indicators, + compute_limit_signals, + compute_signals, + ) warmup = 60 start = target_date - timedelta(days=min((lookback_days + warmup) * 2, 180)) @@ -519,147 +360,41 @@ class ScreenerService: elapsed_ms=elapsed, ) - def run_preset( + def build_strategy_context( self, - strategy_id: str, + engine, as_of: date, - pool: list[str] | None = None, - precomputed: pl.DataFrame | None = None, - basic_filter: dict | None = None, - filter_fn: Callable[[pl.DataFrame, dict], pl.Expr] | None = None, - strategy_params: dict | None = None, - display_limit: int | None = None, - ) -> ScreenerResult: - """预设策略选股 — 从 enriched 读取预计算好的指标列后过滤。 + strategy_ids: list[str], + *, + timeframe: str = "1d", + params_map: dict[str, dict] | None = None, + overrides_map: dict[str, dict] | None = None, + current: pl.DataFrame | None = None, + market=None, + cache_key: str | None = None, + ): + """按调用方要求装配标准策略数据上下文，不解释策略公式。""" + from app.strategy.engine import StrategyDataContext - - precomputed 不为空: 直接复用（run_all 场景） - - precomputed 为空: 从 enriched 读目标日期 - - basic_filter: 用户保存的基础参数过滤（boards、价格等） - - filter_fn/strategy_params: 内置策略文件的参数化过滤；未传时兼容旧预设表达式 - """ - t0 = time.perf_counter() - - strat = PRESET_STRATEGIES.get(strategy_id) - if not strat: - raise ValueError(f"unknown strategy: {strategy_id}") - - # 资产兼容拦截: 该策略不支持当前资产类型时直接返回空 (避免命中 ETF 不存在的列) - if not strategy_supports_asset(strat, self.asset_type): - return ScreenerResult(as_of=as_of, strategy=strategy_id) - - if precomputed is not None and not precomputed.is_empty(): - df = precomputed - else: - df = self._load_enriched_for_date(as_of) - if df.is_empty(): - return ScreenerResult(as_of=as_of, strategy=strategy_id) - - # 应用用户基础参数过滤（boards、价格区间等） - if basic_filter and basic_filter.get("enabled", True): - df = self._apply_basic_filter(df, basic_filter) - - # 应用策略过滤 - filter_expr = filter_fn(df, strategy_params or {}) if filter_fn else strat["filter"] - df = df.filter(filter_expr) - - # 应用 pool - if pool: - df = df.filter(pl.col("symbol").is_in(pool)) - - # 排序 + 限制 - order_col = strat["order_by"] - if order_col in df.columns: - df = df.sort(order_col, descending=strat.get("descending", True)) - - # display_limit: None=不限制, 0=全部, N=前N个 - if display_limit == 0: - limit = None # 不限制 - elif display_limit is not None: - limit = display_limit - else: - limit = None # 未配置时默认不限制 - if limit is not None and limit > 0: - df = df.head(limit) - - # 基于排序列生成 0-100 评分 (与 StrategyEngine 统一) - if order_col in df.columns and not df.is_empty(): - col_vals = df[order_col].cast(pl.Float64) - col_min = col_vals.min() - col_max = col_vals.max() - col_range = col_max - col_min - if col_range and col_range > 0: - normalized = (col_vals - col_min) / col_range - else: - normalized = pl.Series("norm", [0.5] * len(df)) - if not strat.get("descending", True): - normalized = 1.0 - normalized - df = df.with_columns((normalized * 100).alias("score")) - - rows = df.to_dicts() - elapsed = (time.perf_counter() - t0) * 1000 - - # sanitize - for r in rows: - for k, v in list(r.items()): - if isinstance(v, float) and (v != v or abs(v) == float("inf")): - r[k] = None - - return ScreenerResult( + if current is None: + current = self._load_enriched_for_date(as_of) + history_bars = engine.required_history_bars( + strategy_ids, + params_map=params_map, + overrides_map=overrides_map, + ) + history = None + if history_bars > 1: + history = self._load_enriched_history(as_of, history_bars) + return StrategyDataContext( + asset_type=self.asset_type, + timeframe=timeframe, as_of=as_of, - strategy=strategy_id, - rows=rows, - total=len(rows), - elapsed_ms=elapsed, + current=current, + history=history, + market=market, + cache_key=cache_key, ) - - @staticmethod - def _apply_basic_filter(df: pl.DataFrame, bf: dict) -> pl.DataFrame: - """应用用户基础参数过滤（boards、价格区间、市值等）""" - exprs: list[pl.Expr] = [] - if bf.get("price_min") is not None: - exprs.append(pl.col("close") >= bf["price_min"]) - if bf.get("price_max") is not None: - exprs.append(pl.col("close") <= bf["price_max"]) - if bf.get("float_cap_min") is not None and "float_shares" in df.columns: - exprs.append(pl.col("close") * pl.col("float_shares") >= bf["float_cap_min"]) - if bf.get("float_cap_max") is not None and "float_shares" in df.columns: - exprs.append(pl.col("close") * pl.col("float_shares") <= bf["float_cap_max"]) - if bf.get("amount_min") is not None: - exprs.append(pl.col("amount") >= bf["amount_min"]) - if bf.get("amount_max") is not None: - exprs.append(pl.col("amount") <= bf["amount_max"]) - if bf.get("turnover_min") is not None and "turnover_rate" in df.columns: - exprs.append(pl.col("turnover_rate") >= bf["turnover_min"]) - if bf.get("turnover_max") is not None and "turnover_rate" in df.columns: - exprs.append(pl.col("turnover_rate") <= bf["turnover_max"]) - if bf.get("exclude_st") and "name" in df.columns: - exprs.append(~pl.col("name").str.contains("(?i)ST|\\*ST|退")) - # 板块过滤 - boards = bf.get("boards") - if boards and isinstance(boards, list) and len(boards) > 0: - board_exprs: list[pl.Expr] = [] - for b in boards: - if b == "沪主板": - board_exprs.append(pl.col("symbol").str.starts_with("60")) - elif b == "深主板": - board_exprs.append( - pl.col("symbol").str.starts_with("00") - | pl.col("symbol").str.starts_with("001") - ) - elif b == "创业板": - board_exprs.append( - pl.col("symbol").str.starts_with("300") - | pl.col("symbol").str.starts_with("301") - ) - elif b == "科创板": - board_exprs.append(pl.col("symbol").str.starts_with("688")) - elif b == "北交所": - board_exprs.append(pl.col("symbol").str.contains(r"\.BJ$")) - if board_exprs: - exprs.append(pl.any_horizontal(board_exprs)) - if exprs: - return df.filter(pl.all_horizontal(exprs)) - return df def latest_date(self) -> date | None: if self.asset_type != "stock": diff --git a/backend/app/services/strategy_cache.py b/backend/app/services/strategy_cache.py index 37c9251..3450aa9 100644 --- a/backend/app/services/strategy_cache.py +++ b/backend/app/services/strategy_cache.py @@ -73,6 +73,14 @@ def read_cache(data_dir: Path) -> dict | None: return _read_cache_unlocked(data_dir) +def clear_cache(data_dir: Path) -> None: + """删除策略结果缓存；策略代码 reload 后避免继续展示旧公式结果。""" + path = _cache_path(data_dir) + with _file_lock: + path.unlink(missing_ok=True) + path.with_name(path.name + ".tmp").unlink(missing_ok=True) + + def _read_cache_unlocked(data_dir: Path) -> dict | None: """实际读取逻辑 (不持锁)。供 read_cache 与 write_cache 复用, 避免重入死锁。""" path = _cache_path(data_dir) diff --git a/backend/app/strategy/ai_generator.py b/backend/app/strategy/ai_generator.py index 5f10dfb..fe774bc 100644 --- a/backend/app/strategy/ai_generator.py +++ b/backend/app/strategy/ai_generator.py @@ -20,7 +20,7 @@ _SYSTEM_PREFIX = """你是A股量化策略设计专家。根据用户描述的 1. 只创建这一个策略文件：只生成一个 .py 文件，绝不创建多文件、不拆分模块、不跨文件引用 2. 绝不触碰项目源码：不要写任何会修改 backend/、docs/、frontend/ 等现有文件的代码；不要 import os/sys/pathlib 等文件系统模块 3. 不得放入内置策略目录：AI 生成的策略只属于 data/strategies/ai/，文件名/ID 用 ai_ 前缀；内置目录 backend/app/strategy/builtin/ 由项目维护，AI 不得染指 -4. 只 import polars as pl，不 import 其他模块 +4. polars 策略只 import polars 和 datetime；matrix_native 策略只允许 import numpy 以及 from app.backtest.matrix import 所需矩阵协议和算子 要求: 1. 用户可能调整的策略阈值通过 META["params"] 暴露；公式常数、固定窗口边界、布尔开关不必强行参数化 @@ -116,12 +116,24 @@ class AIStrategyGenerator: return content.split("```", 1)[1].split("```", 1)[0].strip() return content.strip() - # import 白名单: 策略文件只允许 polars + datetime (纯日期运算, 无文件/网络/进程能力)。 + # import 白名单: Polars 与矩阵策略只开放执行协议所需模块。 # 白名单而非黑名单 — 黑名单挡不住 ctypes/importlib/builtins/pickle 等未列出的危险模块。 - _ALLOWED_IMPORT_MODULES = frozenset({"polars", "__future__", "datetime"}) + _ALLOWED_IMPORT_MODULES = frozenset({ + "polars", + "numpy", + "app.backtest.matrix", + "datetime", + "__future__", + }) @classmethod - def _validate_safety(cls, code: str) -> None: + def _validate_safety( + cls, + code: str, + *, + extra_allowed_import_modules: frozenset[str] = frozenset(), + extra_allowed_calls: frozenset[str] = frozenset(), + ) -> None: """AST 级安全检查: import 白名单 + 危险内建调用拦截 + dunder 遍历拦截。 注意: AST 名单不是真正的沙箱, 只能拦截常见攻击模式。真正的隔离需要 @@ -131,9 +143,18 @@ class AIStrategyGenerator: """ tree = ast.parse(code) - forbidden_calls = {"open", "exec", "eval", "compile", "__import__", - "globals", "locals", "vars", "dir", "getattr", - "setattr", "delattr", "type", "input", "breakpoint"} + allowed_import_modules = cls._ALLOWED_IMPORT_MODULES | extra_allowed_import_modules + forbidden_calls = { + "open", "exec", "eval", "compile", "__import__", + "globals", "locals", "vars", "dir", "getattr", + "setattr", "delattr", "type", "input", "breakpoint", + } - extra_allowed_calls + + def _module_allowed(module: str) -> bool: + return ( + module in allowed_import_modules + or module.split(".", 1)[0] in allowed_import_modules + ) # dunder 属性名: 访问这些属性可逃逸出策略沙箱拿到 os/subprocess 等 forbidden_dunder_attrs = { @@ -149,12 +170,12 @@ class AIStrategyGenerator: for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: - if alias.name.split(".")[0] not in cls._ALLOWED_IMPORT_MODULES: - raise ValueError(f"禁止 import {alias.name} (策略只允许 import polars)") + if not _module_allowed(alias.name): + raise ValueError(f"禁止 import {alias.name} (不在策略安全白名单)") if isinstance(node, ast.ImportFrom): - mod = (node.module or "").split(".")[0] - if mod not in cls._ALLOWED_IMPORT_MODULES: - raise ValueError(f"禁止 from {node.module} import (策略只允许 import polars)") + mod = node.module or "" + if not _module_allowed(mod): + raise ValueError(f"禁止 from {node.module} import (不在策略安全白名单)") if isinstance(node, ast.Call): if isinstance(node.func, ast.Name) and node.func.id in forbidden_calls: raise ValueError(f"禁止调用 {node.func.id}()") diff --git a/backend/app/strategy/builtin/boll_breakout.py b/backend/app/strategy/builtin/boll_breakout.py index 0c6623e..87b809a 100644 --- a/backend/app/strategy/builtin/boll_breakout.py +++ b/backend/app/strategy/builtin/boll_breakout.py @@ -1,18 +1,33 @@ """布林突破 — 突破布林上轨 + 放量""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import MarketDataMatrix, SignalMatrix, make_signal_matrix, matrix_feature META = { "id": "boll_breakout", "name": "布林突破", "description": "突破布林上轨 + 放量, 强势加速信号", "tags": ["布林", "突破"], + "asset_types": ["stock", "etf"], + "timeframes": ["1d"], "params": [ - {"id": "require_boll_breakout", "label": "要求突破布林上轨", "type": "bool", - "default": True}, - {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", - "default": True}, - {"id": "vol_ratio_min", "label": "最低量比", "type": "float", - "default": 1.5, "min": 0.5, "max": 5.0, "step": 0.1}, + { + "id": "require_boll_breakout", + "label": "要求突破布林上轨", + "type": "bool", + "default": True, + }, + {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", "default": True}, + { + "id": "vol_ratio_min", + "label": "最低量比", + "type": "float", + "default": 1.5, + "min": 0.5, + "max": 5.0, + "step": 0.1, + }, ], "scoring": {"vol_ratio_5d": 0.4, "change_pct": 0.3, "momentum_20d": 0.3}, "order_by": "score", @@ -20,6 +35,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_boll_breakout_upper"] EXIT_SIGNALS = ["signal_boll_breakdown_lower"] STOP_LOSS = -0.06 @@ -27,11 +43,34 @@ MAX_HOLD_DAYS = 15 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - vol_min = params.get("vol_ratio_min", 1.5) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("require_boll_breakout", True): - expr = expr & pl.col("signal_boll_breakout_upper").fill_null(False) - if params.get("use_volume_filter", True): - expr = expr & (pl.col("vol_ratio_5d") >= vol_min) - return expr +class BollBreakoutMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close", "volume"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + upper = matrix_feature(market, "boll_upper") + lower = matrix_feature(market, "boll_lower") + entry = np.ones(market.shape, dtype=bool) + if params.get("require_boll_breakout", True): + entry &= market.close > upper + if params.get("use_volume_filter", True): + entry &= matrix_feature(market, "vol_ratio_5d") >= float( + params.get("vol_ratio_min", 1.5) + ) + exit_ = market.close < lower + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + entry_signal_ids=("signal_boll_breakout_upper",), + exit_signal_ids=("signal_boll_breakdown_lower",), + ) + + +MATRIX_STRATEGY = BollBreakoutMatrixStrategy() diff --git a/backend/app/strategy/builtin/broken_board_recovery.py b/backend/app/strategy/builtin/broken_board_recovery.py index 69c4735..d45f191 100644 --- a/backend/app/strategy/builtin/broken_board_recovery.py +++ b/backend/app/strategy/builtin/broken_board_recovery.py @@ -1,22 +1,46 @@ """断板反包 — 涨停 + 放量 + 涨幅 >3%""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "broken_board_recovery", "name": "断板反包", "description": "连板≥2后断板1-2天, 出现放量反包信号", "tags": ["涨停", "反包"], + "asset_types": ["stock"], + "timeframes": ["1d"], "params": [ - {"id": "require_limit_up", "label": "要求当日涨停", "type": "bool", - "default": True}, - {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", - "default": True}, - {"id": "vol_ratio_min", "label": "最低量比", "type": "float", - "default": 1.5, "min": 0.5, "max": 5.0, "step": 0.1}, - {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", - "default": True}, - {"id": "change_pct_min", "label": "最低涨幅", "type": "float", - "default": 0.03, "min": 0.01, "max": 0.10, "step": 0.01}, + {"id": "require_limit_up", "label": "要求当日涨停", "type": "bool", "default": True}, + {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", "default": True}, + { + "id": "vol_ratio_min", + "label": "最低量比", + "type": "float", + "default": 1.5, + "min": 0.5, + "max": 5.0, + "step": 0.1, + }, + {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", "default": True}, + { + "id": "change_pct_min", + "label": "最低涨幅", + "type": "float", + "default": 0.03, + "min": 0.01, + "max": 0.10, + "step": 0.01, + }, ], "scoring": {"change_pct": 0.4, "vol_ratio_5d": 0.3, "momentum_5d": 0.3}, "order_by": "score", @@ -24,6 +48,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_limit_up"] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.06 @@ -31,14 +56,37 @@ MAX_HOLD_DAYS = 10 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - vol_min = params.get("vol_ratio_min", 1.5) - chg_min = params.get("change_pct_min", 0.03) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("require_limit_up", True): - expr = expr & pl.col("signal_limit_up").fill_null(False) - if params.get("use_volume_filter", True): - expr = expr & (pl.col("vol_ratio_5d") >= vol_min) - if params.get("use_change_filter", True): - expr = expr & (pl.col("change_pct") > chg_min) - return expr +class BrokenBoardRecoveryMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close", "volume", "raw_close"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + entry = np.ones(market.shape, dtype=bool) + if params.get("require_limit_up", True): + entry &= market.limit_up_locked.astype(bool) + if params.get("use_volume_filter", True): + entry &= matrix_feature(market, "vol_ratio_5d") >= float( + params.get("vol_ratio_min", 1.5) + ) + if params.get("use_change_filter", True): + entry &= matrix_feature(market, "change_pct") > float( + params.get("change_pct_min", 0.03) + ) + ma20 = matrix_feature(market, "ma20") + exit_ = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + entry_signal_ids=("signal_limit_up",), + exit_signal_ids=("signal_ma20_breakdown",), + ) + + +MATRIX_STRATEGY = BrokenBoardRecoveryMatrixStrategy() diff --git a/backend/app/strategy/builtin/bullish_alignment.py b/backend/app/strategy/builtin/bullish_alignment.py index 0aff8f2..fb05474 100644 --- a/backend/app/strategy/builtin/bullish_alignment.py +++ b/backend/app/strategy/builtin/bullish_alignment.py @@ -1,16 +1,37 @@ """均线多头 — MA5>MA10>MA20>MA60 + 短期动量为正""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "bullish_alignment", "name": "均线多头", "description": "MA5>MA10>MA20>MA60多头排列 + 短期动量为正", "tags": ["均线", "多头"], + "asset_types": ["stock", "etf"], + "timeframes": ["1d"], "params": [ - {"id": "require_ma_alignment", "label": "要求均线多头排列", "type": "bool", - "default": True}, - {"id": "require_positive_momentum", "label": "要求20日动量为正", "type": "bool", - "default": True}, + { + "id": "require_ma_alignment", + "label": "要求均线多头排列", + "type": "bool", + "default": True, + }, + { + "id": "require_positive_momentum", + "label": "要求20日动量为正", + "type": "bool", + "default": True, + }, ], "scoring": {"momentum_60d": 0.4, "momentum_20d": 0.3, "turnover_rate": 0.3}, "order_by": "score", @@ -18,6 +39,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_ma_golden_5_20", "signal_ma_golden_20_60"] EXIT_SIGNALS = ["signal_ma_dead_5_20", "signal_ma20_breakdown"] STOP_LOSS = -0.06 @@ -25,15 +47,36 @@ MAX_HOLD_DAYS = 20 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("require_ma_alignment", True): - expr = ( - expr - & (pl.col("ma5") > pl.col("ma10")) - & (pl.col("ma10") > pl.col("ma20")) - & (pl.col("ma20") > pl.col("ma60")) +class BullishAlignmentMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + ma5 = matrix_feature(market, "ma5") + ma10 = matrix_feature(market, "ma10") + ma20 = matrix_feature(market, "ma20") + ma60 = matrix_feature(market, "ma60") + entry = np.ones(market.shape, dtype=bool) + if params.get("require_ma_alignment", True): + entry &= (ma5 > ma10) & (ma10 > ma20) & (ma20 > ma60) + if params.get("require_positive_momentum", True): + entry &= matrix_feature(market, "momentum_20d") > 0 + ma_dead = (ma5 < ma20) & (shift(ma5, 1) >= shift(ma20, 1)) + ma20_breakdown = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + exit_ = ma_dead | ma20_breakdown + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(ma_dead, 0, np.where(ma20_breakdown, 1, -1)).astype(np.int16), + entry_signal_ids=("signal_ma_golden_5_20", "signal_ma_golden_20_60"), + exit_signal_ids=("signal_ma_dead_5_20", "signal_ma20_breakdown"), ) - if params.get("require_positive_momentum", True): - expr = expr & (pl.col("momentum_20d") > 0) - return expr + + +MATRIX_STRATEGY = BullishAlignmentMatrixStrategy() diff --git a/backend/app/strategy/builtin/consecutive_limit_ups.py b/backend/app/strategy/builtin/consecutive_limit_ups.py index eef4444..7bc234c 100644 --- a/backend/app/strategy/builtin/consecutive_limit_ups.py +++ b/backend/app/strategy/builtin/consecutive_limit_ups.py @@ -1,18 +1,28 @@ """连板股 — 涨停且连续涨停≥2天""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import MarketDataMatrix, SignalMatrix, make_signal_matrix, matrix_feature META = { "id": "consecutive_limit_ups", "name": "连板股", "description": "当日涨停且连续涨停≥2天, 强势追涨", "tags": ["涨停", "连板"], + "asset_types": ["stock"], + "timeframes": ["1d"], "params": [ - {"id": "require_limit_up", "label": "要求当日涨停", "type": "bool", - "default": True}, - {"id": "use_boards_filter", "label": "启用连板数过滤", "type": "bool", - "default": True}, - {"id": "min_boards", "label": "最少连板数", "type": "int", - "default": 2, "min": 1, "max": 20, "step": 1}, + {"id": "require_limit_up", "label": "要求当日涨停", "type": "bool", "default": True}, + {"id": "use_boards_filter", "label": "启用连板数过滤", "type": "bool", "default": True}, + { + "id": "min_boards", + "label": "最少连板数", + "type": "int", + "default": 2, + "min": 1, + "max": 20, + "step": 1, + }, ], "scoring": {"consecutive_limit_ups": 0.5, "change_pct": 0.3, "amount": 0.2}, "order_by": "score", @@ -20,6 +30,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_limit_up"] EXIT_SIGNALS = [] STOP_LOSS = -0.05 @@ -27,11 +38,28 @@ MAX_HOLD_DAYS = 5 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - min_boards = params.get("min_boards", 2) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("require_limit_up", True): - expr = expr & pl.col("signal_limit_up").fill_null(False) - if params.get("use_boards_filter", True): - expr = expr & (pl.col("consecutive_limit_ups") >= min_boards) - return expr +class ConsecutiveLimitUpsMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"consecutive_limit_ups", "raw_close"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + entry = np.ones(market.shape, dtype=bool) + if params.get("require_limit_up", True): + entry &= market.limit_up_locked.astype(bool) + if params.get("use_boards_filter", True): + entry &= matrix_feature(market, "consecutive_limit_ups") >= int( + params.get("min_boards", 2) + ) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + entry_signal_ids=("signal_limit_up",), + ) + + +MATRIX_STRATEGY = ConsecutiveLimitUpsMatrixStrategy() diff --git a/backend/app/strategy/builtin/high_turnover_surge.py b/backend/app/strategy/builtin/high_turnover_surge.py index 710d396..e4eaee2 100644 --- a/backend/app/strategy/builtin/high_turnover_surge.py +++ b/backend/app/strategy/builtin/high_turnover_surge.py @@ -1,20 +1,45 @@ """高换手拉升 — 换手率 > 5% 且涨幅 > 3%, 资金活跃""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "high_turnover_surge", "name": "高换手拉升", "description": "换手率 > 5% 且涨幅 > 3%, 资金活跃", "tags": ["换手率", "放量", "资金"], + "asset_types": ["stock"], + "timeframes": ["1d"], "params": [ - {"id": "use_turnover_filter", "label": "启用换手率过滤", "type": "bool", - "default": True}, - {"id": "min_turnover", "label": "最低换手率%", "type": "float", - "default": 5.0, "min": 1.0, "max": 20.0, "step": 0.5}, - {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", - "default": True}, - {"id": "min_change", "label": "最低涨幅%", "type": "float", - "default": 3.0, "min": 1.0, "max": 10.0, "step": 0.5}, + {"id": "use_turnover_filter", "label": "启用换手率过滤", "type": "bool", "default": True}, + { + "id": "min_turnover", + "label": "最低换手率%", + "type": "float", + "default": 5.0, + "min": 1.0, + "max": 20.0, + "step": 0.5, + }, + {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", "default": True}, + { + "id": "min_change", + "label": "最低涨幅%", + "type": "float", + "default": 3.0, + "min": 1.0, + "max": 10.0, + "step": 0.5, + }, ], "scoring": {"turnover_rate": 0.4, "change_pct": 0.3, "momentum_5d": 0.3}, "order_by": "score", @@ -22,6 +47,7 @@ META = { "limit": 50, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_volume_surge"] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.05 @@ -29,12 +55,35 @@ MAX_HOLD_DAYS = 10 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - min_to = params.get("min_turnover", 5.0) - min_chg = params.get("min_change", 3.0) / 100.0 - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("use_turnover_filter", True): - expr = expr & (pl.col("turnover_rate") > min_to) - if params.get("use_change_filter", True): - expr = expr & (pl.col("change_pct") > min_chg) - return expr +class HighTurnoverSurgeMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close", "turnover_rate"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + entry = np.ones(market.shape, dtype=bool) + if params.get("use_turnover_filter", True): + entry &= matrix_feature(market, "turnover_rate") > float( + params.get("min_turnover", 5.0) + ) + if params.get("use_change_filter", True): + entry &= ( + matrix_feature(market, "change_pct") > float(params.get("min_change", 3.0)) / 100.0 + ) + ma20 = matrix_feature(market, "ma20") + exit_ = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + entry_signal_ids=("signal_volume_surge",), + exit_signal_ids=("signal_ma20_breakdown",), + ) + + +MATRIX_STRATEGY = HighTurnoverSurgeMatrixStrategy() diff --git a/backend/app/strategy/builtin/limit_up_momentum.py b/backend/app/strategy/builtin/limit_up_momentum.py index 6b653a8..7f4b52a 100644 --- a/backend/app/strategy/builtin/limit_up_momentum.py +++ b/backend/app/strategy/builtin/limit_up_momentum.py @@ -1,20 +1,37 @@ """连板接力 — 近2日涨停且今日涨幅 > 5%, 连板股追踪""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import MarketDataMatrix, SignalMatrix, make_signal_matrix, matrix_feature META = { "id": "limit_up_momentum", "name": "连板接力", "description": "连板股 + 今日涨幅 > 5%, 连板接力追踪", "tags": ["涨停", "连板", "接力"], + "asset_types": ["stock"], + "timeframes": ["1d"], "params": [ - {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", - "default": True}, - {"id": "min_change", "label": "最低涨幅%", "type": "float", - "default": 5.0, "min": 2.0, "max": 15.0, "step": 0.5}, - {"id": "use_boards_filter", "label": "启用连板数过滤", "type": "bool", - "default": True}, - {"id": "min_boards", "label": "最少连板", "type": "int", - "default": 1, "min": 1, "max": 10, "step": 1}, + {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", "default": True}, + { + "id": "min_change", + "label": "最低涨幅%", + "type": "float", + "default": 5.0, + "min": 2.0, + "max": 15.0, + "step": 0.5, + }, + {"id": "use_boards_filter", "label": "启用连板数过滤", "type": "bool", "default": True}, + { + "id": "min_boards", + "label": "最少连板", + "type": "int", + "default": 1, + "min": 1, + "max": 10, + "step": 1, + }, ], "scoring": {"consecutive_limit_ups": 0.4, "change_pct": 0.3, "amount": 0.3}, "order_by": "score", @@ -22,6 +39,7 @@ META = { "limit": 50, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_limit_up"] EXIT_SIGNALS = [] STOP_LOSS = -0.05 @@ -29,12 +47,30 @@ MAX_HOLD_DAYS = 5 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - min_chg = params.get("min_change", 5.0) / 100.0 - min_boards = params.get("min_boards", 1) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("use_change_filter", True): - expr = expr & (pl.col("change_pct") > min_chg) - if params.get("use_boards_filter", True): - expr = expr & (pl.col("consecutive_limit_ups") >= min_boards) - return expr +class LimitUpMomentumMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close", "consecutive_limit_ups"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + entry = np.ones(market.shape, dtype=bool) + if params.get("use_change_filter", True): + entry &= ( + matrix_feature(market, "change_pct") > float(params.get("min_change", 5.0)) / 100.0 + ) + if params.get("use_boards_filter", True): + entry &= matrix_feature(market, "consecutive_limit_ups") >= int( + params.get("min_boards", 1) + ) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + entry_signal_ids=("signal_limit_up",), + ) + + +MATRIX_STRATEGY = LimitUpMomentumMatrixStrategy() diff --git a/backend/app/strategy/builtin/low_volatility_leader.py b/backend/app/strategy/builtin/low_volatility_leader.py index 7232514..f750f49 100644 --- a/backend/app/strategy/builtin/low_volatility_leader.py +++ b/backend/app/strategy/builtin/low_volatility_leader.py @@ -1,20 +1,47 @@ """低波动龙头 — 正动量 + 低波动 + MA20上方""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "low_volatility_leader", "name": "低波动龙头", "description": "20日动量为正 + 年化波动 < 30% + MA20上方", "tags": ["低波动", "龙头"], + "asset_types": ["stock", "etf"], + "timeframes": ["1d"], "params": [ - {"id": "require_positive_momentum", "label": "要求20日动量为正", "type": "bool", - "default": True}, - {"id": "use_volatility_filter", "label": "启用波动率过滤", "type": "bool", - "default": True}, - {"id": "vol_max", "label": "最大年化波动", "type": "float", - "default": 0.30, "min": 0.05, "max": 1.0, "step": 0.01}, - {"id": "require_above_ma20", "label": "要求收盘价在MA20上方", "type": "bool", - "default": True}, + { + "id": "require_positive_momentum", + "label": "要求20日动量为正", + "type": "bool", + "default": True, + }, + {"id": "use_volatility_filter", "label": "启用波动率过滤", "type": "bool", "default": True}, + { + "id": "vol_max", + "label": "最大年化波动", + "type": "float", + "default": 0.30, + "min": 0.05, + "max": 1.0, + "step": 0.01, + }, + { + "id": "require_above_ma20", + "label": "要求收盘价在MA20上方", + "type": "bool", + "default": True, + }, ], "scoring": {"momentum_60d": 0.4, "momentum_20d": 0.3, "turnover_rate": 0.3}, "order_by": "score", @@ -22,6 +49,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_ma20_breakout"] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.05 @@ -29,13 +57,33 @@ MAX_HOLD_DAYS = 30 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - vol_max = params.get("vol_max", 0.30) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("require_positive_momentum", True): - expr = expr & (pl.col("momentum_20d") > 0) - if params.get("use_volatility_filter", True): - expr = expr & (pl.col("annual_vol_20d") < vol_max) - if params.get("require_above_ma20", True): - expr = expr & (pl.col("close") > pl.col("ma20")) - return expr +class LowVolatilityLeaderMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + ma20 = matrix_feature(market, "ma20") + entry = np.ones(market.shape, dtype=bool) + if params.get("require_positive_momentum", True): + entry &= matrix_feature(market, "momentum_20d") > 0 + if params.get("use_volatility_filter", True): + entry &= matrix_feature(market, "annual_vol_20d") < float(params.get("vol_max", 0.30)) + if params.get("require_above_ma20", True): + entry &= market.close > ma20 + exit_ = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + entry_signal_ids=("signal_ma20_breakout",), + exit_signal_ids=("signal_ma20_breakdown",), + ) + + +MATRIX_STRATEGY = LowVolatilityLeaderMatrixStrategy() diff --git a/backend/app/strategy/builtin/ma_golden_cross.py b/backend/app/strategy/builtin/ma_golden_cross.py index 6fb90ee..fe4f57b 100644 --- a/backend/app/strategy/builtin/ma_golden_cross.py +++ b/backend/app/strategy/builtin/ma_golden_cross.py @@ -1,20 +1,42 @@ """MA金叉 — MA5上穿MA20 + 量能配合 + MA60上方""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "ma_golden_cross", "name": "MA 金叉", "description": "MA5上穿MA20当日触发, 量能配合", "tags": ["均线", "金叉"], + "asset_types": ["stock", "etf"], + "timeframes": ["1d"], "params": [ - {"id": "require_ma_golden", "label": "要求MA5上穿MA20", "type": "bool", - "default": True}, - {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", - "default": True}, - {"id": "vol_ratio_min", "label": "最低量比", "type": "float", - "default": 1.2, "min": 0.5, "max": 5.0, "step": 0.1}, - {"id": "require_above_ma60", "label": "要求收盘价在MA60上方", "type": "bool", - "default": True}, + {"id": "require_ma_golden", "label": "要求MA5上穿MA20", "type": "bool", "default": True}, + {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", "default": True}, + { + "id": "vol_ratio_min", + "label": "最低量比", + "type": "float", + "default": 1.2, + "min": 0.5, + "max": 5.0, + "step": 0.1, + }, + { + "id": "require_above_ma60", + "label": "要求收盘价在MA60上方", + "type": "bool", + "default": True, + }, ], "scoring": {"momentum_20d": 0.5, "vol_ratio_5d": 0.3, "change_pct": 0.2}, "order_by": "score", @@ -22,6 +44,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_ma_golden_5_20"] EXIT_SIGNALS = ["signal_ma_dead_5_20"] STOP_LOSS = -0.06 @@ -29,13 +52,37 @@ MAX_HOLD_DAYS = 15 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - vol_min = params.get("vol_ratio_min", 1.2) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("require_ma_golden", True): - expr = expr & pl.col("signal_ma_golden_5_20").fill_null(False) - if params.get("use_volume_filter", True): - expr = expr & (pl.col("vol_ratio_5d") >= vol_min) - if params.get("require_above_ma60", True): - expr = expr & (pl.col("close") > pl.col("ma60")) - return expr +class MAGoldenCrossMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close", "volume"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + ma5 = matrix_feature(market, "ma5") + ma20 = matrix_feature(market, "ma20") + golden = (ma5 > ma20) & (shift(ma5, 1) <= shift(ma20, 1)) + dead = (ma5 < ma20) & (shift(ma5, 1) >= shift(ma20, 1)) + entry = np.ones(market.shape, dtype=bool) + if params.get("require_ma_golden", True): + entry &= golden + if params.get("use_volume_filter", True): + entry &= matrix_feature(market, "vol_ratio_5d") >= float( + params.get("vol_ratio_min", 1.2) + ) + if params.get("require_above_ma60", True): + entry &= market.close > matrix_feature(market, "ma60") + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=dead.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(dead, 0, -1).astype(np.int16), + entry_signal_ids=("signal_ma_golden_5_20",), + exit_signal_ids=("signal_ma_dead_5_20",), + ) + + +MATRIX_STRATEGY = MAGoldenCrossMatrixStrategy() diff --git a/backend/app/strategy/builtin/macd_golden.py b/backend/app/strategy/builtin/macd_golden.py index 42a05cb..0785ef7 100644 --- a/backend/app/strategy/builtin/macd_golden.py +++ b/backend/app/strategy/builtin/macd_golden.py @@ -1,18 +1,39 @@ """MACD金叉放量 — MACD金叉当日 + 量能放大""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_ewm_adjust_false as ewm_adjust_false, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "macd_golden", "name": "MACD 金叉放量", "description": "MACD金叉当日 + 量能放大", "tags": ["MACD", "金叉", "放量"], + "asset_types": ["stock", "etf"], + "timeframes": ["1d"], "params": [ - {"id": "require_macd_golden", "label": "要求MACD金叉", "type": "bool", - "default": True}, - {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", - "default": True}, - {"id": "vol_ratio_min", "label": "最低量比", "type": "float", - "default": 1.5, "min": 0.5, "max": 5.0, "step": 0.1}, + {"id": "require_macd_golden", "label": "要求MACD金叉", "type": "bool", "default": True}, + {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", "default": True}, + { + "id": "vol_ratio_min", + "label": "最低量比", + "type": "float", + "default": 1.5, + "min": 0.5, + "max": 5.0, + "step": 0.1, + }, ], "scoring": {"momentum_60d": 0.4, "vol_ratio_5d": 0.3, "change_pct": 0.3}, "order_by": "score", @@ -22,16 +43,58 @@ META = { ENTRY_SIGNALS = ["signal_macd_golden"] EXIT_SIGNALS = ["signal_macd_dead"] +EXECUTION_BACKEND = "matrix_native" STOP_LOSS = -0.07 MAX_HOLD_DAYS = 20 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - vol_min = params.get("vol_ratio_min", 1.5) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("require_macd_golden", True): - expr = expr & pl.col("signal_macd_golden").fill_null(False) - if params.get("use_volume_filter", True): - expr = expr & (pl.col("vol_ratio_5d") >= vol_min) - return expr +class MACDGoldenMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close", "volume"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals( + self, + market: MarketDataMatrix, + params: dict, + ) -> SignalMatrix: + valid = np.isfinite(market.close) + ema12 = ewm_adjust_false(market.close, valid, span=12) + ema26 = ewm_adjust_false(market.close, valid, span=26) + dif = ema12 - ema26 + dif_valid = np.isfinite(dif) + dea = ewm_adjust_false(dif, dif_valid, span=9) + previous_dif = shift(dif, 1, dif_valid) + previous_dea = shift(dea, 1, np.isfinite(dea)) + golden = (dif > dea) & (previous_dif <= previous_dea) + dead = (dif < dea) & (previous_dif >= previous_dea) + + entry = ( + golden + if params.get("require_macd_golden", True) + else np.ones( + market.shape, + dtype=bool, + ) + ) + if params.get("use_volume_filter", True): + entry &= matrix_feature(market, "vol_ratio_5d") >= float( + params.get("vol_ratio_min", 1.5) + ) + + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=dead.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(dead, 0, -1).astype(np.int16), + entry_signal_ids=("signal_macd_golden",), + exit_signal_ids=("signal_macd_dead",), + ) + + +MATRIX_STRATEGY = MACDGoldenMatrixStrategy() diff --git a/backend/app/strategy/builtin/n_day_low_reversal.py b/backend/app/strategy/builtin/n_day_low_reversal.py index 662c31a..f263c22 100644 --- a/backend/app/strategy/builtin/n_day_low_reversal.py +++ b/backend/app/strategy/builtin/n_day_low_reversal.py @@ -1,20 +1,37 @@ """新低反转 — 60日新低后收阳放量""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "n_day_low_reversal", "name": "新低反转", "description": "触及60日新低后当日收阳放量, 反转信号", "tags": ["反转", "新低"], + "asset_types": ["stock", "etf"], + "timeframes": ["1d"], "params": [ - {"id": "require_n_day_low", "label": "要求60日新低", "type": "bool", - "default": True}, - {"id": "require_bullish_candle", "label": "要求收阳", "type": "bool", - "default": True}, - {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", - "default": True}, - {"id": "vol_ratio_min", "label": "最低量比", "type": "float", - "default": 1.5, "min": 0.5, "max": 5.0, "step": 0.1}, + {"id": "require_n_day_low", "label": "要求60日新低", "type": "bool", "default": True}, + {"id": "require_bullish_candle", "label": "要求收阳", "type": "bool", "default": True}, + {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", "default": True}, + { + "id": "vol_ratio_min", + "label": "最低量比", + "type": "float", + "default": 1.5, + "min": 0.5, + "max": 5.0, + "step": 0.1, + }, ], "scoring": {"change_pct": 0.4, "vol_ratio_5d": 0.3, "momentum_5d": 0.3}, "order_by": "score", @@ -22,6 +39,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_n_day_low"] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.06 @@ -29,13 +47,35 @@ MAX_HOLD_DAYS = 15 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - vol_min = params.get("vol_ratio_min", 1.5) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("require_n_day_low", True): - expr = expr & pl.col("signal_n_day_low").fill_null(False) - if params.get("require_bullish_candle", True): - expr = expr & (pl.col("close") > pl.col("open")) - if params.get("use_volume_filter", True): - expr = expr & (pl.col("vol_ratio_5d") >= vol_min) - return expr +class NDayLowReversalMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"open", "close", "volume"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + entry = np.ones(market.shape, dtype=bool) + if params.get("require_n_day_low", True): + entry &= market.close <= matrix_feature(market, "low_60d") + if params.get("require_bullish_candle", True): + entry &= market.close > market.open + if params.get("use_volume_filter", True): + entry &= matrix_feature(market, "vol_ratio_5d") >= float( + params.get("vol_ratio_min", 1.5) + ) + ma20 = matrix_feature(market, "ma20") + exit_ = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + entry_signal_ids=("signal_n_day_low",), + exit_signal_ids=("signal_ma20_breakdown",), + ) + + +MATRIX_STRATEGY = NDayLowReversalMatrixStrategy() diff --git a/backend/app/strategy/builtin/near_limit_up.py b/backend/app/strategy/builtin/near_limit_up.py index 1b933cf..519d64d 100644 --- a/backend/app/strategy/builtin/near_limit_up.py +++ b/backend/app/strategy/builtin/near_limit_up.py @@ -1,41 +1,50 @@ """逼近涨停 — 涨幅 > 7% 且距涨停 < 3%, 盘后选股""" -import polars as pl +import numpy as np -def _limit_pct() -> pl.Expr: - """根据板块和 ST 动态计算涨跌幅限制 (小数)。 - 创业板(300/301)/科创板(688): 20% (含其 ST) - 北交所(.BJ): 30% - 主板 ST: 5% ← ST 5% 仅主板生效, 创业板/科创板 ST 仍是 20% - 主板普通: 10% - """ - is_st = pl.col("name").str.contains("(?i)ST").fill_null(False) - is_cyb = pl.col("symbol").str.starts_with("300") | pl.col("symbol").str.starts_with("301") - is_kcb = pl.col("symbol").str.starts_with("688") - is_bj = pl.col("symbol").str.contains(r"\.BJ$") - return ( - # 板块判定优先于 ST: 创业板/科创板 ST 保留 20%, 北交所 30%; ST 5% 只剩主板 - pl.when(is_cyb | is_kcb).then(0.20) - .when(is_bj).then(0.30) - .when(is_st).then(0.05) - .otherwise(0.10) - ) - +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "near_limit_up", "name": "逼近涨停", "description": "涨幅 > 7% 且距涨停 < 3%, 追涨信号", "tags": ["涨停", "追涨"], + "asset_types": ["stock"], + "timeframes": ["1d"], "params": [ - {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", - "default": True}, - {"id": "min_change", "label": "最低涨幅%", "type": "float", - "default": 7.0, "min": 3.0, "max": 15.0, "step": 1.0}, - {"id": "use_limit_gap_filter", "label": "启用距涨停空间过滤", "type": "bool", - "default": True}, - {"id": "limit_gap", "label": "距涨停空间%", "type": "float", - "default": 3.0, "min": 1.0, "max": 10.0, "step": 0.5}, + {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", "default": True}, + { + "id": "min_change", + "label": "最低涨幅%", + "type": "float", + "default": 7.0, + "min": 3.0, + "max": 15.0, + "step": 1.0, + }, + { + "id": "use_limit_gap_filter", + "label": "启用距涨停空间过滤", + "type": "bool", + "default": True, + }, + { + "id": "limit_gap", + "label": "距涨停空间%", + "type": "float", + "default": 3.0, + "min": 1.0, + "max": 10.0, + "step": 0.5, + }, ], "scoring": {"change_pct": 0.5, "amount": 0.3, "momentum_5d": 0.2}, "order_by": "score", @@ -43,6 +52,7 @@ META = { "limit": 50, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = [] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.05 @@ -50,13 +60,46 @@ MAX_HOLD_DAYS = 5 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - min_chg = params.get("min_change", 7.0) / 100.0 - gap = params.get("limit_gap", 3.0) / 100.0 - lp = _limit_pct() - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("use_change_filter", True): - expr = expr & (pl.col("change_pct") > min_chg) - if params.get("use_limit_gap_filter", True): - expr = expr & (pl.col("change_pct") < lp - gap) - return expr +class NearLimitUpMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + @staticmethod + def _limit_pct(market: MarketDataMatrix) -> np.ndarray: + values = np.full(len(market.symbols), 0.10, dtype=np.float32) + for asset_id, (symbol, name) in enumerate(zip(market.symbols, market.names, strict=True)): + if symbol.startswith(("300", "301", "688")): + values[asset_id] = 0.20 + elif symbol.endswith(".BJ"): + values[asset_id] = 0.30 + elif "ST" in name.upper(): + values[asset_id] = 0.05 + return values + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + change = matrix_feature(market, "change_pct") + entry = np.ones(market.shape, dtype=bool) + if params.get("use_change_filter", True): + entry &= change > float(params.get("min_change", 7.0)) / 100.0 + if params.get("use_limit_gap_filter", True): + entry &= ( + change + < self._limit_pct(market)[None, :] - float(params.get("limit_gap", 3.0)) / 100.0 + ) + ma20 = matrix_feature(market, "ma20") + exit_ = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + exit_signal_ids=("signal_ma20_breakdown",), + ) + + +MATRIX_STRATEGY = NearLimitUpMatrixStrategy() diff --git a/backend/app/strategy/builtin/oversold_bounce.py b/backend/app/strategy/builtin/oversold_bounce.py index e66fb49..c187996 100644 --- a/backend/app/strategy/builtin/oversold_bounce.py +++ b/backend/app/strategy/builtin/oversold_bounce.py @@ -1,22 +1,46 @@ """超跌反弹 — RSI14 < 30 + 收阳 + 放量""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "oversold_bounce", "name": "超跌反弹", "description": "RSI14 < 30超卖区 + 当日收阳 + 放量, 抄底信号", "tags": ["超跌", "反弹", "RSI"], + "asset_types": ["stock", "etf"], + "timeframes": ["1d"], "params": [ - {"id": "use_rsi_filter", "label": "启用RSI过滤", "type": "bool", - "default": True}, - {"id": "rsi_max", "label": "RSI上限", "type": "float", - "default": 30.0, "min": 10.0, "max": 50.0, "step": 1.0}, - {"id": "require_bullish_candle", "label": "要求收阳", "type": "bool", - "default": True}, - {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", - "default": True}, - {"id": "vol_ratio_min", "label": "最低量比", "type": "float", - "default": 1.2, "min": 0.5, "max": 5.0, "step": 0.1}, + {"id": "use_rsi_filter", "label": "启用RSI过滤", "type": "bool", "default": True}, + { + "id": "rsi_max", + "label": "RSI上限", + "type": "float", + "default": 30.0, + "min": 10.0, + "max": 50.0, + "step": 1.0, + }, + {"id": "require_bullish_candle", "label": "要求收阳", "type": "bool", "default": True}, + {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", "default": True}, + { + "id": "vol_ratio_min", + "label": "最低量比", + "type": "float", + "default": 1.2, + "min": 0.5, + "max": 5.0, + "step": 0.1, + }, ], "scoring": {"change_pct": 0.3, "vol_ratio_5d": 0.3, "momentum_5d": 0.2, "rsi_14": 0.2}, "order_by": "score", @@ -24,6 +48,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = [] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.05 @@ -33,14 +58,34 @@ ALERTS = [ ] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - rsi_max = params.get("rsi_max", 30.0) - vol_min = params.get("vol_ratio_min", 1.2) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("use_rsi_filter", True): - expr = expr & (pl.col("rsi_14") < rsi_max) - if params.get("require_bullish_candle", True): - expr = expr & (pl.col("close") > pl.col("open")) - if params.get("use_volume_filter", True): - expr = expr & (pl.col("vol_ratio_5d") >= vol_min) - return expr +class OversoldBounceMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"open", "close", "volume"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + entry = np.ones(market.shape, dtype=bool) + if params.get("use_rsi_filter", True): + entry &= matrix_feature(market, "rsi_14") < float(params.get("rsi_max", 30.0)) + if params.get("require_bullish_candle", True): + entry &= market.close > market.open + if params.get("use_volume_filter", True): + entry &= matrix_feature(market, "vol_ratio_5d") >= float( + params.get("vol_ratio_min", 1.2) + ) + ma20 = matrix_feature(market, "ma20") + exit_ = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + exit_signal_ids=("signal_ma20_breakdown",), + ) + + +MATRIX_STRATEGY = OversoldBounceMatrixStrategy() diff --git a/backend/app/strategy/builtin/oversold_reversal.py b/backend/app/strategy/builtin/oversold_reversal.py index 5c1b382..4334e63 100644 --- a/backend/app/strategy/builtin/oversold_reversal.py +++ b/backend/app/strategy/builtin/oversold_reversal.py @@ -1,22 +1,51 @@ """超跌反弹 — RSI14 < 30 + 涨幅 > 1% + 站上 MA5, 超卖反弹信号""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "oversold_reversal", "name": "超跌反转", "description": "RSI14 < 30超卖 + 涨幅 > 1% + 站上MA5, 超卖反转信号", "tags": ["超跌", "反弹", "RSI"], + "asset_types": ["stock"], + "timeframes": ["1d"], "params": [ - {"id": "use_rsi_filter", "label": "启用RSI过滤", "type": "bool", - "default": True}, - {"id": "rsi_max", "label": "RSI上限", "type": "float", - "default": 30.0, "min": 10.0, "max": 50.0, "step": 1.0}, - {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", - "default": True}, - {"id": "min_change", "label": "最低涨幅%", "type": "float", - "default": 1.0, "min": 0.5, "max": 5.0, "step": 0.5}, - {"id": "require_above_ma5", "label": "要求收盘价在MA5上方", "type": "bool", - "default": True}, + {"id": "use_rsi_filter", "label": "启用RSI过滤", "type": "bool", "default": True}, + { + "id": "rsi_max", + "label": "RSI上限", + "type": "float", + "default": 30.0, + "min": 10.0, + "max": 50.0, + "step": 1.0, + }, + {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", "default": True}, + { + "id": "min_change", + "label": "最低涨幅%", + "type": "float", + "default": 1.0, + "min": 0.5, + "max": 5.0, + "step": 0.5, + }, + { + "id": "require_above_ma5", + "label": "要求收盘价在MA5上方", + "type": "bool", + "default": True, + }, ], "scoring": {"change_pct": 0.4, "rsi_14": 0.3, "vol_ratio_5d": 0.3}, "order_by": "score", @@ -24,6 +53,7 @@ META = { "limit": 50, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = [] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.05 @@ -33,14 +63,34 @@ ALERTS = [ ] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - rsi_max = params.get("rsi_max", 30.0) - min_chg = params.get("min_change", 1.0) / 100.0 - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("use_rsi_filter", True): - expr = expr & (pl.col("rsi_14") < rsi_max) - if params.get("use_change_filter", True): - expr = expr & (pl.col("change_pct") > min_chg) - if params.get("require_above_ma5", True): - expr = expr & (pl.col("close") > pl.col("ma5")) - return expr +class OversoldReversalMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + entry = np.ones(market.shape, dtype=bool) + if params.get("use_rsi_filter", True): + entry &= matrix_feature(market, "rsi_14") < float(params.get("rsi_max", 30.0)) + if params.get("use_change_filter", True): + entry &= ( + matrix_feature(market, "change_pct") > float(params.get("min_change", 1.0)) / 100.0 + ) + if params.get("require_above_ma5", True): + entry &= market.close > matrix_feature(market, "ma5") + ma20 = matrix_feature(market, "ma20") + exit_ = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + exit_signal_ids=("signal_ma20_breakdown",), + ) + + +MATRIX_STRATEGY = OversoldReversalMatrixStrategy() diff --git a/backend/app/strategy/builtin/pullback_ma20_bounce.py b/backend/app/strategy/builtin/pullback_ma20_bounce.py index 3ab19ad..e7491c9 100644 --- a/backend/app/strategy/builtin/pullback_ma20_bounce.py +++ b/backend/app/strategy/builtin/pullback_ma20_bounce.py @@ -1,20 +1,42 @@ """均线回踩反弹 — 价格在 MA20 附近(±2%)且 MA 多头排列, 回踩买入""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "pullback_ma20_bounce", "name": "均线回踩反弹", "description": "价格在MA20附近(±2%)且MA5>MA20>MA60多头排列, 回踩买入", "tags": ["回踩", "均线", "反弹"], + "asset_types": ["stock"], + "timeframes": ["1d"], "params": [ - {"id": "use_ma20_proximity", "label": "启用MA20附近过滤", "type": "bool", - "default": True}, - {"id": "ma_proximity", "label": "MA偏离度%", "type": "float", - "default": 2.0, "min": 0.5, "max": 5.0, "step": 0.5}, - {"id": "require_ma_alignment", "label": "要求MA5>MA20>MA60", "type": "bool", - "default": True}, - {"id": "require_positive_change", "label": "要求当日上涨", "type": "bool", - "default": True}, + {"id": "use_ma20_proximity", "label": "启用MA20附近过滤", "type": "bool", "default": True}, + { + "id": "ma_proximity", + "label": "MA偏离度%", + "type": "float", + "default": 2.0, + "min": 0.5, + "max": 5.0, + "step": 0.5, + }, + { + "id": "require_ma_alignment", + "label": "要求MA5>MA20>MA60", + "type": "bool", + "default": True, + }, + {"id": "require_positive_change", "label": "要求当日上涨", "type": "bool", "default": True}, ], "scoring": {"momentum_60d": 0.4, "change_pct": 0.3, "momentum_20d": 0.3}, "order_by": "score", @@ -22,6 +44,7 @@ META = { "limit": 50, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_ma_golden_5_20"] EXIT_SIGNALS = ["signal_ma20_breakdown", "signal_ma_dead_5_20"] STOP_LOSS = -0.05 @@ -29,17 +52,40 @@ MAX_HOLD_DAYS = 15 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - proximity = params.get("ma_proximity", 2.0) / 100.0 - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("use_ma20_proximity", True): - expr = ( - expr - & (pl.col("close") > pl.col("ma20") * (1 - proximity)) - & (pl.col("close") < pl.col("ma20") * (1 + proximity)) +class PullbackMA20BounceMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + ma5 = matrix_feature(market, "ma5") + ma20 = matrix_feature(market, "ma20") + ma60 = matrix_feature(market, "ma60") + entry = np.ones(market.shape, dtype=bool) + if params.get("use_ma20_proximity", True): + proximity = float(params.get("ma_proximity", 2.0)) / 100.0 + entry &= (market.close > ma20 * (1.0 - proximity)) & ( + market.close < ma20 * (1.0 + proximity) + ) + if params.get("require_ma_alignment", True): + entry &= (ma5 > ma20) & (ma20 > ma60) + if params.get("require_positive_change", True): + entry &= matrix_feature(market, "change_pct") > 0 + ma20_breakdown = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + ma_dead = (ma5 < ma20) & (shift(ma5, 1) >= shift(ma20, 1)) + exit_ = ma20_breakdown | ma_dead + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(ma20_breakdown, 0, np.where(ma_dead, 1, -1)).astype(np.int16), + entry_signal_ids=("signal_ma_golden_5_20",), + exit_signal_ids=("signal_ma20_breakdown", "signal_ma_dead_5_20"), ) - if params.get("require_ma_alignment", True): - expr = expr & (pl.col("ma5") > pl.col("ma20")) & (pl.col("ma20") > pl.col("ma60")) - if params.get("require_positive_change", True): - expr = expr & (pl.col("change_pct") > 0) - return expr + + +MATRIX_STRATEGY = PullbackMA20BounceMatrixStrategy() diff --git a/backend/app/strategy/builtin/pullback_to_support.py b/backend/app/strategy/builtin/pullback_to_support.py index 0bed03f..07fe8b1 100644 --- a/backend/app/strategy/builtin/pullback_to_support.py +++ b/backend/app/strategy/builtin/pullback_to_support.py @@ -1,24 +1,57 @@ """缩量回踩 — 回踩MA20附近 + 缩量 + 中期趋势向上""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "pullback_to_support", "name": "缩量回踩", "description": "回踩MA20附近 + 缩量 + 中期趋势向上", "tags": ["回踩", "支撑"], + "asset_types": ["stock", "etf"], + "timeframes": ["1d"], "params": [ - {"id": "use_ma20_proximity", "label": "启用MA20附近过滤", "type": "bool", - "default": True}, - {"id": "ma_proximity", "label": "均线偏离度", "type": "float", - "default": 0.02, "min": 0.01, "max": 0.05, "step": 0.005}, - {"id": "use_volume_filter", "label": "启用缩量过滤", "type": "bool", - "default": True}, - {"id": "vol_ratio_max", "label": "最大量比", "type": "float", - "default": 0.8, "min": 0.2, "max": 1.5, "step": 0.1}, - {"id": "require_above_ma60", "label": "要求收盘价在MA60上方", "type": "bool", - "default": True}, - {"id": "require_positive_momentum", "label": "要求20日动量为正", "type": "bool", - "default": True}, + {"id": "use_ma20_proximity", "label": "启用MA20附近过滤", "type": "bool", "default": True}, + { + "id": "ma_proximity", + "label": "均线偏离度", + "type": "float", + "default": 0.02, + "min": 0.01, + "max": 0.05, + "step": 0.005, + }, + {"id": "use_volume_filter", "label": "启用缩量过滤", "type": "bool", "default": True}, + { + "id": "vol_ratio_max", + "label": "最大量比", + "type": "float", + "default": 0.8, + "min": 0.2, + "max": 1.5, + "step": 0.1, + }, + { + "id": "require_above_ma60", + "label": "要求收盘价在MA60上方", + "type": "bool", + "default": True, + }, + { + "id": "require_positive_momentum", + "label": "要求20日动量为正", + "type": "bool", + "default": True, + }, ], "scoring": {"momentum_60d": 0.4, "momentum_20d": 0.3, "turnover_rate": 0.3}, "order_by": "score", @@ -26,6 +59,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_ma_golden_5_20"] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.05 @@ -33,20 +67,40 @@ MAX_HOLD_DAYS = 20 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - proximity = params.get("ma_proximity", 0.02) - vol_max = params.get("vol_ratio_max", 0.8) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("use_ma20_proximity", True): - expr = ( - expr - & (pl.col("close") > pl.col("ma20") * (1 - proximity)) - & (pl.col("close") < pl.col("ma20") * (1 + proximity)) +class PullbackToSupportMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close", "volume"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + ma20 = matrix_feature(market, "ma20") + entry = np.ones(market.shape, dtype=bool) + if params.get("use_ma20_proximity", True): + proximity = float(params.get("ma_proximity", 0.02)) + entry &= (market.close > ma20 * (1.0 - proximity)) & ( + market.close < ma20 * (1.0 + proximity) + ) + if params.get("use_volume_filter", True): + entry &= matrix_feature(market, "vol_ratio_5d") < float( + params.get("vol_ratio_max", 0.8) + ) + if params.get("require_above_ma60", True): + entry &= market.close > matrix_feature(market, "ma60") + if params.get("require_positive_momentum", True): + entry &= matrix_feature(market, "momentum_20d") > 0 + exit_ = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + entry_signal_ids=("signal_ma_golden_5_20",), + exit_signal_ids=("signal_ma20_breakdown",), ) - if params.get("use_volume_filter", True): - expr = expr & (pl.col("vol_ratio_5d") < vol_max) - if params.get("require_above_ma60", True): - expr = expr & (pl.col("close") > pl.col("ma60")) - if params.get("require_positive_momentum", True): - expr = expr & (pl.col("momentum_20d") > 0) - return expr + + +MATRIX_STRATEGY = PullbackToSupportMatrixStrategy() diff --git a/backend/app/strategy/builtin/strong_open.py b/backend/app/strategy/builtin/strong_open.py index 7d3dd98..537a25e 100644 --- a/backend/app/strategy/builtin/strong_open.py +++ b/backend/app/strategy/builtin/strong_open.py @@ -1,22 +1,51 @@ """强势高开 — 高开 > 3% 且保持上涨, 集合竞价强势""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "strong_open", "name": "强势高开", "description": "高开 > 3% 且收盘高于开盘价, 集合竞价强势", "tags": ["高开", "强势"], + "asset_types": ["stock"], + "timeframes": ["1d"], "params": [ - {"id": "use_open_gap_filter", "label": "启用高开过滤", "type": "bool", - "default": True}, - {"id": "min_open_gap", "label": "最低高开%", "type": "float", - "default": 3.0, "min": 1.0, "max": 10.0, "step": 0.5}, - {"id": "require_close_above_open", "label": "要求收盘高于开盘", "type": "bool", - "default": True}, - {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", - "default": True}, - {"id": "min_change", "label": "最低涨幅%", "type": "float", - "default": 3.0, "min": 1.0, "max": 10.0, "step": 0.5}, + {"id": "use_open_gap_filter", "label": "启用高开过滤", "type": "bool", "default": True}, + { + "id": "min_open_gap", + "label": "最低高开%", + "type": "float", + "default": 3.0, + "min": 1.0, + "max": 10.0, + "step": 0.5, + }, + { + "id": "require_close_above_open", + "label": "要求收盘高于开盘", + "type": "bool", + "default": True, + }, + {"id": "use_change_filter", "label": "启用涨幅过滤", "type": "bool", "default": True}, + { + "id": "min_change", + "label": "最低涨幅%", + "type": "float", + "default": 3.0, + "min": 1.0, + "max": 10.0, + "step": 0.5, + }, ], "scoring": {"change_pct": 0.4, "amplitude": 0.2, "amount": 0.4}, "order_by": "score", @@ -24,6 +53,7 @@ META = { "limit": 50, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = [] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.05 @@ -31,14 +61,36 @@ MAX_HOLD_DAYS = 10 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - min_gap = params.get("min_open_gap", 3.0) / 100.0 - min_chg = params.get("min_change", 3.0) / 100.0 - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("use_open_gap_filter", True): - expr = expr & (pl.col("open") > pl.col("prev_close") * (1 + min_gap)) - if params.get("require_close_above_open", True): - expr = expr & (pl.col("close") > pl.col("open")) - if params.get("use_change_filter", True): - expr = expr & (pl.col("change_pct") > min_chg) - return expr +class StrongOpenMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"open", "close"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + entry = np.ones(market.shape, dtype=bool) + if params.get("use_open_gap_filter", True): + entry &= market.open > shift(market.close, 1) * ( + 1.0 + float(params.get("min_open_gap", 3.0)) / 100.0 + ) + if params.get("require_close_above_open", True): + entry &= market.close > market.open + if params.get("use_change_filter", True): + entry &= ( + matrix_feature(market, "change_pct") > float(params.get("min_change", 3.0)) / 100.0 + ) + ma20 = matrix_feature(market, "ma20") + exit_ = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + exit_signal_ids=("signal_ma20_breakdown",), + ) + + +MATRIX_STRATEGY = StrongOpenMatrixStrategy() diff --git a/backend/app/strategy/builtin/trend_breakout.py b/backend/app/strategy/builtin/trend_breakout.py index c00c325..a2824ff 100644 --- a/backend/app/strategy/builtin/trend_breakout.py +++ b/backend/app/strategy/builtin/trend_breakout.py @@ -1,11 +1,24 @@ """趋势突破 — MA60上方 + 60日新高 + 放量""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "trend_breakout", "name": "趋势突破", "description": "MA60上方 + 60日新高 + 量能 ≥ 2倍均量", "tags": ["趋势", "突破", "放量"], + "asset_types": ["stock", "etf"], + "timeframes": ["1d"], "basic_filter": { "price_min": 5, "price_max": 200, @@ -15,14 +28,23 @@ META = { "exclude_new_days": 60, }, "params": [ - {"id": "require_above_ma60", "label": "要求收盘价在MA60上方", "type": "bool", - "default": True}, - {"id": "require_n_day_high", "label": "要求60日新高", "type": "bool", - "default": True}, - {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", - "default": True}, - {"id": "vol_ratio_min", "label": "最低量比", "type": "float", - "default": 2.0, "min": 0.5, "max": 10.0, "step": 0.1}, + { + "id": "require_above_ma60", + "label": "要求收盘价在MA60上方", + "type": "bool", + "default": True, + }, + {"id": "require_n_day_high", "label": "要求60日新高", "type": "bool", "default": True}, + {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", "default": True}, + { + "id": "vol_ratio_min", + "label": "最低量比", + "type": "float", + "default": 2.0, + "min": 0.5, + "max": 10.0, + "step": 0.1, + }, ], "scoring": {"momentum_60d": 0.4, "vol_ratio_5d": 0.3, "change_pct": 0.3}, "order_by": "score", @@ -30,6 +52,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_n_day_high"] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.08 @@ -39,13 +62,35 @@ ALERTS = [ ] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - vol_min = params.get("vol_ratio_min", 2.0) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("require_above_ma60", True): - expr = expr & (pl.col("close") > pl.col("ma60")) - if params.get("require_n_day_high", True): - expr = expr & pl.col("signal_n_day_high").fill_null(False) - if params.get("use_volume_filter", True): - expr = expr & (pl.col("vol_ratio_5d") >= vol_min) - return expr +class TrendBreakoutMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close", "volume"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + entry = np.ones(market.shape, dtype=bool) + if params.get("require_above_ma60", True): + entry &= market.close > matrix_feature(market, "ma60") + if params.get("require_n_day_high", True): + entry &= market.close >= matrix_feature(market, "high_60d") + if params.get("use_volume_filter", True): + entry &= matrix_feature(market, "vol_ratio_5d") >= float( + params.get("vol_ratio_min", 2.0) + ) + ma20 = matrix_feature(market, "ma20") + exit_ = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + entry_signal_ids=("signal_n_day_high",), + exit_signal_ids=("signal_ma20_breakdown",), + ) + + +MATRIX_STRATEGY = TrendBreakoutMatrixStrategy() diff --git a/backend/app/strategy/builtin/volume_price_surge.py b/backend/app/strategy/builtin/volume_price_surge.py index 47dc475..ec70ac0 100644 --- a/backend/app/strategy/builtin/volume_price_surge.py +++ b/backend/app/strategy/builtin/volume_price_surge.py @@ -1,20 +1,37 @@ """量价齐升 — 突破MA20 + 放量 + 收阳""" -import polars as pl + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) +from app.backtest.matrix import ( + valid_shift as shift, +) META = { "id": "volume_price_surge", "name": "量价齐升", "description": "突破MA20 + 放量 + 收阳", "tags": ["量价", "突破"], + "asset_types": ["stock", "etf"], + "timeframes": ["1d"], "params": [ - {"id": "require_ma20_breakout", "label": "要求突破MA20", "type": "bool", - "default": True}, - {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", - "default": True}, - {"id": "vol_ratio_min", "label": "最低量比", "type": "float", - "default": 2.0, "min": 0.5, "max": 10.0, "step": 0.1}, - {"id": "require_bullish_candle", "label": "要求收阳", "type": "bool", - "default": True}, + {"id": "require_ma20_breakout", "label": "要求突破MA20", "type": "bool", "default": True}, + {"id": "use_volume_filter", "label": "启用量比过滤", "type": "bool", "default": True}, + { + "id": "vol_ratio_min", + "label": "最低量比", + "type": "float", + "default": 2.0, + "min": 0.5, + "max": 10.0, + "step": 0.1, + }, + {"id": "require_bullish_candle", "label": "要求收阳", "type": "bool", "default": True}, ], "scoring": {"vol_ratio_5d": 0.4, "change_pct": 0.3, "momentum_20d": 0.3}, "order_by": "score", @@ -22,6 +39,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "matrix_native" ENTRY_SIGNALS = ["signal_ma20_breakout"] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.06 @@ -29,13 +47,36 @@ MAX_HOLD_DAYS = 15 ALERTS = [] -def filter(df: pl.DataFrame, params: dict) -> pl.Expr: - vol_min = params.get("vol_ratio_min", 2.0) - expr = pl.col("symbol").is_not_null() | pl.col("symbol").is_null() - if params.get("require_ma20_breakout", True): - expr = expr & pl.col("signal_ma20_breakout").fill_null(False) - if params.get("use_volume_filter", True): - expr = expr & (pl.col("vol_ratio_5d") >= vol_min) - if params.get("require_bullish_candle", True): - expr = expr & (pl.col("close") > pl.col("open")) - return expr +class VolumePriceSurgeMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"open", "close", "volume"}) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + ma20 = matrix_feature(market, "ma20") + breakout = (market.close > ma20) & (shift(market.close, 1) <= shift(ma20, 1)) + breakdown = (market.close < ma20) & (shift(market.close, 1) >= shift(ma20, 1)) + entry = np.ones(market.shape, dtype=bool) + if params.get("require_ma20_breakout", True): + entry &= breakout + if params.get("use_volume_filter", True): + entry &= matrix_feature(market, "vol_ratio_5d") >= float( + params.get("vol_ratio_min", 2.0) + ) + if params.get("require_bullish_candle", True): + entry &= market.close > market.open + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=breakdown.astype(np.uint8), + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(breakdown, 0, -1).astype(np.int16), + entry_signal_ids=("signal_ma20_breakout",), + exit_signal_ids=("signal_ma20_breakdown",), + ) + + +MATRIX_STRATEGY = VolumePriceSurgeMatrixStrategy() diff --git a/backend/app/strategy/custom_signals.py b/backend/app/strategy/custom_signals.py index 31669d6..eb957c4 100644 --- a/backend/app/strategy/custom_signals.py +++ b/backend/app/strategy/custom_signals.py @@ -214,17 +214,39 @@ def build_expressions(signals: list[dict], allow_shift: bool = True) -> dict[str return out -def inject(df: pl.DataFrame, exprs: dict[str, pl.Expr]) -> pl.DataFrame: - """把编译好的信号表达式作为列加入 df。仅添加 df 已含其依赖列的信号。""" +def expression_dependencies(exprs: dict[str, pl.Expr] | None = None) -> dict[str, frozenset[str]]: + """返回自定义信号列到根字段的依赖映射。""" + source = exprs if exprs is not None else {} + return {name: frozenset(_expr_root_columns(expr)) for name, expr in source.items()} + + +def inject( + df: pl.DataFrame, + exprs: dict[str, pl.Expr], + needed: set[str] | None = None, +) -> pl.DataFrame: + """把编译好的信号表达式作为列加入 df。 + + ``needed=None`` 保持历史全量语义；传入集合时只注入被请求的自定义信号。 + 缺失依赖会明确告警，避免回测静默丢失信号。 + """ if df.is_empty() or not exprs: return df cols = set(df.columns) add: dict[str, pl.Expr] = {} for name, expr in exprs.items(): + if needed is not None and name not in needed: + continue # 提取该表达式引用的所有字段列，缺失则跳过（避免运行时报错） - needed = _expr_root_columns(expr) - if needed.issubset(cols): + required = _expr_root_columns(expr) + if required.issubset(cols): add[name] = expr + else: + logger.warning( + "custom signal %s missing dependencies: %s", + name, + sorted(required - cols), + ) if add: df = df.with_columns([e.alias(n) for n, e in add.items()]) return df diff --git a/backend/app/strategy/engine.py b/backend/app/strategy/engine.py index 63ce7df..1220681 100644 --- a/backend/app/strategy/engine.py +++ b/backend/app/strategy/engine.py @@ -8,12 +8,15 @@ from __future__ import annotations import importlib.util import logging +import sys +import threading import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import date from pathlib import Path from typing import Any, Callable +import numpy as np import polars as pl logger = logging.getLogger(__name__) @@ -93,6 +96,19 @@ def _normalize_param_item(item: dict) -> dict: return norm +@dataclass +class StrategyDataContext: + """一次策略调用所需的标准数据上下文。""" + + asset_type: str + timeframe: str + as_of: date + current: pl.DataFrame | None = None + history: pl.DataFrame | None = None + market: Any | None = None + cache_key: str | None = None + + @dataclass class StrategyDef: """加载后的策略定义（只读数据 + filter 函数引用）""" @@ -110,7 +126,10 @@ class StrategyDef: filter_history_fn: Callable[[pl.DataFrame, dict], pl.DataFrame] | None lookback_days: int source: str # "builtin" | "custom" | "ai" + required_features: frozenset[str] = field(default_factory=frozenset) file_path: Path | None = None + execution_backend: str = "polars_expr" + matrix_strategy: Any | None = None @dataclass @@ -124,31 +143,34 @@ class StrategyResult: scores: dict[str, float] = field(default_factory=dict) +@dataclass +class _RealtimeMatrixEntry: + fingerprint: tuple[Any, ...] + buffer: Any + + class StrategyEngine: """策略引擎 — 策略加载 + 执行 + 评分""" - def __init__(self, enriched_loader: Callable[[date], pl.DataFrame], - enriched_history_loader: Callable[[date, int], pl.DataFrame] | None = None, - strategy_dirs: list[Path] | None = None): - """ - Args: - enriched_loader: (date) -> pl.DataFrame, 加载指定日期的 enriched 数据 - strategy_dirs: 策略文件搜索目录列表 - """ - self._loader = enriched_loader - self._history_loader = enriched_history_loader + _module_load_lock = threading.RLock() + + def __init__(self, strategy_dirs: list[Path] | None = None): self._strategies: dict[str, StrategyDef] = {} self._load_errors: list[dict] = [] # 加载失败的策略 [{file, error}] self._strategy_dirs = strategy_dirs or [] - self._load_all() + self._realtime_matrices: dict[str, _RealtimeMatrixEntry] = {} + self._realtime_matrix_lock = threading.RLock() + self._load_all(retain_previous_on_error=False) # ================================================================ # 加载 # ================================================================ - def _load_all(self) -> None: - self._strategies.clear() - self._load_errors = [] + def _load_all(self, *, retain_previous_on_error: bool) -> bool: + candidates: dict[str, StrategyDef] = {} + candidate_paths: dict[str, Path] = {} + errors: list[dict] = [] + duplicate_ids: set[str] = set() for d in self._strategy_dirs: if not d.exists(): continue @@ -157,12 +179,37 @@ class StrategyEngine: continue try: s = self._load_file(f) - self._strategies[s.meta["id"]] = s - logger.debug("loaded strategy: %s (%s)", s.meta["id"], s.source) + strategy_id = str(s.meta["id"]) + if strategy_id in duplicate_ids: + errors.append({ + "file": str(f), + "error": f"duplicate strategy id {strategy_id!r}", + }) + continue + if strategy_id in candidates: + previous_path = candidate_paths.pop(strategy_id) + candidates.pop(strategy_id) + duplicate_ids.add(strategy_id) + message = f"duplicate strategy id {strategy_id!r}" + errors.extend([ + {"file": str(previous_path), "error": message}, + {"file": str(f), "error": message}, + ]) + continue + candidates[strategy_id] = s + candidate_paths[strategy_id] = f except Exception as e: - # 不再静默吞掉: 记录失败项, 供前端可见(避免"策略静默消失"误判)。 logger.warning("load strategy %s failed: %s", f.name, e) - self._load_errors.append({"file": f.name, "error": str(e)}) + errors.append({"file": str(f), "error": str(e)}) + + self._load_errors = errors + if errors and retain_previous_on_error: + return False + + self._strategies = candidates + for strategy_id, strategy in candidates.items(): + logger.debug("loaded strategy: %s (%s)", strategy_id, strategy.source) + return not errors def load_errors(self) -> list[dict]: """返回最近一次 _load_all 中加载失败的策略 [{file, error}]。""" @@ -173,13 +220,32 @@ class StrategyEngine: """从 Python 文件加载策略定义""" # 纵深防御: 执行前再跑一次 AST 安全校验, 防止策略文件被直接篡改 # 绕过 API 校验后, 在 exec_module 时执行恶意代码。 + dependency_paths = [ + candidate + for candidate in path.parent.glob("_*.py") + if candidate != path + ] + dependency_names = frozenset(candidate.stem for candidate in dependency_paths) try: code = path.read_text(encoding="utf-8") from app.strategy.ai_generator import AIStrategyGenerator - AIStrategyGenerator._validate_safety(code) + AIStrategyGenerator._validate_safety( + code, + extra_allowed_import_modules=dependency_names, + ) + for dependency_path in dependency_paths: + AIStrategyGenerator._validate_safety( + dependency_path.read_text(encoding="utf-8"), + extra_allowed_import_modules=frozenset({ + "collections.abc", + "types", + "typing", + }), + extra_allowed_calls=frozenset({"vars"}), + ) except ValueError: raise - except Exception as e: # noqa: BLE001 + except Exception: # noqa: BLE001 # 文件读不到/语法错等: 不阻断, 让下方 exec_module 抛原样错误 pass @@ -187,9 +253,28 @@ class StrategyEngine: if spec is None or spec.loader is None: raise ValueError(f"cannot load module from {path}") mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) + with StrategyEngine._module_load_lock: + previous_module = sys.modules.get(spec.name) + sys.modules[spec.name] = mod + inserted_path = str(path.parent) + sys.path.insert(0, inserted_path) + try: + for dependency_name in dependency_names: + sys.modules.pop(dependency_name, None) + spec.loader.exec_module(mod) + except Exception: + if previous_module is None: + sys.modules.pop(spec.name, None) + else: + sys.modules[spec.name] = previous_module + raise + finally: + try: + sys.path.remove(inserted_path) + except ValueError: + pass - meta = getattr(mod, "META", {}) + meta = dict(getattr(mod, "META", {}) or {}) meta.setdefault("id", path.stem) meta.setdefault("name", path.stem) meta.setdefault("description", "") @@ -200,6 +285,27 @@ class StrategyEngine: meta.setdefault("descending", True) meta.setdefault("limit", 100) + source = "custom" + normalized_path = str(path).replace("\\", "/") + if "/builtin/" in normalized_path: + source = "builtin" + elif "/ai/" in normalized_path: + source = "ai" + + if source == "builtin" and "asset_types" not in meta: + raise ValueError("builtin strategy META must declare asset_types") + meta.setdefault("asset_types", ["stock"]) + meta.setdefault("timeframes", ["1d"]) + for field_name in ("asset_types", "timeframes"): + values = meta.get(field_name) + if ( + not isinstance(values, (list, tuple)) + or not values + or any(not isinstance(value, str) or not value for value in values) + ): + raise ValueError(f"META[{field_name!r}] must be a non-empty string list") + meta[field_name] = list(dict.fromkeys(values)) + # 归一化 params 为标准 list[dict]: custom/AI 策略的 META["params"] 可能是 # dict / list[str] 等非标准格式 (LLM 偶发漂移 / 用户手改), 不归一化的话会在 # _strategy_detail() 的 {p["id"]: p["default"] for p in params} 处抛 TypeError, @@ -216,11 +322,40 @@ class StrategyEngine: if meta_bf: bf.update(meta_bf) - source = "custom" - if "builtin" in str(path).replace("\\", "/"): - source = "builtin" - elif "/ai/" in str(path).replace("\\", "/") or "\\ai\\" in str(path): - source = "ai" + filter_fn = getattr(mod, "filter", None) + filter_history_fn = getattr(mod, "filter_history", None) + execution_backend = str( + getattr( + mod, + "EXECUTION_BACKEND", + meta.get( + "execution_backend", + "python_history_legacy" if filter_history_fn else "polars_expr", + ), + ) + ) + valid_backends = {"polars_expr", "matrix_native", "python_history_legacy"} + if execution_backend not in valid_backends: + raise ValueError( + f"unsupported execution backend {execution_backend!r}; " + f"expected one of {sorted(valid_backends)}" + ) + + matrix_strategy = getattr(mod, "MATRIX_STRATEGY", None) + if execution_backend == "matrix_native": + from app.backtest.matrix import MatrixStrategy + + if matrix_strategy is None: + raise ValueError("matrix_native strategy must declare MATRIX_STRATEGY") + if not isinstance(matrix_strategy, MatrixStrategy): + raise TypeError("MATRIX_STRATEGY must implement MatrixStrategy") + if filter_fn is not None or filter_history_fn is not None: + raise ValueError("matrix_native strategy must not declare filter or filter_history") + elif execution_backend == "polars_expr": + if filter_fn is None or filter_history_fn is not None: + raise ValueError("polars_expr strategy must declare only filter") + elif filter_history_fn is None or filter_fn is not None: + raise ValueError("python_history_legacy strategy must declare only filter_history") return StrategyDef( meta=meta, @@ -233,16 +368,26 @@ class StrategyEngine: trailing_take_profit_drawdown=getattr(mod, "TRAILING_TAKE_PROFIT_DRAWDOWN", None), max_hold_days=getattr(mod, "MAX_HOLD_DAYS", None), alerts=getattr(mod, "ALERTS", []), - filter_fn=getattr(mod, "filter", None), - filter_history_fn=getattr(mod, "filter_history", None), + filter_fn=filter_fn, + filter_history_fn=filter_history_fn, + required_features=frozenset(meta.get("required_features", []) or []) + | frozenset(getattr(mod, "REQUIRED_FEATURES", []) or []), lookback_days=int(getattr(mod, "LOOKBACK_DAYS", meta.get("lookback_days", 1)) or 1), source=source, file_path=path, + execution_backend=execution_backend, + matrix_strategy=matrix_strategy, ) def reload(self) -> None: - """热重载所有策略""" - self._load_all() + """原子热重载；任一策略失败时保留上一版注册表。""" + if not self._load_all(retain_previous_on_error=True): + details = "; ".join( + f"{item['file']}: {item['error']}" for item in self._load_errors + ) + raise ValueError(f"strategy reload failed: {details}") + with self._realtime_matrix_lock: + self._realtime_matrices.clear() # ================================================================ # 查询 @@ -252,9 +397,17 @@ class StrategyEngine: """返回所有策略的元信息""" result = [] for s in self._strategies.values(): - result.append({**s.meta, "source": s.source}) + result.append({ + **s.meta, + "source": s.source, + "execution_backend": s.execution_backend, + }) return result + def strategy_definitions(self) -> tuple[StrategyDef, ...]: + """Return the immutable registry snapshot for framework dependency planning.""" + return tuple(self._strategies.values()) + def get(self, strategy_id: str) -> StrategyDef: s = self._strategies.get(strategy_id) if not s: @@ -264,6 +417,177 @@ class StrategyEngine: def has(self, strategy_id: str) -> bool: return strategy_id in self._strategies + @staticmethod + def validate_context(strategy: StrategyDef, context: StrategyDataContext) -> None: + asset_types = strategy.meta.get("asset_types", ["stock"]) + if context.asset_type not in asset_types: + raise ValueError( + f"strategy {strategy.meta['id']} does not support asset_type " + f"{context.asset_type!r}; supported={asset_types}" + ) + timeframes = strategy.meta.get("timeframes", ["1d"]) + if context.timeframe not in timeframes: + raise ValueError( + f"strategy {strategy.meta['id']} does not support timeframe " + f"{context.timeframe!r}; supported={timeframes}" + ) + + @staticmethod + def resolve_params( + strategy: StrategyDef, + params: dict | None = None, + overrides: dict | None = None, + ) -> dict: + """Resolve one parameter source of truth for every strategy consumer.""" + resolved = { + item["id"]: item.get("default") + for item in strategy.meta.get("params", []) + if isinstance(item, dict) and item.get("id") + } + saved = (overrides or {}).get("params") + if isinstance(saved, dict): + resolved.update(saved) + if params: + resolved.update(params) + return resolved + + @staticmethod + def _result_limit(strategy: StrategyDef, overrides: dict | None) -> int | None: + if overrides and "display_limit" in overrides: + value = overrides.get("display_limit") + if value in (None, 0): + return None + return max(0, int(value)) + value = strategy.meta.get("limit", 100) + if value in (None, 0): + return None + return max(0, int(value)) + + def required_history_bars( + self, + strategy_ids: list[str], + *, + params_map: dict[str, dict] | None = None, + overrides_map: dict[str, dict] | None = None, + ) -> int: + params_map = params_map or {} + overrides_map = overrides_map or {} + required = 1 + for strategy_id in strategy_ids: + strategy = self.get(strategy_id) + if strategy.execution_backend == "matrix_native": + params = self.resolve_params( + strategy, + params_map.get(strategy_id), + overrides_map.get(strategy_id), + ) + required = max( + required, + int(strategy.matrix_strategy.required_warmup_bars(params)) + 1, + ) + elif strategy.filter_history_fn: + required = max(required, int(strategy.lookback_days)) + return required + + def prepare_realtime_matrix( + self, + context: StrategyDataContext, + strategy_ids: list[str], + *, + params_map: dict[str, dict] | None = None, + overrides_map: dict[str, dict] | None = None, + ): + """Build once, then update only the latest live bar for matrix strategies.""" + from app.backtest.matrix import RealtimeMarketDataMatrix + + current = context.current + if current is None: + raise ValueError("realtime matrix context requires current data") + if current.is_empty() or not strategy_ids: + return None + params_map = params_map or {} + overrides_map = overrides_map or {} + field_columns: set[str] = set() + max_warmup = 1 + matrix_ids: list[str] = [] + for strategy_id in strategy_ids: + strategy = self.get(strategy_id) + self.validate_context(strategy, context) + if strategy.execution_backend != "matrix_native": + continue + params = self.resolve_params( + strategy, + params_map.get(strategy_id), + overrides_map.get(strategy_id), + ) + matrix_ids.append(strategy_id) + max_warmup = max( + max_warmup, + int(strategy.matrix_strategy.required_warmup_bars(params)) + 1, + ) + field_columns.update( + self._matrix_field_columns(strategy, overrides_map.get(strategy_id)) + ) + if not matrix_ids: + return None + + timestamp_col = "datetime" if "datetime" in current.columns else "date" + if timestamp_col not in current.columns: + raise ValueError("realtime matrix current data requires date or datetime") + latest_value = current[timestamp_col].max() + as_of = latest_value.date() if hasattr(latest_value, "date") else latest_value + if not isinstance(as_of, date): + raise ValueError("realtime matrix timestamp cannot be converted to date") + symbols = tuple(current["symbol"].cast(pl.Utf8).unique().sort().to_list()) + fingerprint = ( + tuple(sorted(field_columns)), + max_warmup, + symbols, + ) + + with self._realtime_matrix_lock: + cache_key = context.cache_key or f"{context.asset_type}:{context.timeframe}" + entry = self._realtime_matrices.get(cache_key) + if entry is not None and entry.fingerprint == fingerprint: + try: + entry.buffer.update(current) + return entry.buffer.snapshot() + except ValueError as exc: + logger.info("realtime matrix %s invalidated: %s", cache_key, exc) + + history = context.history + if history is None: + raise ValueError("matrix strategy realtime context requires history data") + if history is None or history.is_empty(): + raise ValueError("matrix strategy realtime history is empty") + if timestamp_col in history.columns: + history = history.filter(pl.col(timestamp_col) != latest_value) + elif "date" in history.columns: + history = history.filter(pl.col("date") != as_of) + panel = pl.concat([history, current], how="diagonal_relaxed") + previous_builds = entry.buffer.build_count if entry is not None else 0 + buffer = RealtimeMarketDataMatrix( + panel, + field_columns=field_columns, + build_count=previous_builds + 1, + ) + self._realtime_matrices[cache_key] = _RealtimeMatrixEntry( + fingerprint=fingerprint, + buffer=buffer, + ) + return buffer.snapshot() + + def realtime_matrix_stats(self, cache_key: str) -> dict[str, int]: + with self._realtime_matrix_lock: + entry = self._realtime_matrices.get(cache_key) + if entry is None: + return {"generation": 0, "build_count": 0, "update_count": 0} + return { + "generation": int(entry.buffer.generation), + "build_count": int(entry.buffer.build_count), + "update_count": int(entry.buffer.update_count), + } + # ================================================================ # 执行 # ================================================================ @@ -271,39 +595,45 @@ class StrategyEngine: def run( self, strategy_id: str, - as_of: date, + context: StrategyDataContext, pool: list[str] | None = None, params: dict | None = None, overrides: dict | None = None, - precomputed: pl.DataFrame | None = None, - precomputed_history: pl.DataFrame | None = None, ) -> StrategyResult: """执行策略: 基础过滤 → 策略过滤 → 评分排序 Args: strategy_id: 策略 ID - as_of: 选股日期 + context: 调用级行情、资产和周期上下文 pool: 限定股票池 params: 本次执行显式传入的策略参数 overrides: 用户覆盖配置 (params/basic_filter/scoring/stop_loss 等) - precomputed: 已加载的 enriched 数据 (run_all 场景复用) - precomputed_history: 已加载的历史窗口数据 (run_all 场景复用) """ t0 = time.perf_counter() s = self.get(strategy_id) + self.validate_context(s, context) + as_of = context.as_of overrides = overrides or {} - params = {**(overrides.get("params") or {}), **(params or {})} + params = self.resolve_params(s, params, overrides) - # 加载数据。普通策略只读目标日期；声明 filter_history 的策略读取历史窗口。 + if s.execution_backend == "matrix_native": + return self._run_matrix_strategy( + strategy_id, + s, + as_of, + pool=pool, + params=params, + overrides=overrides, + context=context, + started_at=t0, + ) + + # 普通策略只读目标日期；历史策略读取调用方注入的历史窗口。 if s.filter_history_fn: - if precomputed_history is not None and not precomputed_history.is_empty(): - df = precomputed_history - elif self._history_loader: - df = self._history_loader(as_of, max(1, s.lookback_days)) - else: - logger.warning("strategy %s requires history loader", strategy_id) - return StrategyResult(as_of=as_of, strategy_id=strategy_id) + if context.history is None: + raise ValueError(f"strategy {strategy_id} requires history data") + df = context.history if df.is_empty(): return StrategyResult(as_of=as_of, strategy_id=strategy_id) df = s.filter_history_fn(df, params) @@ -311,10 +641,10 @@ class StrategyEngine: return StrategyResult(as_of=as_of, strategy_id=strategy_id) if "date" in df.columns: df = df.filter(pl.col("date") == as_of) - elif precomputed is not None and not precomputed.is_empty(): - df = precomputed else: - df = self._loader(as_of) + if context.current is None: + raise ValueError(f"strategy {strategy_id} requires current data") + df = context.current if df.is_empty(): return StrategyResult(as_of=as_of, strategy_id=strategy_id) @@ -345,7 +675,7 @@ class StrategyEngine: df = self._apply_scoring(df, scoring) # 排序 + 限制 - limit = s.meta.get("limit", 100) + limit = self._result_limit(s, overrides) order_desc = s.meta.get("descending", True) if "score" in df.columns: df = df.sort("score", descending=order_desc) @@ -353,7 +683,8 @@ class StrategyEngine: ob = s.meta["order_by"] if ob in df.columns: df = df.sort(ob, descending=order_desc) - df = df.head(limit) + if limit is not None: + df = df.head(limit) # 输出 rows = _sanitize(df.to_dicts()) @@ -373,48 +704,221 @@ class StrategyEngine: scores=scores, ) - def run_all(self, as_of: date, params_map: dict | None = None, - overrides_map: dict | None = None) -> dict[str, StrategyResult]: - """批量执行所有策略 (enriched 只加载一次，基础过滤按策略分组缓存，历史数据共享)""" - df = self._loader(as_of) + def run_all( + self, + context: StrategyDataContext, + params_map: dict | None = None, + overrides_map: dict | None = None, + *, + strategy_ids: list[str] | None = None, + ) -> dict[str, StrategyResult]: + """批量执行策略；当前数据、历史和矩阵均来自同一个调用上下文。""" + if context.current is None: + raise ValueError("strategy run_all context requires current data") + df = context.current params_map = params_map or {} overrides_map = overrides_map or {} + selected_ids = list(self._strategies) if strategy_ids is None else strategy_ids + selected = [(sid, self.get(sid)) for sid in selected_ids] + for _, strategy in selected: + self.validate_context(strategy, context) - # 历史策略: 找最大 lookback，一次加载共享 - history_strats = [(sid, s) for sid, s in self._strategies.items() if s.filter_history_fn] - if history_strats and self._history_loader: - max_lookback = max(s.lookback_days for _, s in history_strats) - shared_history = self._history_loader(as_of, max(1, max_lookback)) - else: - shared_history = None + history_strats = [ + (sid, strategy) + for sid, strategy in selected + if strategy.filter_history_fn or strategy.execution_backend == "matrix_native" + ] + shared_history = context.history + if history_strats and shared_history is None: + raise ValueError("selected strategies require history data") + + shared_matrix = context.market + matrix_strats = [ + (sid, strategy) + for sid, strategy in selected + if strategy.execution_backend == "matrix_native" + ] + if ( + shared_matrix is None + and matrix_strats + and shared_history is not None + and not shared_history.is_empty() + ): + from app.backtest.matrix import build_market_data_matrix + + field_columns: set[str] = set() + for sid, strategy in matrix_strats: + field_columns.update( + self._matrix_field_columns(strategy, overrides_map.get(sid)) + ) + shared_matrix = build_market_data_matrix( + shared_history, + field_columns=field_columns, + ) - # 按 basic_filter hash 分组，避免重复过滤 - bf_cache: dict[str, pl.DataFrame] = {} results: dict[str, StrategyResult] = {} - for sid, strat in self._strategies.items(): - try: - bf_key = _dict_hash(strat.basic_filter) - if bf_key not in bf_cache: - if strat.basic_filter.get("enabled", True): - bf_cache[bf_key] = self._apply_basic_filter(df, strat.basic_filter) - else: - bf_cache[bf_key] = df - base = bf_cache[bf_key] - - # 从已过滤的 base 执行 (filter_history 策略使用共享历史) - results[sid] = self.run( - sid, as_of, - params=params_map.get(sid), - overrides=overrides_map.get(sid), - precomputed=base, - precomputed_history=shared_history, - ) - except Exception as e: - logger.warning("run strategy %s failed: %s", sid, e) + for sid, _ in selected: + results[sid] = self.run( + sid, + replace( + context, + current=df, + history=shared_history, + market=shared_matrix, + ), + params=params_map.get(sid), + overrides=overrides_map.get(sid), + ) return results + @staticmethod + def _matrix_field_columns(strategy: StrategyDef, overrides: dict | None = None) -> set[str]: + fields = set(strategy.matrix_strategy.required_fields()) + basic_filter = dict(strategy.basic_filter or {}) + if (overrides or {}).get("basic_filter"): + basic_filter.update(overrides["basic_filter"]) + for prefix, field_name in ( + ("market_cap", "total_shares"), + ("float_cap", "float_shares"), + ("amount", "amount"), + ("turnover", "turnover_rate"), + ): + if ( + basic_filter.get(f"{prefix}_min") is not None + or basic_filter.get(f"{prefix}_max") is not None + ): + fields.add(field_name) + scoring = dict(strategy.meta.get("scoring", {}) or {}) + scoring.update((overrides or {}).get("scoring") or {}) + fields.update(scoring) + order_by = strategy.meta.get("order_by") + if order_by and order_by != "score": + fields.add(str(order_by)) + return fields + + def _run_matrix_strategy( + self, + strategy_id: str, + strategy: StrategyDef, + as_of: date, + *, + pool: list[str] | None, + params: dict, + overrides: dict, + context: StrategyDataContext, + started_at: float, + ) -> StrategyResult: + from app.backtest.matrix import ( + MatrixPipelineConfig, + MatrixStrategyPipeline, + build_market_data_matrix, + ) + + source_panel = context.history + market = context.market + if market is None: + if source_panel is None: + raise ValueError(f"matrix strategy {strategy_id} requires history data") + if source_panel is None or source_panel.is_empty(): + return StrategyResult(as_of=as_of, strategy_id=strategy_id) + market = build_market_data_matrix( + source_panel, + field_columns=self._matrix_field_columns(strategy, overrides), + ) + + if source_panel is None or source_panel.is_empty(): + source_panel = context.current + if source_panel is None or source_panel.is_empty(): + return StrategyResult(as_of=as_of, strategy_id=strategy_id) + + basic_filter = dict(strategy.basic_filter or {}) + if overrides.get("basic_filter"): + basic_filter.update(overrides["basic_filter"]) + scoring = dict(strategy.meta.get("scoring", {}) or {}) + scoring.update(overrides.get("scoring") or {}) + asset_mask = None + if pool: + pool_set = set(pool) + asset_mask = np.fromiter( + (symbol in pool_set for symbol in market.symbols), + dtype=bool, + count=len(market.symbols), + ) + + signals = MatrixStrategyPipeline().run( + strategy.matrix_strategy, + market, + params, + MatrixPipelineConfig( + basic_filter=basic_filter, + scoring=scoring, + order_by=strategy.meta.get("order_by"), + descending=bool(strategy.meta.get("descending", True)), + asset_mask=asset_mask, + ), + ) + target_ids = [ + time_id + for time_id, label in enumerate(market.timestamp_labels) + if label[:10] == str(as_of) + ] + if not target_ids: + return StrategyResult(as_of=as_of, strategy_id=strategy_id) + target_time = target_ids[-1] + selected_assets = np.flatnonzero(signals.entry[target_time] != 0) + if selected_assets.size == 0: + return StrategyResult( + as_of=as_of, + strategy_id=strategy_id, + elapsed_ms=(time.perf_counter() - started_at) * 1000, + ) + + target_frame = self._matrix_target_frame(source_panel, as_of) + row_by_symbol = { + str(row["symbol"]): row + for row in target_frame.iter_rows(named=True) + } + ranked: list[tuple[float, dict]] = [] + for asset_id in selected_assets: + symbol = market.symbols[int(asset_id)] + row = row_by_symbol.get(symbol) + if row is None: + continue + score = float(signals.score[target_time, int(asset_id)]) + ranked.append((score, {**row, "score": score})) + ranked.sort( + key=lambda item: item[0], + reverse=bool(strategy.meta.get("descending", True)), + ) + limit = self._result_limit(strategy, overrides) + selected_rows = ranked if limit is None else ranked[:limit] + rows = _sanitize([row for _, row in selected_rows]) + scores = {str(row["symbol"]): float(row.get("score") or 0.0) for row in rows} + return StrategyResult( + as_of=as_of, + strategy_id=strategy_id, + rows=rows, + total=len(rows), + elapsed_ms=(time.perf_counter() - started_at) * 1000, + scores=scores, + ) + + @staticmethod + def _matrix_target_frame(panel: pl.DataFrame, as_of: date) -> pl.DataFrame: + if "datetime" in panel.columns: + target = panel.filter(pl.col("datetime").cast(pl.Date) == as_of) + if target.is_empty(): + return target + latest = target["datetime"].max() + target = target.filter(pl.col("datetime") == latest) + elif "date" in panel.columns: + target = panel.filter(pl.col("date") == as_of) + else: + return panel.head(0) + return target.unique(subset=["symbol"], keep="last") + # ================================================================ # 内部: 基础过滤 # ================================================================ @@ -530,8 +1034,3 @@ def _sanitize(rows: list[dict]) -> list[dict]: if isinstance(v, float) and (v != v or abs(v) == float("inf")): r[k] = None return rows - - -def _dict_hash(d: dict) -> str: - """用于 basic_filter 分组缓存""" - return str(sorted(d.items())) diff --git a/backend/app/strategy/monitor.py b/backend/app/strategy/monitor.py index 8ec8c34..6cecfdf 100644 --- a/backend/app/strategy/monitor.py +++ b/backend/app/strategy/monitor.py @@ -21,8 +21,8 @@ from typing import Any, Callable import polars as pl from app.market_time import cn_today -from app.strategy.custom_signals import _OP_BUILDERS # type: ignore # 复用运算符构造器 from app.strategy import config as _strategy_config +from app.strategy.custom_signals import _OP_BUILDERS # type: ignore # 复用运算符构造器 logger = logging.getLogger(__name__) @@ -327,7 +327,7 @@ class MonitorRuleEngine: # symbol → 股票名 (enriched DataFrame 已 drop name 列, 触发时从此映射回填) self._name_map: dict[str, str] = {} # 策略选股池状态: strategy_id → 上期选股符号集合 (用于 diff 变更) - self._strategy_pools: dict[str, set[str]] = {} + self._strategy_pools: dict[tuple[str, str, str], set[str]] = {} # 数据目录 (用于加载策略 overrides) self._data_dir = None # 历史窗口加载器: (target_date, lookback_days) → 多日 enriched DataFrame。 @@ -336,6 +336,7 @@ class MonitorRuleEngine: self._history_loader: Callable[[_dt.date, int], "pl.DataFrame"] | None = None # ETF 版历史窗口加载器 (asset_type=etf 的规则用)。为 None 时 ETF filter_history 策略跳过。 self._history_loader_etf: Callable[[_dt.date, int], "pl.DataFrame"] | None = None + self._active_matrix_snapshots: dict[str, Any] = {} # 本轮 evaluate() 产出的策略选股结果: strategy_id → {rows, total, as_of} # 供策略页实时回显复用 (/api/screener/cached 端点直接读取, 避免重跑)。 # 注意: 始终是「完整」的 dict —— evaluate 重算时先写到 _building_strategy_results, @@ -355,6 +356,14 @@ class MonitorRuleEngine: """注入数据目录, 用于加载策略的用户覆盖配置。""" self._data_dir = data_dir + def invalidate_strategy_state(self) -> None: + """策略注册表变更后清除选股池、结果和矩阵快照。""" + self._strategy_pools.clear() + self._latest_strategy_results = {} + self._building_strategy_results = {} + self._latest_strategy_result_ids.clear() + self._active_matrix_snapshots.clear() + def set_history_loader(self, fn) -> None: """注入历史窗口加载器, 用于声明 filter_history 的策略跑实时监控。 @@ -485,6 +494,80 @@ class MonitorRuleEngine: self._building_strategy_results = {} self._latest_strategy_result_ids.clear() + matrix_rules: list[dict] = [] + params_map: dict[str, dict] = {} + overrides_map: dict[str, dict] = {} + if self._strategy_engine is not None: + for rule in list(self._rules.values()): + if ( + not rule.get("enabled", True) + or rule.get("type") != "strategy" + or rule.get("asset_type", "stock") != asset_type + ): + continue + sid = rule.get("strategy_id") + if not sid: + continue + try: + strategy = self._strategy_engine.get(sid) + except Exception: + continue + if getattr(strategy, "execution_backend", "polars_expr") != "matrix_native": + continue + overrides = {} + if self._data_dir: + overrides = _strategy_config.load_override(self._data_dir, sid) + matrix_rules.append(rule) + overrides_map[sid] = overrides + params_map[sid] = dict(overrides.get("params") or {}) + if matrix_rules: + try: + history_loader = self._history_loader_for(matrix_rules[0]) + if history_loader is None: + raise ValueError("matrix strategy monitor requires history loader") + matrix_ids = [str(rule["strategy_id"]) for rule in matrix_rules] + history_bars = self._strategy_engine.required_history_bars( + matrix_ids, + params_map=params_map, + overrides_map=overrides_map, + ) + from app.strategy.engine import StrategyDataContext + context = StrategyDataContext( + asset_type=asset_type, + timeframe="1d", + as_of=cn_today(), + current=df, + cache_key=f"monitor:{asset_type}", + ) + try: + snapshot = self._strategy_engine.prepare_realtime_matrix( + context, + matrix_ids, + params_map=params_map, + overrides_map=overrides_map, + ) + except ValueError as exc: + if "requires history data" not in str(exc): + raise + history = history_loader(cn_today(), history_bars) + snapshot = self._strategy_engine.prepare_realtime_matrix( + StrategyDataContext( + asset_type=asset_type, + timeframe="1d", + as_of=cn_today(), + current=df, + history=history, + cache_key=f"monitor:{asset_type}", + ), + matrix_ids, + params_map=params_map, + overrides_map=overrides_map, + ) + self._active_matrix_snapshots[asset_type] = snapshot + except Exception as e: + self._active_matrix_snapshots.pop(asset_type, None) + logger.warning("%s 矩阵策略实时缓存准备失败: %s", asset_type, e) + # list() 快照: 本方法跑在行情轮询线程, API 线程同时 add/remove 规则 # 会触发 "dictionary changed size during iteration", 整轮告警丢失 for rule_id, rule in list(self._rules.items()): @@ -498,6 +581,7 @@ class MonitorRuleEngine: # 一次性提交本轮结果 (原子替换): /cached 读方要么拿到上一轮完整结果, # 要么拿到本轮完整结果, 不会读到空中间态。 self._latest_strategy_results = self._building_strategy_results + self._active_matrix_snapshots.pop(asset_type, None) return events @@ -620,7 +704,7 @@ class MonitorRuleEngine: if not sid: return [] at = rule.get("asset_type", "stock") - pool_key = (sid, at) + pool_key = (str(rule.get("id", sid)), sid, at) try: s = self._strategy_engine.get(sid) except Exception: @@ -640,11 +724,26 @@ class MonitorRuleEngine: # 旧实现因"实时监控不支持 history loader"直接跳过 → 反包等策略盘中永不触发。 # 现接入 history_loader, 拼历史窗口 + 今日实时行情, 经 precomputed_history 喂给引擎。 # loader 为 None (未装配) 时退回跳过, 保持旧行为, 不破坏无历史场景。 - run_kwargs: dict = { - "as_of": cn_today(), - "overrides": overrides, - } - if s.filter_history_fn: + from app.strategy.engine import StrategyDataContext + current_context = StrategyDataContext( + asset_type=at, + timeframe="1d", + as_of=cn_today(), + current=df, + ) + if getattr(s, "execution_backend", "polars_expr") == "matrix_native": + matrix = self._active_matrix_snapshots.get(at) + if matrix is None: + logger.debug("策略 %s 缺少本轮实时矩阵快照, 跳过", sid) + return [] + current_context = StrategyDataContext( + asset_type=at, + timeframe="1d", + as_of=cn_today(), + current=df, + market=matrix, + ) + elif s.filter_history_fn: history_loader = self._history_loader_for(rule) if history_loader is None: logger.debug("策略 %s 需要历史数据但未注入 history_loader (asset_type=%s), 跳过实时监控", @@ -663,18 +762,28 @@ class MonitorRuleEngine: if "date" in hist_df.columns: hist_df = hist_df.filter(pl.col("date") != today) # 拼接历史窗口 + 今日实时行情 (filter_history 用 .over("symbol") 窗口, 多日天然可用) - run_kwargs["precomputed_history"] = pl.concat( - [hist_df, df], how="diagonal_relaxed" + current_context = StrategyDataContext( + asset_type=at, + timeframe="1d", + as_of=today, + current=df, + history=pl.concat( + [hist_df, df], how="diagonal_relaxed" + ), ) except Exception as e: logger.warning("策略 %s 加载历史窗口失败, 跳过: %s", sid, e) return [] - else: - # 普通策略: 复用当前 enriched DataFrame 跳过数据加载 - run_kwargs["precomputed"] = df - try: - result = self._strategy_engine.run(sid, **run_kwargs) + result = self._strategy_engine.run( + sid, + current_context, + pool=(df["symbol"].cast(pl.Utf8).to_list() + if getattr(s, "execution_backend", "polars_expr") == "matrix_native" + else None), + overrides=overrides, + params=dict(overrides.get("params") or {}), + ) except Exception as e: logger.warning("策略 %s 选股执行失败: %s", sid, e) return [] diff --git a/backend/app/strategy/prompt_builder.py b/backend/app/strategy/prompt_builder.py index 0b11d99..f3f008d 100644 --- a/backend/app/strategy/prompt_builder.py +++ b/backend/app/strategy/prompt_builder.py @@ -22,7 +22,14 @@ def _load_doc(name: str) -> str: DIRECTION_CN = {"long": "做多", "short": "做空", "monitor": "监控"} -def build_step1(name: str, description: str, direction: str, rules: str, strategy_id: str = "") -> str: +def build_step1( + name: str, + description: str, + direction: str, + rules: str, + strategy_id: str = "", + execution_backend: str = "polars_expr", +) -> str: """步骤1：规则 → 完整策略代码（参数 + 信号 + 评分 + 告警） 注意: 生成规范已在 ai_generator.py 的 system prompt 中加载， @@ -35,13 +42,14 @@ def build_step1(name: str, description: str, direction: str, rules: str, strateg 策略名称：{name}{id_line} 策略描述：{description} 选股方向：{DIRECTION_CN.get(direction, direction)} +执行后端：{execution_backend} 策略规则： {rules} 输出要求： 1. 严格遵循系统提示中的策略文件结构和安全限制。 -2. 根据规则自行判断使用 filter() 或 filter_history()。 -3. 生成完整 META、ENTRY_SIGNALS、EXIT_SIGNALS、STOP_LOSS、MAX_HOLD_DAYS、ALERTS、RULES 和筛选函数。 +2. 严格使用指定执行后端；matrix_native 只定义 MATRIX_STRATEGY，polars_expr 只定义 filter()。若 polars 规则确需历史窗口，改用 python_history_legacy + filter_history()。 +3. META 必须声明 asset_types 和 timeframes，并生成完整执行与交易元数据。 4. 只输出 Python 代码。""" diff --git a/backend/app/strategy/prompts/strategy-builder-step1.md b/backend/app/strategy/prompts/strategy-builder-step1.md index a76f0a8..96aa663 100644 --- a/backend/app/strategy/prompts/strategy-builder-step1.md +++ b/backend/app/strategy/prompts/strategy-builder-step1.md @@ -7,7 +7,7 @@ 1. **只创建这一个策略文件**：只生成一个 `.py` 文件，绝不创建多文件、不拆分模块、不跨文件 import 2. **绝不触碰项目源码**：不要写任何会修改 `backend/`、`docs/`、`frontend/` 等现有文件的代码；不要 `import os/sys/pathlib` 等文件系统模块 3. **不得放入内置策略目录**：AI 生成的策略只属于 `data/strategies/ai/`，文件名/ID 用 `ai_` 前缀；内置目录 `backend/app/strategy/builtin/` 由项目维护，AI 不得染指 -4. 只 `import polars as pl`，不 import 其他模块 +4. Polars 策略只 `import polars as pl`；矩阵策略只 import NumPy 和 `app.backtest.matrix` 协议/算子 5. 贴合用户需求优先：不要为了套模板而歪曲策略含义 ## 选择策略模式 @@ -37,7 +37,7 @@ 3. **STOP_LOSS / MAX_HOLD_DAYS**：根据策略类型合理设定，做多止损一般为 -5%~-8%，短线持有 5~20 天 4. **ALERTS**：列出需要监控提醒的条件 5. **RULES**：中文逐条列出核心筛选逻辑（至少 3 条），准确完整 -6. **filter() 或 filter_history()**：核心筛选逻辑 +6. **EXECUTION_BACKEND + filter() 或 filter_history()**：只选择一个后端和一份核心筛选逻辑 ## 性能原则 diff --git a/backend/app/strategy/prompts/strategy-builder-step2.md b/backend/app/strategy/prompts/strategy-builder-step2.md index cd2661c..b330e90 100644 --- a/backend/app/strategy/prompts/strategy-builder-step2.md +++ b/backend/app/strategy/prompts/strategy-builder-step2.md @@ -25,7 +25,7 @@ - 修改止损/持有 → 更新 STOP_LOSS / MAX_HOLD_DAYS - 增减告警 → 更新 ALERTS - 调整评分 → 更新 META["scoring"]，权重总和保持 100 -- 修改筛选逻辑 → 更新 filter()；如果新增/删除了历史回溯逻辑，同步改为或移除 filter_history() 与 LOOKBACK_DAYS +- 修改筛选逻辑 → 更新唯一公式；新增历史回溯时切换为 `python_history_legacy` + `filter_history()`，移除回溯时切回 `polars_expr` + `filter()`，不得同时保留两套公式 ## 规则 diff --git a/backend/app/strategy/prompts/strategy-example.md b/backend/app/strategy/prompts/strategy-example.md index 7e0cca2..788c2cc 100644 --- a/backend/app/strategy/prompts/strategy-example.md +++ b/backend/app/strategy/prompts/strategy-example.md @@ -26,6 +26,8 @@ META = { "name": "强势反包", "description": "筛选前日阴线下跌、今日放量阳线反包的短线强势股", "tags": ["反包", "短线", "放量"], + "asset_types": ["stock"], + "timeframes": ["1d"], "basic_filter": { "price_min": 3, "price_max": 200, @@ -73,6 +75,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "python_history_legacy" LOOKBACK_DAYS = 2 ENTRY_SIGNALS = ["signal_broken_board_recovery"] diff --git a/backend/app/strategy/prompts/strategy-guide-compact.md b/backend/app/strategy/prompts/strategy-guide-compact.md index db18a81..b82f640 100644 --- a/backend/app/strategy/prompts/strategy-guide-compact.md +++ b/backend/app/strategy/prompts/strategy-guide-compact.md @@ -4,7 +4,7 @@ ## 必须遵守 -1. 只允许 `import polars as pl` 和 `from datetime import date/datetime`（date 类型参数比较需要），禁止 import 其他模块。 +1. Polars/历史策略允许 `polars`、`datetime`；矩阵策略允许 `numpy`、`app.backtest.matrix`。 2. AI 策略只属于 `data/strategies/ai/`，`META.id` 使用用户给定的 `ai_` ID。 3. 不要读写文件，不要使用 `open/exec/eval/compile/__import__/globals/locals/vars/dir/getattr/setattr/delattr/type/input`。 4. `META.params` 只放用户可能调整的阈值；公式常数和固定窗口边界不必参数化。 @@ -24,6 +24,8 @@ META = { "name": "策略中文名", "description": "一句话说明策略逻辑", "tags": ["标签"], + "asset_types": ["stock"], + "timeframes": ["1d"], "basic_filter": { "price_min": 3, "price_max": 200, @@ -39,6 +41,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "polars_expr" ENTRY_SIGNALS = [] EXIT_SIGNALS = [] STOP_LOSS = -0.05 @@ -68,6 +71,7 @@ def filter(df: pl.DataFrame, params: dict) -> pl.Expr: ```python LOOKBACK_DAYS = 8 +EXECUTION_BACKEND = "python_history_legacy" def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: if df.is_empty() or "date" not in df.columns: @@ -78,9 +82,56 @@ def filter_history(df: pl.DataFrame, params: dict) -> pl.DataFrame: return hist.filter(pl.col("close") > pl.col("_prev_close")) ``` +使用 `filter_history()` 时必须同时声明其读取的最终公开字段，例如： + +```python +REQUIRED_FEATURES = {"ma20", "momentum_20d"} +``` + `filter_history()` 必须返回所有匹配行，不要只过滤最新日期；回测需要全区间命中。 -**date 类型参数必须先转换再与 `date` 列比较**：params 里 `"type": "date"` 的参数从 JSON 传来是字符串（如 `"2024-01-01"`），而数据中 `date` 列是 Polars Date 类型，**不能直接比较**（报 InvalidOperationError）。必须先转换： +## matrix_native 文件结构 + +当请求明确指定 `matrix_native` 时，不得生成 `filter()` 或 `filter_history()`： + +```python +import numpy as np +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + make_signal_matrix, + matrix_feature, +) + +META = { + "id": "custom_matrix_example", + "name": "矩阵示例", + "description": "...", + "asset_types": ["stock"], + "timeframes": ["1d"], + "params": [], + "scoring": {}, + "order_by": "score", + "descending": True, + "limit": 100, +} +EXECUTION_BACKEND = "matrix_native" + +class ExampleMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close", "ma20"}) + + def required_warmup_bars(self, params: dict) -> int: + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + entry = market.close > matrix_feature(market, "ma20") + return make_signal_matrix(market.shape, entry=entry.astype(np.uint8)) + +MATRIX_STRATEGY = ExampleMatrixStrategy() +``` + +**date 参数先转换再与 Polars Date 列比较**（JSON 值是字符串）： ```python from datetime import date as _date diff --git a/backend/app/strategy/prompts/strategy-guide.md b/backend/app/strategy/prompts/strategy-guide.md index 2d7e5de..b1583e4 100644 --- a/backend/app/strategy/prompts/strategy-guide.md +++ b/backend/app/strategy/prompts/strategy-guide.md @@ -98,6 +98,8 @@ def filter(df: pl.DataFrame, params: dict) -> pl.Expr: **不需要** `filter_history()` 的场景：只用当日指标列做比较（如 close > ma60、rsi_14 < 30）。 +策略必须显式声明唯一执行后端：普通表达式使用 `EXECUTION_BACKEND = "polars_expr"`，历史窗口使用 `EXECUTION_BACKEND = "python_history_legacy"`；不要同时定义 `filter()` 和 `filter_history()`。`META` 同时声明 `asset_types` 与 `timeframes`。 + ```python LOOKBACK_DAYS = 8 # 回看交易日数，根据策略需要设置 @@ -155,7 +157,13 @@ anchor_date = _date.fromisoformat(anchor_raw) if isinstance(anchor_raw, str) els ## 3. 常用指标列（参考，可直接使用） -以下列在数据中已预计算，可直接引用。**但如果这些列无法满足策略需求，可以不用，自行在 `filter_history()` 中基于 enriched 表的数据（已复权，含所有指标列和信号列）计算任何需要的字段。** +以下列可直接引用。使用 `filter_history()` 或其他无法静态解析的 Python 逻辑时，必须声明最终依赖的公开字段，基础 OHLCV 和指标中间列不需要声明： + +```python +REQUIRED_FEATURES = {"ma20", "momentum_20d", "vol_ratio_5d"} +``` + +未声明时回测会明确告警并暂时回退到全量特征计算。 ### 通用列 @@ -275,7 +283,7 @@ anchor_date = _date.fromisoformat(anchor_raw) if isinstance(anchor_raw, str) els 3. 用户可能调节的数值阈值通过 `params` 暴露；公式常数、固定窗口边界、一次性内部变量不必强行参数化 4. `scoring` 权重总和必须为 1.0 5. 遵循 A 股 T+1 规则 (当日买入次日才能卖出) -6. 只允许 `import polars as pl` 和 `from datetime import date/datetime`（date 类型参数比较需要），禁止 import 其他模块 +6. Polars 策略允许 `import polars as pl` 和 `from datetime import date/datetime`；矩阵策略只允许 NumPy 和 `app.backtest.matrix` 协议/算子 7. 禁止使用 `open()`, `exec()`, `eval()`, `os`, `sys`, `subprocess` 8. **贴合用户需求优先**：第3/4节的指标列和信号列仅供参考，能用则用；如果用户需求需要自定义计算（如"前高""上次涨停价""N日内某个事件后X天"），直接在 `filter_history()` 中自行设计和计算，不需要局限于已有列 9. `filter_history()` 中优先用 Polars 向量化语法；仅在复杂状态机无法清晰表达时，才用 `partition_by("symbol")` 逐股票分析 diff --git a/backend/app/tickflow/repository.py b/backend/app/tickflow/repository.py index a5cb285..24eeba0 100644 --- a/backend/app/tickflow/repository.py +++ b/backend/app/tickflow/repository.py @@ -12,10 +12,12 @@ """ from __future__ import annotations +import json import logging import sys import threading import time +import uuid from collections.abc import Callable from datetime import date from pathlib import Path @@ -320,6 +322,8 @@ class KlineRepository: self._warmup_lock = threading.Lock() # 预热完成后的回调 (lifespan 注入, 用于设置 app.state.indicators_ready) self._on_warmup_done: Callable[[], None] | None = None + # parquet/instruments 同步刷新完成后的轻量回调；用于调度派生缓存预热。 + self._on_refresh_done: Callable[[], None] | None = None # parquet glob 路径 self._enriched_glob = str(store.data_dir / "kline_daily_enriched" / "**" / "*.parquet") @@ -384,6 +388,7 @@ class KlineRepository: logger.info("cache refresh step start: enriched") self._refresh_enriched() logger.info("cache refresh step done: enriched (%.2fs)", time.perf_counter() - step) + self._notify_refresh_done() logger.info("cache refresh done (%.2fs)", time.perf_counter() - started) @@ -405,6 +410,7 @@ class KlineRepository: logger.info("enriched warmup thread started") self._refresh_enriched() logger.info("enriched warmup thread done (%.1fs)", time.perf_counter() - t0) + self._notify_refresh_done() except Exception: # noqa: BLE001 logger.exception("enriched warmup thread failed") finally: @@ -422,6 +428,15 @@ class KlineRepository: ) self._warmup_thread.start() + def _notify_refresh_done(self) -> None: + callback = self._on_refresh_done + if callback is None: + return + try: + callback() + except Exception: # noqa: BLE001 + logger.warning("repository refresh callback failed", exc_info=True) + @property def enriched_ready(self) -> bool: """enriched 缓存是否已就绪 (非 None 且不在后台预热中)。""" @@ -1571,6 +1586,48 @@ class KlineRepository: return None return None + def latest_enriched_date(self, asset_type: str = "stock") -> date | None: + """Return the newest partition available to matrix-native consumers.""" + dirname = enriched_dirname(asset_type) + root = self.store.data_dir / dirname + latest: date | None = None + if not root.exists(): + return None + for partition in root.glob("date=*"): + try: + value = date.fromisoformat(partition.name.removeprefix("date=")) + except ValueError: + continue + if latest is None or value > latest: + latest = value + return latest + + def get_matrix_data_generation(self, asset_type: str = "stock") -> str: + """Return a persistent generation bumped by every managed enriched write.""" + path = self.store.data_dir / f".matrix_generation_{asset_type}.json" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + generation = str(payload.get("generation") or "") + if generation: + return generation + except (OSError, TypeError, ValueError, json.JSONDecodeError): + pass + return self._bump_matrix_data_generation(asset_type) + + def _bump_matrix_data_generation(self, asset_type: str) -> str: + generation = uuid.uuid4().hex + path = self.store.data_dir / f".matrix_generation_{asset_type}.json" + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + temporary.write_text( + json.dumps({ + "generation": generation, + "updated_at_ns": time.time_ns(), + }, separators=(",", ":")), + encoding="utf-8", + ) + temporary.replace(path) + return generation + def symbols_lagging(self, reference_date: date, min_gap_days: int = 3) -> list[str]: """返回日K覆盖落后的标的: 其最新 bar 早于 reference_date - min_gap_days。 @@ -1782,6 +1839,12 @@ class KlineRepository: ) date_df = date_df.sort(["symbol", "date"]) self._atomic_write_parquet(date_df, out) + generation_asset = { + "kline_daily_enriched": "stock", + "kline_etf_enriched": "etf", + }.get(table) + if generation_asset is not None: + self._bump_matrix_data_generation(generation_asset) def merge_live_daily_asset(self, asset_type: str, df: pl.DataFrame) -> None: """按 symbol 合并当天指定资产日K分区。用于少量自选实时，不覆盖全市场。""" @@ -1852,6 +1915,8 @@ class KlineRepository: subset=["symbol", "date"], keep="last" ) self._atomic_write_parquet(df_storage.sort(["symbol"]), out) + if asset_type in {"stock", "etf"}: + self._bump_matrix_data_generation(asset_type) def flush_live_daily(self, df: pl.DataFrame) -> None: """覆写当天 kline_daily 分区 (实时行情落盘, 非merge)。""" @@ -1912,3 +1977,5 @@ class KlineRepository: out.parent.mkdir(parents=True, exist_ok=True) with self._write_lock: self._atomic_write_parquet(df_storage, out) + if asset_type in {"stock", "etf"}: + self._bump_matrix_data_generation(asset_type) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index f5a1422..42e0b9e 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -18,6 +18,8 @@ dependencies = [ "duckdb>=1.0", "pyarrow>=16.0", "pandas>=2.2", # 仅在 BacktestService 边界使用,见 §7.4 / ADR-19 + "psutil>=5.9", # 独立回测 worker 的峰值 RSS 与退出后内存指标 + "numba>=0.65.1", # Matrix 有效 K 线通用编译内核 "fastexcel>=0.10", # Polars 读取 xlsx/xls # TickFlow 官方 SDK "tickflow[all]>=0.1.23", @@ -45,8 +47,8 @@ legacy-cpu = [ "polars[rtcompat]>=1.0", ] -# 回测依赖 vectorbt → numba → llvmlite,体积大且 macOS/Intel 上无预构建 wheel 时 -# 需要 brew install cmake 现场编译。挪到可选 extras,主依赖瘦身。 +# vectorbt 还会引入绘图、交互组件等完整分析栈，仍保持为可选 extras。 +# Matrix 引擎只直接依赖上面的 numba/llvmlite 编译内核。 # 启用:`uv sync --extra backtest` backtest = [ "vectorbt>=0.26", diff --git a/backend/tests/backtest/test_dependencies.py b/backend/tests/backtest/test_dependencies.py new file mode 100644 index 0000000..2c3c424 --- /dev/null +++ b/backend/tests/backtest/test_dependencies.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import polars as pl + +from app.backtest.strategy import StrategyDependencyResolver +from app.strategy.engine import StrategyDef + + +def _strategy(**overrides) -> StrategyDef: + values = dict( + meta={"id": "deps", "scoring": {"momentum_20d": 1.0}, "order_by": "score"}, + basic_filter={"enabled": False}, + entry_signals=["signal_macd_golden"], + exit_signals=["signal_ma20_breakdown"], + stop_loss=None, + trailing_stop=None, + trailing_take_profit_activate=None, + trailing_take_profit_drawdown=None, + max_hold_days=None, + alerts=[], + filter_fn=lambda df, params: pl.col("rsi_14") < params["rsi_max"], + filter_history_fn=None, + lookback_days=20, + source="builtin", + ) + values.update(overrides) + return StrategyDef(**values) + + +def test_resolver_merges_signals_scoring_filter_and_execution_columns(): + plan = StrategyDependencyResolver().resolve( + _strategy(), + params={"rsi_max": 30}, + basic_filter={"enabled": False}, + entry_signals=["signal_macd_golden"], + exit_signals=["signal_ma20_breakdown"], + ) + + assert {"macd_dif", "macd_dea", "ma20", "momentum_20d", "rsi_14"} <= set(plan.indicator_columns) + assert {"signal_macd_golden", "signal_ma20_breakdown", "signal_limit_up", "signal_limit_down"} <= set(plan.signal_columns) + assert {"symbol", "date", "open", "high", "low", "close", "volume", "raw_close", "raw_high"} <= set(plan.base_columns) + assert "raw_low" not in plan.base_columns + assert "rsi_6" not in plan.indicator_columns + assert plan.full_feature_fallback is False + + +def test_history_strategy_without_required_features_falls_back_to_full(caplog): + strategy = _strategy( + filter_fn=None, + filter_history_fn=lambda df, params: df, + required_features=frozenset(), + source="custom", + ) + + plan = StrategyDependencyResolver().resolve( + strategy, + params={}, + basic_filter={"enabled": False}, + entry_signals=[], + exit_signals=[], + ) + + assert plan.full_feature_fallback is True + assert "rsi_14" in plan.indicator_columns + assert "falls back to full feature computation" in caplog.text + + +def test_history_strategy_required_features_avoids_fallback(): + strategy = _strategy( + filter_fn=None, + filter_history_fn=lambda df, params: df, + required_features=frozenset({"ma20", "momentum_20d"}), + source="custom", + ) + + plan = StrategyDependencyResolver().resolve( + strategy, + params={}, + basic_filter={"enabled": False}, + entry_signals=[], + exit_signals=[], + ) + + assert plan.full_feature_fallback is False + assert {"ma20", "momentum_20d"} <= set(plan.indicator_columns) + assert "rsi_14" not in plan.indicator_columns + + +def test_matrix_native_resolves_raw_fields_and_protocol_warmup_without_indicators(): + class NativeStrategy: + def required_fields(self): + return frozenset({"open", "high", "low", "close", "volume"}) + + def required_warmup_bars(self, params): + return 120 + + def compute_signals(self, market, params): # pragma: no cover - resolver only + raise AssertionError + + strategy = _strategy( + filter_fn=None, + filter_history_fn=None, + execution_backend="matrix_native", + matrix_strategy=NativeStrategy(), + required_features=frozenset(), + ) + plan = StrategyDependencyResolver().resolve( + strategy, + params={}, + basic_filter={"enabled": True, "amount_min": 100.0}, + entry_signals=[], + exit_signals=[], + ) + + assert plan.execution_backend == "matrix_native" + assert plan.indicator_columns == frozenset() + assert {"open", "high", "low", "close", "volume", "amount"} <= set(plan.base_columns) + assert plan.warmup_bars == 120 + assert plan.full_feature_fallback is False diff --git a/backend/tests/backtest/test_market_matrix.py b/backend/tests/backtest/test_market_matrix.py new file mode 100644 index 0000000..d70603d --- /dev/null +++ b/backend/tests/backtest/test_market_matrix.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +from dataclasses import asdict +from datetime import date, timedelta + +import numpy as np +import polars as pl +import pytest + +from app.backtest.engine import BacktestEngine, MatcherConfig, SimulationOptions +from app.backtest.matrix import build_market_matrix + + +def _row(symbol: str, day: int, price: float, **overrides) -> dict: + return { + "symbol": symbol, + "name": symbol, + "date": date(2024, 1, 1) + timedelta(days=day), + "open": overrides.get("open", price), + "high": overrides.get("high", price), + "low": overrides.get("low", price), + "close": overrides.get("close", price), + "volume": overrides.get("volume", 100_000), + "score": overrides.get("score", 0.0), + "signal_limit_up": overrides.get("signal_limit_up", False), + "signal_limit_down": overrides.get("signal_limit_down", False), + "signal_entry": overrides.get("signal_entry", False), + "signal_exit": overrides.get("signal_exit", False), + } + + +def test_sparse_mapping_is_stable_read_only_and_not_forward_filled(): + panel = pl.DataFrame([ + _row("B", 1, 20, signal_entry=True), + _row("A", 2, 12), + _row("A", 0, 10, signal_entry=True), + ]) + entries = panel["signal_entry"] + matrix = build_market_matrix(panel, entries, None) + + assert matrix.symbols == ("A", "B") + assert matrix.timestamp_labels == ("2024-01-01", "2024-01-02", "2024-01-03") + assert matrix.shape == (3, 2) + assert np.isnan(matrix.close[1, 0]) + assert matrix.tradable[1, 0] == 0 + assert matrix.entry[1, 0] == 0 + assert matrix.close.flags.writeable is False + assert matrix.entry.flags.writeable is False + + reversed_panel = panel.reverse() + reversed_matrix = build_market_matrix(reversed_panel, reversed_panel["signal_entry"], None) + np.testing.assert_allclose(matrix.close, reversed_matrix.close, equal_nan=True) + np.testing.assert_array_equal(matrix.entry, reversed_matrix.entry) + + +def test_duplicate_timestamp_symbol_is_rejected(): + panel = pl.DataFrame([_row("A", 0, 10), _row("A", 0, 11)]) + with pytest.raises(ValueError, match="unique timestamp/symbol"): + build_market_matrix(panel, None, None) + + +def test_intraday_timestamps_share_daily_session_id(): + panel = pl.DataFrame({ + "symbol": ["A", "A", "A"], + "datetime": [ + "2024-01-01 09:30:00", + "2024-01-01 10:30:00", + "2024-01-02 09:30:00", + ], + "open": [10.0, 10.1, 10.2], + "high": [10.0, 10.1, 10.2], + "low": [10.0, 10.1, 10.2], + "close": [10.0, 10.1, 10.2], + "volume": [100, 100, 100], + }).with_columns(pl.col("datetime").str.to_datetime()) + + matrix = build_market_matrix(panel, None, None) + assert matrix.session_ids.tolist() == [0, 0, 1] + + +def test_tradable_matches_legacy_suspension_rules(): + panel = pl.DataFrame([ + _row("A", 0, 10, volume=100), + _row("B", 0, 10, volume=0), + _row("C", 0, 10, volume=0, high=11, low=9), + _row("D", 0, 0, volume=100), + ]) + matrix = build_market_matrix(panel, None, None) + tradable = dict(zip(matrix.symbols, matrix.tradable[0].tolist())) + assert tradable == {"A": 1, "B": 0, "C": 1, "D": 0} + + +def test_open_t_plus_one_keeps_legacy_next_asset_bar_semantics(): + panel = pl.DataFrame([ + _row("A", 0, 10, signal_entry=True), + _row("B", 1, 20), + _row("A", 2, 12), + ]) + matrix = build_market_matrix( + panel, + panel["signal_entry"], + None, + entry_delay_bars=1, + entry_signal_ids=["signal_entry"], + ) + + assert matrix.entry[:, 0].tolist() == [0, 0, 1] + assert matrix.entry_signal_time[2, 0] == 0 + + +def test_matrix_matcher_matches_legacy_trade_records_and_equity(): + rows = [] + for symbol, score in (("A", 90), ("B", 80), ("C", 70)): + for day in range(5): + overrides = {"score": score} + if symbol == "A" and day == 2: + overrides.update(open=9, high=9, low=9, close=9, signal_limit_down=True) + if symbol == "B" and day == 3: + overrides.update(open=8.5, high=9, low=8, close=8.5) + rows.append(_row(symbol, day, 10 + day * 0.1, **overrides)) + panel = pl.DataFrame(rows).sort(["symbol", "date"]) + entries = pl.Series([ + row["date"] == date(2024, 1, 1) + for row in panel.select("date").iter_rows(named=True) + ]) + exits = pl.Series([ + row["symbol"] == "A" and row["date"] == date(2024, 1, 2) + for row in panel.select(["symbol", "date"]).iter_rows(named=True) + ]) + config = MatcherConfig( + matching="open_t+1", + fees_pct=0, + slippage_bps=0, + max_positions=2, + max_exposure_pct=0.8, + stop_loss_pct=0.1, + initial_capital=100_000, + ) + engine = BacktestEngine(repo=None) # type: ignore[arg-type] + + matrix_result = engine.simulate_portfolio(panel, entries, exits, config) + legacy_result = engine.simulate_portfolio_legacy(panel, entries, exits, config) + + assert [asdict(trade) for trade in matrix_result.trades] == [ + asdict(trade) for trade in legacy_result.trades + ] + assert matrix_result.equity_curve == legacy_result.equity_curve + assert matrix_result.drawdown_curve == legacy_result.drawdown_curve + assert matrix_result.stats["execution"] == legacy_result.stats["execution"] + assert matrix_result.stats["pending_exit_positions"] == legacy_result.stats["pending_exit_positions"] + + +def test_independent_matrix_matches_legacy_candidates(): + panel = pl.DataFrame([ + _row("A", 0, 10, signal_entry=True), + _row("A", 1, 11, signal_entry=True), + _row("A", 2, 12), + _row("A", 3, 9, low=8.5), + _row("A", 4, 10), + ]).sort(["symbol", "date"]) + entries = panel["signal_entry"] + exits = panel["signal_exit"] + config = MatcherConfig( + matching="close_t", + fees_pct=0, + slippage_bps=0, + max_hold_days=2, + stop_loss_pct=0.1, + ) + engine = BacktestEngine(repo=None) # type: ignore[arg-type] + + matrix_result = engine.simulate_independent_candidates(panel, entries, exits, config) + legacy_result = engine.simulate_independent_candidates_legacy(panel, entries, exits, config) + + assert [asdict(trade) for trade in matrix_result.trades] == [ + asdict(trade) for trade in legacy_result.trades + ] + assert matrix_result.stats["execution"] == legacy_result.stats["execution"] + + +def test_lightweight_portfolio_keeps_stats_without_curves_or_monte_carlo(monkeypatch): + panel = pl.DataFrame([ + _row("A", 0, 10, signal_entry=True), + _row("A", 1, 11), + _row("A", 2, 12, signal_exit=True), + _row("A", 3, 11), + ]).sort(["symbol", "date"]) + matrix = build_market_matrix( + panel, + panel["signal_entry"], + panel["signal_exit"], + ) + config = MatcherConfig( + matching="close_t", + fees_pct=0, + slippage_bps=0, + max_positions=1, + initial_capital=100_000, + ) + engine = BacktestEngine(repo=None) # type: ignore[arg-type] + full = engine.simulate_market_matrix(matrix, config) + + def unexpected(_pnls): + raise AssertionError("Monte Carlo should not run") + + monkeypatch.setattr(BacktestEngine, "_mc_drawdown_percentiles", unexpected) + light = engine.simulate_market_matrix( + matrix, + config, + options=SimulationOptions( + include_monte_carlo=False, + include_curves=False, + include_trades=False, + include_per_symbol_stats=False, + include_return_distribution=False, + ), + ) + + assert light.equity_curve == [] + assert light.drawdown_curve == [] + assert light.trades == [] + assert light.per_symbol_stats == [] + assert "mc_maxdd_p50" not in light.stats + for name in ("total_return", "annual_return", "max_drawdown", "sharpe", "sortino"): + assert light.stats[name] == full.stats[name] diff --git a/backend/tests/backtest/test_matrix_compute_cache.py b/backend/tests/backtest/test_matrix_compute_cache.py new file mode 100644 index 0000000..99ce254 --- /dev/null +++ b/backend/tests/backtest/test_matrix_compute_cache.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +from datetime import date, timedelta +from pathlib import Path + +import numpy as np +import pandas as pd +import polars as pl +import pytest + +from app.backtest.matrix import ( + MatrixComputeCache, + MatrixPipelineConfig, + MatrixStrategyPipeline, + build_market_data_matrix, + rolling_max, + rolling_mean, + rolling_min, + rolling_quantile, + rolling_std, + rolling_sum, + shift, +) +from app.strategy.engine import StrategyEngine + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def _market(scale: float = 1.0): + start = date(2024, 1, 1) + rows = [] + for asset_id, symbol in enumerate(("000001.SZ", "600000.SH")): + for time_id in range(140): + close = scale * (10.0 + asset_id + time_id * 0.01) + rows.append({ + "symbol": symbol, + "name": symbol, + "date": start + timedelta(days=time_id), + "open": close * 0.99, + "high": close * 1.01, + "low": close * 0.98, + "close": close, + "volume": 100_000.0 + time_id, + "amount": close * 100_000.0, + "signal_limit_up": False, + "signal_limit_down": False, + }) + return build_market_data_matrix(pl.DataFrame(rows), field_columns={"amount"}) + + +def test_cache_hits_are_read_only_and_temporary_arrays_use_content_fingerprint(): + market = _market() + cache = MatrixComputeCache(max_bytes=8 * 1024 * 1024) + + with cache.activate(market): + first = rolling_mean(market.close, 5) + second = rolling_mean(market.close, 5) + temp_first = rolling_mean(market.close * np.float32(2.0), 7) + temp_second = rolling_mean(market.close * np.float32(2.0), 7) + + assert first is second + assert temp_first is temp_second + assert first.flags.writeable is False + stats = cache.snapshot() + assert stats["operations"]["rolling_mean"]["hits"] == 2 + assert stats["fingerprint_bytes"] > 0 + + +def test_cache_key_separates_market_lineage_and_operator_parameters(): + first_market = _market() + second_market = _market() + cache = MatrixComputeCache(max_bytes=8 * 1024 * 1024) + + with cache.activate(first_market): + first_window = rolling_min(first_market.close, 3) + second_window = rolling_min(first_market.close, 4) + with cache.activate(second_market): + other_market = rolling_min(second_market.close, 3) + + assert first_window is not second_window + assert first_window is not other_market + assert cache.snapshot()["operations"]["rolling_min"]["misses"] == 3 + + +def test_cache_lru_evicts_by_bytes_and_close_releases_all_entries(): + market = _market() + item_bytes = market.close.nbytes + cache = MatrixComputeCache(max_bytes=item_bytes, max_item_bytes=item_bytes) + + with cache.activate(market): + rolling_min(market.close, 3) + rolling_max(market.close, 3) + + before_close = cache.snapshot() + assert before_close["entries"] == 1 + assert before_close["evictions"] == 1 + cache.close() + assert cache.snapshot()["current_bytes"] == 0 + with pytest.raises(RuntimeError, match="closed"), cache.activate(market): + pass + + +def test_shift_stays_out_of_cache_to_protect_expensive_working_set(): + market = _market() + cache = MatrixComputeCache(max_bytes=8 * 1024 * 1024) + + with cache.activate(market): + first = shift(market.close, 1) + second = shift(market.close, 1) + + assert first is not second + assert "shift" not in cache.snapshot()["operations"] + + +def test_additional_rolling_operators_match_pandas_and_hit_cache(): + market = _market() + cache = MatrixComputeCache(max_bytes=8 * 1024 * 1024) + expected = pd.DataFrame(market.close) + + with cache.activate(market): + actual_sum = rolling_sum(market.close, 5) + actual_std = rolling_std(market.close, 5) + actual_quantile = rolling_quantile(market.close, 5, 0.25) + assert rolling_sum(market.close, 5) is actual_sum + assert rolling_std(market.close, 5) is actual_std + assert rolling_quantile(market.close, 5, 0.25) is actual_quantile + + np.testing.assert_allclose(actual_sum, expected.rolling(5).sum(), equal_nan=True) + np.testing.assert_allclose(actual_std, expected.rolling(5).std(ddof=0), atol=1e-6, equal_nan=True) + np.testing.assert_allclose( + actual_quantile, + expected.rolling(5).quantile(0.25), + atol=1e-6, + equal_nan=True, + ) + operations = cache.snapshot()["operations"] + assert operations["rolling_sum"]["hits"] == 1 + assert operations["rolling_std"]["hits"] == 1 + assert operations["rolling_quantile"]["hits"] == 1 + + +def test_builtin_matrix_strategy_formula_is_unchanged_with_cache(): + market = _market() + strategy_path = ( + REPO_ROOT / "backend" / "app" / "strategy" / "builtin" / "macd_golden.py" + ) + strategy_def = StrategyEngine._load_file(strategy_path) + strategy = strategy_def.matrix_strategy + assert strategy is not None + params = {} + uncached = strategy.compute_signals(market, params) + cache = MatrixComputeCache(max_bytes=64 * 1024 * 1024) + + with cache.activate(market): + first = strategy.compute_signals(market, params) + second = strategy.compute_signals(market, params) + + np.testing.assert_array_equal(first.entry, uncached.entry) + np.testing.assert_array_equal(second.entry, uncached.entry) + assert cache.snapshot()["hits"] > 0 + + +def test_pipeline_reuses_basic_asset_filter_and_raw_scoring_features(): + market = _market() + cache = MatrixComputeCache(max_bytes=64 * 1024 * 1024) + + class AllEntries: + def compute_signals(self, market, params): + from app.backtest.matrix import make_signal_matrix + + return make_signal_matrix( + market.shape, + entry=np.ones(market.shape, dtype=np.uint8), + ) + + config = MatrixPipelineConfig( + basic_filter={"enabled": True, "amount_min": 1.0}, + scoring={"momentum_5d": 1.0}, + order_by="score", + descending=True, + asset_mask=np.array([True, False]), + ) + pipeline = MatrixStrategyPipeline() + with cache.activate(market): + first = pipeline.run(AllEntries(), market, {}, config) + second = pipeline.run(AllEntries(), market, {}, config) + + np.testing.assert_array_equal(first.entry, second.entry) + operations = cache.snapshot()["operations"] + assert operations["basic_filter_mask"]["hits"] == 1 + assert operations["pipeline_filter_mask"]["hits"] == 1 + assert operations["matrix_feature"]["hits"] == 1 + + +def test_pipeline_protects_strategy_working_set_when_scoring_would_overflow_cache(): + from app.backtest.matrix import make_signal_matrix + + market = _market() + item_bytes = market.close.nbytes + cache = MatrixComputeCache(max_bytes=item_bytes * 4) + + class RollingStrategy: + def compute_signals(self, market, params): + rolling_min(market.close, 3) + rolling_max(market.close, 4) + rolling_mean(market.close, 5) + return make_signal_matrix( + market.shape, + entry=np.ones(market.shape, dtype=np.uint8), + ) + + config = MatrixPipelineConfig( + basic_filter={"enabled": False}, + scoring={ + "momentum_5d": 0.3, + "change_pct": 0.3, + "vol_ratio_5d": 0.2, + "momentum_20d": 0.2, + }, + order_by="score", + descending=True, + protect_strategy_cache=True, + ) + pipeline = MatrixStrategyPipeline() + with cache.activate(market): + pipeline.run(RollingStrategy(), market, {}, config) + pipeline.run(RollingStrategy(), market, {}, config) + + operations = cache.snapshot()["operations"] + assert operations["rolling_min"]["hits"] == 1 + assert operations["rolling_max"]["hits"] == 1 + assert operations["rolling_mean"]["hits"] == 1 + assert "matrix_feature" not in operations + assert "basic_filter_mask" not in operations diff --git a/backend/tests/backtest/test_matrix_strategy.py b/backend/tests/backtest/test_matrix_strategy.py new file mode 100644 index 0000000..99e9d6f --- /dev/null +++ b/backend/tests/backtest/test_matrix_strategy.py @@ -0,0 +1,972 @@ +from __future__ import annotations + +import gc +from dataclasses import replace +from datetime import date, timedelta +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import polars as pl +import pytest + +from app.backtest import matrix as matrix_module +from app.backtest.matrix import ( + MatrixPipelineConfig, + MatrixStrategyPipeline, + RealtimeMarketDataMatrix, + apply_time_masks, + build_market_data_matrix, + build_matrix_score, + load_market_data_matrix_from_parquet, + make_signal_matrix, + matrix_feature, + slice_signal_matrix, + validate_signal_matrix, +) +from app.backtest.strategy import build_matrix_cache_profile +from app.indicators.pipeline import ( + compute_indicators, + compute_limit_signals, +) +from app.indicators.pipeline import ( + compute_signals as compute_indicator_signals, +) +from app.strategy.engine import StrategyDataContext, StrategyEngine + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def test_common_matrix_features_match_polars_indicator_pipeline(): + rows = [] + start = date(2024, 1, 1) + for offset in range(100): + close = 10.0 + offset * 0.03 + np.sin(offset / 4.0) * 0.8 + rows.append({ + "symbol": "000001.SZ", + "date": start + timedelta(days=offset), + "open": close - 0.1, + "high": close + 0.3 + (offset % 3) * 0.02, + "low": close - 0.25, + "close": close, + "volume": 1000.0 + (offset % 7) * 130.0, + }) + panel = pl.DataFrame(rows) + features = { + "prev_close", "change_pct", "change_amount", "amplitude", + "ma5", "ma20", "ma60", "boll_upper", "boll_lower", + "high_60d", "low_60d", "momentum_60d", "vol_ratio_5d", + "annual_vol_20d", "rsi_14", + } + enriched = compute_indicators(panel, needed=features) + market = build_market_data_matrix(panel) + + for name in sorted(features): + expected = enriched.sort(["date", "symbol"])[name].to_numpy() + actual = matrix_feature(market, name)[:, 0] + np.testing.assert_allclose(actual, expected, rtol=2e-5, atol=2e-5, equal_nan=True) + + +def _panel_with_missing_asset_bar() -> pl.DataFrame: + rows = [] + start = date(2024, 1, 1) + for offset in range(110): + for asset_id, symbol in enumerate(("000001.SZ", "600000.SH")): + if symbol == "000001.SZ" and offset == 43: + continue + if asset_id == 0: + close = ( + 20.0 - offset * 0.15 + if offset < 35 + else 14.75 + (offset - 35) * 0.25 + ) + else: + close = 18.0 + offset * 0.025 + np.sin(offset / 5.0) * 0.7 + rows.append({ + "symbol": symbol, + "date": start + timedelta(days=offset), + "open": close - 0.1, + "high": close + 0.3, + "low": close - 0.25, + "close": close, + "volume": 1000.0 + asset_id * 200.0 + (offset % 7) * 130.0, + }) + return pl.DataFrame(rows) + + +def test_matrix_features_skip_missing_asset_bars_like_polars_groups(): + panel = _panel_with_missing_asset_bar() + features = { + "prev_close", "change_pct", "change_amount", "amplitude", + "ma5", "ma20", "ma60", "boll_upper", "boll_lower", + "high_60d", "low_60d", "momentum_60d", "vol_ratio_5d", + "annual_vol_20d", "rsi_14", + } + enriched = compute_indicators(panel, needed=features) + market = build_market_data_matrix(panel) + time_id_by_date = { + label[:10]: time_id + for time_id, label in enumerate(market.timestamp_labels) + } + + for symbol in market.symbols: + expected_rows = enriched.filter(pl.col("symbol") == symbol).sort("date") + time_ids = np.array( + [time_id_by_date[str(value)] for value in expected_rows["date"]], + dtype=np.intp, + ) + asset_id = market.symbols.index(symbol) + for name in sorted(features): + expected = expected_rows[name].to_numpy() + actual = matrix_feature(market, name)[time_ids, asset_id] + np.testing.assert_allclose( + actual, + expected, + rtol=2e-5, + atol=2e-5, + equal_nan=True, + err_msg=f"{symbol} {name}", + ) + + missing_time_id = time_id_by_date["2024-02-13"] + missing_asset_id = market.symbols.index("000001.SZ") + for name in features: + assert np.isnan(matrix_feature(market, name)[missing_time_id, missing_asset_id]) + + +def test_matrix_pipeline_builds_and_reuses_one_compact_valid_bar_index(): + panel = _panel_with_missing_asset_bar() + market = build_market_data_matrix(panel) + base_bytes = market.nbytes + strategy = StrategyEngine._load_file( + REPO_ROOT / "backend" / "app" / "strategy" / "builtin" / "ma_golden_cross.py" + ).matrix_strategy + + with patch.object( + matrix_module, + "_build_valid_bar_index", + wraps=matrix_module._build_valid_bar_index, + ) as build_index: + MatrixStrategyPipeline().run( + strategy, + market, + { + "require_ma_golden": True, + "use_volume_filter": False, + "require_above_ma60": False, + }, + MatrixPipelineConfig( + basic_filter={"enabled": False}, + scoring={}, + order_by=None, + descending=True, + ), + ) + + index = market.valid_bars + assert build_index.call_count == 1 + assert index.rows.size == panel.height + assert index.offsets.tolist() == [0, 109, 219] + assert market.valid_bars is index + assert market.nbytes == base_bytes + index.nbytes + + +@pytest.mark.parametrize( + ("strategy_file", "entry_column", "exit_column", "params"), + [ + ( + "ma_golden_cross.py", + "signal_ma_golden_5_20", + "signal_ma_dead_5_20", + { + "require_ma_golden": True, + "use_volume_filter": False, + "require_above_ma60": False, + }, + ), + ( + "macd_golden.py", + "signal_macd_golden", + "signal_macd_dead", + {"require_macd_golden": True, "use_volume_filter": False}, + ), + ], +) +def test_matrix_crossovers_skip_missing_asset_bars_like_polars_signals( + strategy_file: str, + entry_column: str, + exit_column: str, + params: dict, +): + panel = _panel_with_missing_asset_bar() + indicator_names = ( + {"ma5", "ma20"} + if strategy_file == "ma_golden_cross.py" + else {"macd_dif", "macd_dea"} + ) + enriched = compute_indicators(panel, needed=indicator_names) + expected = compute_indicator_signals( + enriched, + needed={entry_column, exit_column}, + ) + market = build_market_data_matrix(panel) + strategy_def = StrategyEngine._load_file( + REPO_ROOT / "backend" / "app" / "strategy" / "builtin" / strategy_file + ) + actual = strategy_def.matrix_strategy.compute_signals(market, params) + time_id_by_date = { + label[:10]: time_id + for time_id, label in enumerate(market.timestamp_labels) + } + + for symbol in market.symbols: + expected_rows = expected.filter(pl.col("symbol") == symbol).sort("date") + time_ids = np.array( + [time_id_by_date[str(value)] for value in expected_rows["date"]], + dtype=np.intp, + ) + asset_id = market.symbols.index(symbol) + expected_entry = ( + expected_rows[entry_column].fill_null(False).cast(pl.UInt8).to_numpy() + ) + expected_exit = ( + expected_rows[exit_column].fill_null(False).cast(pl.UInt8).to_numpy() + ) + np.testing.assert_array_equal(actual.entry[time_ids, asset_id], expected_entry) + np.testing.assert_array_equal(actual.exit[time_ids, asset_id], expected_exit) + + missing_time_id = time_id_by_date["2024-02-13"] + missing_asset_id = market.symbols.index("000001.SZ") + assert actual.entry[missing_time_id, missing_asset_id] == 0 + assert actual.exit[missing_time_id, missing_asset_id] == 0 + assert actual.entry.any() or actual.exit.any() + + +def test_builtin_matrix_strategies_use_their_declared_formula_modules(): + strategy_dir = REPO_ROOT / "backend" / "app" / "strategy" / "builtin" + strategy_files = sorted( + path for path in strategy_dir.glob("*.py") if path.name != "__init__.py" + ) + + assert len(strategy_files) == 18 + for strategy_path in strategy_files: + strategy = StrategyEngine._load_file(strategy_path) + assert strategy.execution_backend == "matrix_native" + assert strategy.matrix_strategy is not None + assert strategy.matrix_strategy.__class__.__module__ == strategy_path.stem + assert strategy.filter_fn is None + assert strategy.filter_history_fn is None + + +def test_market_matrix_derives_live_raw_close_when_requested(): + panel = pl.DataFrame({ + "symbol": ["000001.SZ", "600000.SH"], + "date": [date(2024, 1, 1)] * 2, + "open": [10.0, 20.0], + "high": [10.2, 20.2], + "low": [9.8, 19.8], + "close": [10.1, 20.1], + "volume": [1_000.0, 2_000.0], + }) + market = build_market_data_matrix(panel, field_columns={"raw_close"}) + + np.testing.assert_array_equal(market.field("raw_close"), market.close) + + live = RealtimeMarketDataMatrix( + panel, + field_columns={"raw_close"}, + ) + live.update(panel.with_columns(pl.lit(date(2024, 1, 2)).alias("date"))) + snapshot = live.snapshot() + np.testing.assert_array_equal(snapshot.field("raw_close")[-1], snapshot.close[-1]) + + +def test_direct_parquet_matrix_matches_panel_builder_and_reuses_mmap(tmp_path): + market_root = tmp_path / "kline_daily_enriched" + days = (date(2024, 1, 2), date(2024, 1, 3), date(2024, 1, 4)) + rows = [] + closes = { + "000001.SZ": (10.0, 11.0, 12.0), + "000002.SZ": (10.0, 10.5, None), + "300001.SZ": (10.0, 12.0, 12.2), + } + for symbol, values in closes.items(): + for current, close in zip(days, values, strict=True): + if close is None: + continue + rows.append({ + "symbol": symbol, + "date": current, + "open": close, + "high": close, + "low": close, + "close": close, + "volume": 1_000.0, + "amount": close * 100_000.0, + "raw_close": close, + "raw_high": close, + "raw_low": close, + "turnover_rate": 1.5, + }) + panel = pl.DataFrame(rows).sort(["symbol", "date"]) + for current in days: + partition = market_root / f"date={current.isoformat()}" + partition.mkdir(parents=True) + panel.filter(pl.col("date") == current).write_parquet(partition / "part.parquet") + + instruments = pl.DataFrame({ + "symbol": ["000001.SZ", "000002.SZ", "300001.SZ"], + "name": ["普通", "ST测试", "创业板"], + "total_shares": [1_000_000.0, 2_000_000.0, 3_000_000.0], + "float_shares": [800_000.0, 1_500_000.0, 2_000_000.0], + "limit_up": [12.0, 11.0, 13.0], + "limit_down": [10.0, 9.0, 10.0], + }) + enriched = compute_limit_signals( + panel, + instruments, + needed={"signal_limit_up", "signal_limit_down"}, + ).join( + instruments.select("symbol", "name", "total_shares", "float_shares"), + on="symbol", + how="left", + ) + field_columns = { + "amount", + "raw_close", + "raw_high", + "raw_low", + "turnover_rate", + "total_shares", + "float_shares", + } + expected = build_market_data_matrix(enriched, field_columns=field_columns) + cache_root = tmp_path / "matrix_cache" + actual = load_market_data_matrix_from_parquet( + market_root, + days[0], + days[-1], + field_columns=field_columns, + instruments=instruments, + cache_root=cache_root, + ) + assert actual.cache_status == "built" + + assert actual.timestamp_labels == expected.timestamp_labels + assert actual.symbols == expected.symbols + assert actual.names == expected.names + for name in ( + "timestamps", + "session_ids", + "open", + "high", + "low", + "close", + "volume", + "tradable", + "limit_up_locked", + "limit_down_locked", + ): + np.testing.assert_array_equal(getattr(actual, name), getattr(expected, name)) + for name in field_columns: + np.testing.assert_array_equal(actual.field(name), expected.field(name)) + + cached = load_market_data_matrix_from_parquet( + market_root, + days[0], + days[-1], + field_columns=field_columns, + instruments=instruments, + cache_root=cache_root, + ) + assert cached.cache_status == "exact" + assert isinstance(cached.close, np.memmap) + assert not cached.close.flags.writeable + np.testing.assert_array_equal(cached.close, actual.close) + + latest_path = market_root / f"date={days[-1].isoformat()}" / "part.parquet" + latest = pl.read_parquet(latest_path).with_columns( + pl.when(pl.col("symbol") == "000001.SZ") + .then(12.5) + .otherwise(pl.col("close")) + .alias("close") + ) + latest.write_parquet(latest_path) + refreshed = load_market_data_matrix_from_parquet( + market_root, + days[0], + days[-1], + field_columns=field_columns, + instruments=instruments, + cache_root=cache_root, + ) + target_asset = refreshed.symbols.index("000001.SZ") + assert refreshed.close[-1, target_asset] == pytest.approx(12.5) + assert len(list(cache_root.glob("v*-*"))) == 2 + + +def test_covering_matrix_cache_reuses_wider_dates_and_fields(tmp_path): + market_root = tmp_path / "kline_daily_enriched" + days = [date(2024, 1, 2) + timedelta(days=offset) for offset in range(4)] + rows = [] + for current in days: + partition = market_root / f"date={current.isoformat()}" + partition.mkdir(parents=True) + for asset_id, symbol in enumerate(("000001.SZ", "600000.SH")): + close = 10.0 + asset_id + (current - days[0]).days + rows.append({ + "symbol": symbol, + "date": current, + "open": close, + "high": close, + "low": close, + "close": close, + "volume": 1_000.0, + "amount": close * 100_000.0, + "raw_close": close, + }) + pl.DataFrame([row for row in rows if row["date"] == current]).write_parquet( + partition / "part.parquet" + ) + instruments = pl.DataFrame({ + "symbol": ["000001.SZ", "600000.SH"], + "name": ["A", "B"], + }) + cache_root = tmp_path / "matrix_cache" + broad = load_market_data_matrix_from_parquet( + market_root, + days[0], + days[-1], + field_columns={"amount", "raw_close"}, + instruments=instruments, + cache_root=cache_root, + ) + narrow = load_market_data_matrix_from_parquet( + market_root, + days[1], + days[2], + field_columns={"raw_close"}, + instruments=instruments, + cache_root=cache_root, + ) + + assert broad.cache_status == "built" + assert narrow.cache_status == "covering" + assert narrow.cache_path == broad.cache_path + assert narrow.timestamp_labels == tuple(value.isoformat() for value in days[1:3]) + assert set(narrow.fields) == {"raw_close"} + assert isinstance(narrow.close, np.memmap) + assert not narrow.close.flags.writeable + assert len(list(cache_root.glob("v*-*"))) == 1 + + +def test_covering_cache_ignores_outside_slice_change_but_invalidates_inside(tmp_path): + market_root = tmp_path / "kline_daily_enriched" + days = [date(2024, 2, 1) + timedelta(days=offset) for offset in range(4)] + for offset, current in enumerate(days): + partition = market_root / f"date={current.isoformat()}" + partition.mkdir(parents=True) + pl.DataFrame({ + "symbol": ["000001.SZ"], + "date": [current], + "open": [10.0 + offset], + "high": [10.0 + offset], + "low": [10.0 + offset], + "close": [10.0 + offset], + "volume": [1_000.0], + }).write_parquet(partition / "part.parquet") + instruments = pl.DataFrame({"symbol": ["000001.SZ"], "name": ["A"]}) + cache_root = tmp_path / "matrix_cache" + load_market_data_matrix_from_parquet( + market_root, + days[0], + days[-1], + field_columns=set(), + instruments=instruments, + cache_root=cache_root, + ) + + outside_path = market_root / f"date={days[-1].isoformat()}" / "part.parquet" + pl.read_parquet(outside_path).with_columns(pl.lit(99.0).alias("close")).write_parquet( + outside_path + ) + outside = load_market_data_matrix_from_parquet( + market_root, + days[1], + days[2], + field_columns=set(), + instruments=instruments, + cache_root=cache_root, + ) + assert outside.cache_status == "covering" + + inside_path = market_root / f"date={days[2].isoformat()}" / "part.parquet" + pl.read_parquet(inside_path).with_columns(pl.lit(77.0).alias("close")).write_parquet( + inside_path + ) + inside = load_market_data_matrix_from_parquet( + market_root, + days[1], + days[2], + field_columns=set(), + instruments=instruments, + cache_root=cache_root, + ) + assert inside.cache_status == "built" + assert inside.close[-1, 0] == pytest.approx(77.0) + + +def test_matrix_cache_can_be_disabled(tmp_path): + market_root = tmp_path / "kline_daily_enriched" + current = date(2024, 3, 1) + partition = market_root / f"date={current.isoformat()}" + partition.mkdir(parents=True) + pl.DataFrame({ + "symbol": ["000001.SZ"], + "date": [current], + "open": [10.0], + "high": [10.0], + "low": [10.0], + "close": [10.0], + "volume": [1_000.0], + }).write_parquet(partition / "part.parquet") + + market = load_market_data_matrix_from_parquet( + market_root, + current, + current, + field_columns=set(), + cache_root=None, + ) + assert market.cache_status == "disabled" + assert not isinstance(market.close, np.memmap) + + +def test_matrix_cache_prunes_by_bytes_and_leaves_no_staging_directory(tmp_path): + market_root = tmp_path / "kline_daily_enriched" + current = date(2024, 3, 4) + partition = market_root / f"date={current.isoformat()}" + partition.mkdir(parents=True) + path = partition / "part.parquet" + + def write_close(value: float) -> None: + pl.DataFrame({ + "symbol": ["000001.SZ"], + "date": [current], + "open": [value], + "high": [value], + "low": [value], + "close": [value], + "volume": [1_000.0], + }).write_parquet(path) + + cache_root = tmp_path / "matrix_cache" + write_close(10.0) + first = load_market_data_matrix_from_parquet( + market_root, + current, + current, + field_columns=set(), + cache_root=cache_root, + cache_max_bytes=1, + ) + write_close(11.0) + second = load_market_data_matrix_from_parquet( + market_root, + current, + current, + field_columns=set(), + cache_root=cache_root, + cache_max_bytes=1, + ) + + assert first.cache_path != second.cache_path + del first + gc.collect() + assert second.close[0, 0] == pytest.approx(11.0) + assert len(list(cache_root.glob("v3-*"))) == 1 + assert list(cache_root.glob(".*.tmp")) == [] + assert len(list(cache_root.glob(".axes-v1-*.json"))) == 1 + + +def test_managed_source_generation_skips_file_walk_and_invalidates_explicitly(tmp_path): + market_root = tmp_path / "kline_daily_enriched" + current = date(2024, 3, 5) + partition = market_root / f"date={current.isoformat()}" + partition.mkdir(parents=True) + pl.DataFrame({ + "symbol": ["000001.SZ"], + "date": [current], + "open": [10.0], + "high": [10.0], + "low": [10.0], + "close": [10.0], + "volume": [1_000.0], + }).write_parquet(partition / "part.parquet") + cache_root = tmp_path / "matrix_cache" + + first = load_market_data_matrix_from_parquet( + market_root, + current, + current, + field_columns=set(), + cache_root=cache_root, + source_generation="generation-a", + ) + repeated = load_market_data_matrix_from_parquet( + market_root, + current, + current, + field_columns=set(), + cache_root=cache_root, + source_generation="generation-a", + ) + changed = load_market_data_matrix_from_parquet( + market_root, + current, + current, + field_columns=set(), + cache_root=cache_root, + source_generation="generation-b", + ) + + assert first.cache_status == "built" + assert repeated.cache_status == "exact" + assert repeated.cache_path == first.cache_path + assert changed.cache_status == "built" + assert changed.cache_path != first.cache_path + del first, repeated + gc.collect() + assert len(list(cache_root.glob("v3-*"))) == 1 + + +def test_registered_builtin_matrix_strategies_share_one_cache_profile(): + engine = StrategyEngine( + strategy_dirs=[REPO_ROOT / "backend" / "app" / "strategy" / "builtin"] + ) + profile = build_matrix_cache_profile(engine, "stock") + strategies = engine.strategy_definitions() + + assert len(strategies) == 18 + assert all(strategy.execution_backend == "matrix_native" for strategy in strategies) + assert profile.warmup_bars > 0 + assert profile.forward_bars == max(int(strategy.max_hold_days or 0) for strategy in strategies) + assert {"open", "high", "low", "close", "volume"}.issubset(profile.field_columns) + + +def test_chunked_matrix_score_matches_previous_full_matrix_formula(): + row_count = 4 + asset_count = 600 + symbols = [f"{asset_id:06d}.SZ" for asset_id in range(asset_count)] + rows = [] + rng = np.random.default_rng(20260715) + for time_id in range(row_count): + for asset_id, symbol in enumerate(symbols): + rows.append({ + "symbol": symbol, + "date": date(2024, 1, 1) + timedelta(days=time_id), + "open": 10.0, + "high": 10.0, + "low": 10.0, + "close": 10.0, + "volume": 1_000.0, + "feature_a": float(rng.normal()), + "feature_b": float((asset_id % 7) - 3), + }) + market = build_market_data_matrix( + pl.DataFrame(rows), + field_columns={"feature_a", "feature_b"}, + ) + universe = rng.random(market.shape) > 0.2 + weights = {"feature_a": 0.75, "feature_b": 0.25} + + expected = np.zeros(market.shape, dtype=np.float32) + all_finite = universe.copy() + total_weight = sum(weights.values()) + for name, weight in weights.items(): + values = market.field(name) + finite = universe & np.isfinite(values) + all_finite &= np.isfinite(values) + row_min = np.min(np.where(finite, values, np.inf), axis=1) + row_max = np.max(np.where(finite, values, -np.inf), axis=1) + row_range = row_max - row_min + normalized = np.zeros(market.shape, dtype=np.float32) + varying = finite & np.isfinite(row_range[:, None]) & (row_range[:, None] > 0) + equal = finite & ~varying + np.divide( + values - row_min[:, None], + row_range[:, None], + out=normalized, + where=varying, + ) + normalized[equal] = np.float32(0.5) + expected += normalized * np.float32(weight / total_weight) + expected *= np.float32(100.0) + expected[~universe | ~all_finite] = 0.0 + + actual = build_matrix_score( + market, + universe, + weights, + "score", + True, + fallback=np.zeros(market.shape, dtype=np.float32), + ) + np.testing.assert_array_equal(actual, expected) + + +def test_signal_slice_is_zero_copy_and_masking_only_allocates_final_flags(): + entry = np.ones((5, 2), dtype=np.uint8) + codes = np.arange(10, dtype=np.int16).reshape(5, 2) + score = np.arange(10, dtype=np.float32).reshape(5, 2) + signals = make_signal_matrix( + (5, 2), + entry=entry, + exit=entry, + score=score, + entry_signal_code=codes, + exit_signal_code=codes, + ) + sliced = slice_signal_matrix(signals, 1, 4) + assert np.shares_memory(sliced.entry, signals.entry) + assert np.shares_memory(sliced.score, signals.score) + + masked = apply_time_masks( + sliced, + np.array([True, False, True]), + np.array([False, True, True]), + ) + assert np.shares_memory(masked.score, signals.score) + assert masked.entry.tolist() == [[1, 1], [0, 0], [1, 1]] + assert masked.exit.tolist() == [[0, 0], [1, 1], [1, 1]] + assert masked.entry_signal_code[1].tolist() == [-1, -1] + assert masked.exit_signal_code[0].tolist() == [-1, -1] + assert not masked.entry.flags.writeable + assert signals.entry.tolist() == [[1, 1]] * 5 + + +def test_matrix_pipeline_applies_basic_filter_and_candidate_scoring(): + panel = pl.DataFrame({ + "symbol": ["000001.SZ", "600000.SH"], + "name": ["A", "B"], + "date": [date(2024, 1, 1)] * 2, + "open": [10.0, 20.0], + "high": [10.0, 20.0], + "low": [10.0, 20.0], + "close": [10.0, 20.0], + "volume": [100.0, 100.0], + "amount": [50.0, 500.0], + }) + market = build_market_data_matrix(panel, field_columns={"amount"}) + + class AllEntries: + def required_fields(self): + return frozenset({"close"}) + + def required_warmup_bars(self, params): + return 1 + + def compute_signals(self, market, params): + return make_signal_matrix(market.shape, entry=np.ones(market.shape, dtype=np.uint8)) + + signals = MatrixStrategyPipeline().run( + AllEntries(), + market, + {}, + MatrixPipelineConfig( + basic_filter={"enabled": True, "amount_min": 100.0}, + scoring={"close": 1.0}, + order_by="score", + descending=True, + ), + ) + + assert signals.entry.tolist() == [[0, 1]] + assert signals.score.tolist() == [[0.0, 50.0]] + + +def test_signal_matrix_validation_rejects_mutable_strategy_output(): + signals = make_signal_matrix((2, 1)) + mutable_entry = signals.entry.copy() + invalid = type(signals)( + entry=mutable_entry, + exit=signals.exit, + score=signals.score, + entry_signal_code=signals.entry_signal_code, + exit_signal_code=signals.exit_signal_code, + ) + with pytest.raises(ValueError, match="read-only"): + validate_signal_matrix(invalid, (2, 1)) + + +def test_matrix_pipeline_applies_asset_pool_before_cross_sectional_scoring(): + panel = pl.DataFrame({ + "symbol": ["000001.SZ", "600000.SH"], + "name": ["A", "B"], + "date": [date(2024, 1, 1)] * 2, + "open": [10.0, 20.0], + "high": [10.0, 20.0], + "low": [10.0, 20.0], + "close": [10.0, 20.0], + "volume": [100.0, 100.0], + }) + market = build_market_data_matrix(panel) + + class AllEntries: + def required_fields(self): + return frozenset({"close"}) + + def required_warmup_bars(self, params): + return 1 + + def compute_signals(self, market, params): + return make_signal_matrix(market.shape, entry=np.ones(market.shape, dtype=np.uint8)) + + signals = MatrixStrategyPipeline().run( + AllEntries(), + market, + {}, + MatrixPipelineConfig( + basic_filter={"enabled": False}, + scoring={"close": 1.0}, + order_by="score", + descending=True, + asset_mask=np.array([False, True]), + ), + ) + + assert signals.entry.tolist() == [[0, 1]] + assert signals.score.tolist() == [[0.0, 50.0]] + + +def test_realtime_market_matrix_overwrites_last_row_and_appends_new_bar(): + panel = pl.DataFrame({ + "symbol": ["000001.SZ", "600000.SH"] * 2, + "name": ["A", "B"] * 2, + "date": [date(2024, 1, 1)] * 2 + [date(2024, 1, 2)] * 2, + "open": [10.0, 20.0, 10.5, 20.5], + "high": [10.2, 20.2, 10.7, 20.7], + "low": [9.8, 19.8, 10.3, 20.3], + "close": [10.1, 20.1, 10.6, 20.6], + "volume": [1_000.0, 2_000.0, 1_100.0, 2_100.0], + "amount": [10_100.0, 40_200.0, 11_660.0, 43_260.0], + }) + buffer = RealtimeMarketDataMatrix(panel, field_columns={"amount"}) + + same_bar = panel.filter(pl.col("date") == date(2024, 1, 2)).with_columns( + (pl.col("close") + 5.0).alias("close") + ) + buffer.update(same_bar) + snapshot = buffer.snapshot() + assert snapshot.shape == (2, 2) + assert snapshot.close[-1].tolist() == pytest.approx(same_bar.sort("symbol")["close"].to_list()) + assert buffer.build_count == 1 + assert buffer.update_count == 1 + assert not snapshot.close.flags.writeable + + next_bar = same_bar.with_columns(pl.lit(date(2024, 1, 3)).alias("date")) + buffer.update(next_bar) + assert buffer.snapshot().shape == (3, 2) + assert buffer.build_count == 1 + assert buffer.update_count == 2 + + +def test_strategy_engine_runs_matrix_strategy_without_legacy_filter(): + start = date(2024, 1, 1) + rows = [] + for offset in range(65): + values = ( + ("000001.SZ", 10.0 + offset * 0.05), + ("600000.SH", 20.0 + offset * 0.03), + ) + for symbol, close in values: + rows.append({ + "symbol": symbol, + "name": symbol, + "date": start + timedelta(days=offset), + "open": close, + "high": close * 1.01, + "low": close * 0.99, + "close": close, + "volume": 1_000_000.0, + "amount": 100_000_000.0, + "total_shares": 1_000_000_000.0, + "float_shares": 800_000_000.0, + }) + history = pl.DataFrame(rows) + target = start + timedelta(days=64) + engine = StrategyEngine( + strategy_dirs=[REPO_ROOT / "backend" / "app" / "strategy" / "builtin"], + ) + strategy = engine.get("macd_golden") + assert strategy.filter_fn is None + + result = engine.run( + "macd_golden", + StrategyDataContext( + asset_type="stock", + timeframe="1d", + as_of=target, + current=history.filter(pl.col("date") == target), + history=history, + ), + pool=["600000.SH"], + params={"require_macd_golden": False, "use_volume_filter": False}, + overrides={"basic_filter": {"enabled": False}}, + ) + + assert [row["symbol"] for row in result.rows] == ["600000.SH"] + assert result.total == 1 + + +def test_strategy_engine_run_all_builds_one_shared_matrix(): + start = date(2024, 1, 1) + rows = [] + for offset in range(65): + close = 10.0 + offset * 0.05 + rows.append({ + "symbol": "000001.SZ", + "name": "A", + "date": start + timedelta(days=offset), + "open": close, + "high": close * 1.01, + "low": close * 0.99, + "close": close, + "volume": 1_000_000.0, + "amount": 100_000_000.0, + "total_shares": 1_000_000_000.0, + "float_shares": 800_000_000.0, + }) + history = pl.DataFrame(rows) + target = start + timedelta(days=64) + engine = StrategyEngine( + strategy_dirs=[REPO_ROOT / "backend" / "app" / "strategy" / "builtin"], + ) + original = engine.get("macd_golden") + engine._strategies["macd_copy"] = replace( + original, + meta={**original.meta, "id": "macd_copy"}, + ) + params = {"require_macd_golden": False, "use_volume_filter": False} + overrides = {"basic_filter": {"enabled": False}} + + with patch( + "app.backtest.matrix.build_market_data_matrix", + wraps=matrix_module.build_market_data_matrix, + ) as build: + results = engine.run_all( + StrategyDataContext( + asset_type="stock", + timeframe="1d", + as_of=target, + current=history.filter(pl.col("date") == target), + history=history, + ), + strategy_ids=["macd_golden", "macd_copy"], + params_map={"macd_golden": params, "macd_copy": params}, + overrides_map={"macd_golden": overrides, "macd_copy": overrides}, + ) + + assert build.call_count == 1 + assert results["macd_golden"].total == 1 + assert results["macd_copy"].total == 1 diff --git a/backend/tests/backtest/test_optimizer_api.py b/backend/tests/backtest/test_optimizer_api.py index d21dc9a..e72f00b 100644 --- a/backend/tests/backtest/test_optimizer_api.py +++ b/backend/tests/backtest/test_optimizer_api.py @@ -22,6 +22,10 @@ def test_job_key_distinguishes_grid_and_objective(): base = _make_opt_job_key("s", None, None, None, '{"p":[1,2]}', "sortino", None, sig) assert base != _make_opt_job_key("s", None, None, None, '{"p":[1,3]}', "sortino", None, sig) # grid 不同 assert base != _make_opt_job_key("s", None, None, None, '{"p":[1,2]}', "sharpe", None, sig) # objective 不同 + assert base != _make_opt_job_key( + "s", None, None, None, '{"p":[1,2]}', "sortino", None, sig, + matrix_cache_max_mb=256, + ) def test_cancel_looks_up_job_by_echoed_key(): @@ -58,3 +62,27 @@ def test_cancel_looks_up_job_by_echoed_key(): assert res3["ok"] is False finally: _running_jobs.pop(key, None) + + +def test_finished_job_is_proactively_removed_after_ttl(monkeypatch): + from app.api import backtest as api + + class _ImmediateTimer: + daemon = False + + def __init__(self, interval, callback): + self.interval = interval + self.callback = callback + + def start(self): + self.callback() + + monkeypatch.setattr(api.threading, "Timer", _ImmediateTimer) + job = api._BacktestJob("finished-job") + api._running_jobs[job.key] = job + + api._finish_job(job, result={"ok": True}) + + assert job.done is True + assert job.result == {"ok": True} + assert job.key not in api._running_jobs diff --git a/backend/tests/backtest/test_optimizer_run.py b/backend/tests/backtest/test_optimizer_run.py index 8008c4b..8103b66 100644 --- a/backend/tests/backtest/test_optimizer_run.py +++ b/backend/tests/backtest/test_optimizer_run.py @@ -14,11 +14,15 @@ from app.backtest.optimizer import OptimizeConfig, StrategyOptimizer @dataclass class _FakeDef: meta: dict + execution_backend: str = "polars_expr" class _FakeEngine: - def __init__(self, params_meta): - self._def = _FakeDef(meta={"params": params_meta}) + def __init__(self, params_meta, execution_backend="polars_expr"): + self._def = _FakeDef( + meta={"params": params_meta}, + execution_backend=execution_backend, + ) def get(self, strategy_id): return self._def @@ -30,17 +34,50 @@ class _FakeResult: error: str | None = None +class _FakeCache: + def __init__(self): + self.closed = False + + def snapshot(self): + return {"current_bytes": 64, "hits": 2} + + def close(self): + self.closed = True + + class _FakeService: """run() 依据 params 返回受控 stats: sortino = ma_proximity 的映射, 便于校验排序。""" def __init__(self, score_fn): self.score_fn = score_fn self.calls = [] + self.prepared_calls = [] + self.prepared_values = [] + self.result_policies = [] + self.matrix_cache_max_bytes = None + self.cache = _FakeCache() self._lock = threading.Lock() - def run(self, config, progress_cb=None, cancel_event=None): + def prepare_matrix_optimization(self, configs, *, matrix_cache_max_bytes): + self.prepared_calls.append(configs) + self.matrix_cache_max_bytes = matrix_cache_max_bytes + return type("Prepared", (), { + "market_data": type("Market", (), {"nbytes": 1234})(), + "compute_cache": self.cache, + })() + + def run( + self, + config, + progress_cb=None, + cancel_event=None, + prepared=None, + result_policy=None, + ): with self._lock: self.calls.append(dict(config.params or {})) + self.prepared_values.append(prepared) + self.result_policies.append(result_policy) return self.score_fn(config.params or {}) @@ -49,8 +86,11 @@ PARAMS_META = [ ] -def _optimizer(score_fn): - return StrategyOptimizer(_FakeService(score_fn), _FakeEngine(PARAMS_META)) +def _optimizer(score_fn, execution_backend="polars_expr"): + return StrategyOptimizer( + _FakeService(score_fn), + _FakeEngine(PARAMS_META, execution_backend=execution_backend), + ) def _cfg(**kw): @@ -82,8 +122,13 @@ def test_all_combos_executed_once(): out = opt.optimize(_cfg(param_grid={"ma_proximity": [0.01, 0.02, 0.03, 0.04, 0.05]})) assert out["n_combinations"] == 5 # 每组恰跑一次 - ran = sorted(c["ma_proximity"] for c in opt.service.calls) + ran = sorted( + call["ma_proximity"] + for call, policy in zip(opt.service.calls, opt.service.result_policies, strict=True) + if policy is not None + ) assert ran == [0.01, 0.02, 0.03, 0.04, 0.05] + assert opt.service.result_policies.count(None) == 1 def test_min_direction_objective_restores_display_sign(): @@ -144,7 +189,9 @@ def test_base_params_merged_and_overridden_by_sweep(): opt = _optimizer(score) opt.optimize(_cfg(base_params={"ma_proximity": 0.99, "other": 7})) # 每次 run 收到的 params: ma_proximity 被 combo 覆盖, other 保留 - for call in opt.service.calls: + for call, policy in zip(opt.service.calls, opt.service.result_policies, strict=True): + if policy is None: + continue assert call["other"] == 7 assert call["ma_proximity"] in (0.01, 0.02, 0.03) @@ -187,11 +234,126 @@ def test_progress_callback_reports_done_total(): def cb(msg): seen.append(msg) _optimizer(score).optimize(_cfg(), progress_cb=cb) - assert len(seen) == 3 + assert len(seen) == 4 + assert seen[-1]["type"] == "optimizer_finalize" assert seen[-1]["done"] == 3 assert all(m["total"] == 3 for m in seen) +def test_matrix_optimizer_prepares_once_and_reuses_same_market_data(): + def score(p): + return _FakeResult(stats={"sortino": p["ma_proximity"]}) + + opt = _optimizer(score, execution_backend="matrix_native") + out = opt.optimize(_cfg(max_workers=8)) + + assert len(opt.service.prepared_calls) == 1 + assert len(opt.service.prepared_calls[0]) == 3 + assert len({id(value) for value in opt.service.prepared_values}) == 1 + assert opt.service.prepared_values[0] is not None + assert out["requested_max_workers"] == 8 + assert out["effective_workers"] == 1 + assert out["shared_market_data"] is True + assert out["shared_market_data_bytes"] == 1234 + assert out["best_backtest"] is not None + assert out["matrix_compute_cache"]["released"] is True + assert opt.service.cache.closed is True + assert opt.service.matrix_cache_max_bytes == 512 * 1024 * 1024 + + +def test_optimizer_trial_policy_skips_mc_but_best_backtest_is_full(): + def score(p): + return _FakeResult(stats={"sortino": p["ma_proximity"]}) + + opt = _optimizer(score) + opt.optimize(_cfg()) + + trial_policies = [policy for policy in opt.service.result_policies if policy is not None] + assert trial_policies + assert all(policy.include_monte_carlo is False for policy in trial_policies) + assert opt.service.result_policies[-1] is None + + +def test_mc_objective_trial_policy_keeps_monte_carlo(): + def score(p): + return _FakeResult(stats={"mc_maxdd_p95": -p["ma_proximity"]}) + + opt = _optimizer(score) + opt.optimize(_cfg(objective="mc_maxdd_p95")) + trial_policies = [policy for policy in opt.service.result_policies if policy is not None] + assert all(policy.include_monte_carlo is True for policy in trial_policies) + + +def test_missing_objective_is_an_explicit_trial_error(): + def score(p): + return _FakeResult(stats={"sharpe": 1.0}) + + out = _optimizer(score).optimize(_cfg()) + assert out["best_params"] is None + assert all("缺少优化目标字段" in row["error"] for row in out["results"]) + + +def test_matrix_cache_closes_when_progress_callback_raises_after_prepare(): + def score(p): + return _FakeResult(stats={"sortino": 1.0}) + + opt = _optimizer(score, execution_backend="matrix_native") + + def fail_on_prepare(message): + if message["type"] == "optimizer_prepare": + raise RuntimeError("progress failed") + + with pytest.raises(RuntimeError, match="progress failed"): + opt.optimize(_cfg(), progress_cb=fail_on_prepare) + assert opt.service.cache.closed is True + + +def test_matrix_cache_closes_when_cancelled_after_first_trial(): + event = threading.Event() + + def score(p): + event.set() + return _FakeResult(stats={"sortino": 1.0}) + + opt = _optimizer(score, execution_backend="matrix_native") + out = opt.optimize(_cfg(), cancel_event=event) + + assert out["n_completed"] == 1 + assert out["best_backtest"] is None + assert opt.service.cache.closed is True + + +def test_matrix_cache_closes_when_best_backtest_raises(): + call_count = 0 + + def score(p): + nonlocal call_count + call_count += 1 + if call_count == 4: + raise RuntimeError("final failed") + return _FakeResult(stats={"sortino": p["ma_proximity"]}) + + opt = _optimizer(score, execution_backend="matrix_native") + with pytest.raises(RuntimeError, match="final failed"): + opt.optimize(_cfg()) + assert opt.service.cache.closed is True + + +def test_cancelled_matrix_optimizer_skips_expensive_preparation(): + event = threading.Event() + event.set() + + def score(p): + return _FakeResult(stats={"sortino": 1.0}) + + opt = _optimizer(score, execution_backend="matrix_native") + out = opt.optimize(_cfg(), cancel_event=event) + + assert opt.service.prepared_calls == [] + assert opt.service.calls == [] + assert out["n_completed"] == 0 + + def test_invalid_objective_rejected(): def score(p): return _FakeResult(stats={"sortino": 1.0}) diff --git a/backend/tests/backtest/test_robustness_metrics.py b/backend/tests/backtest/test_robustness_metrics.py index 8f3d7e4..eec80b5 100644 --- a/backend/tests/backtest/test_robustness_metrics.py +++ b/backend/tests/backtest/test_robustness_metrics.py @@ -12,7 +12,7 @@ from datetime import date import numpy as np -from app.backtest.engine import BacktestEngine, TradeRecord +from app.backtest.engine import BacktestEngine, SimulationOptions, TradeRecord # --------------------------------------------------------------- # Sortino @@ -167,6 +167,33 @@ def test_independent_candidate_stats_emits_sortino_and_mc(): assert result.stats["mc_maxdd_p50"] is not None +def test_lightweight_candidate_stats_do_not_call_monte_carlo(monkeypatch): + trades = _trades([0.10, -0.05, 0.08, -0.06, 0.03], [2, 1, 3, 2, 4]) + + def unexpected(_pnls): + raise AssertionError("Monte Carlo should not run") + + monkeypatch.setattr(BacktestEngine, "_mc_drawdown_percentiles", unexpected) + result = BacktestEngine._calc_independent_candidate_result( + trades, + n_candidates=5, + execution_stats={}, + options=SimulationOptions( + include_monte_carlo=False, + include_curves=False, + include_trades=False, + include_per_symbol_stats=False, + include_return_distribution=False, + ), + ) + + assert "mc_maxdd_p50" not in result.stats + assert result.equity_curve == [] + assert result.drawdown_curve == [] + assert result.trades == [] + assert result.per_symbol_stats == [] + + def test_calc_stats_all_wins_reports_sortino_none(): """全盈利交易在 stats 集成层: 无下行波动 → sortino 序列化为 None (非 0)。""" trades = _trades([0.10, 0.05, 0.08], [3, 2, 4]) diff --git a/backend/tests/backtest/test_strategy_backtest_correctness.py b/backend/tests/backtest/test_strategy_backtest_correctness.py index a788fc1..5df49fd 100644 --- a/backend/tests/backtest/test_strategy_backtest_correctness.py +++ b/backend/tests/backtest/test_strategy_backtest_correctness.py @@ -3,9 +3,11 @@ from __future__ import annotations from datetime import date, timedelta from types import SimpleNamespace +import numpy as np import polars as pl from app.backtest.engine import BacktestEngine, SimResult +from app.backtest.matrix import build_market_data_matrix, make_signal_matrix, rolling_mean from app.backtest.strategy import StrategyBacktestConfig, StrategyBacktestService from app.strategy.engine import StrategyDef @@ -50,14 +52,43 @@ class _EngineStub: self.panel = panel self.repo = _RepoStub() self.load_args = None + self.load_count = 0 self.sim_panel: pl.DataFrame | None = None + self.sim_matrix = None self.sim_entries: pl.Series | None = None def load_panel(self, symbols, start: date, end: date, columns=None, asset_type: str = "stock") -> pl.DataFrame: + self.load_count += 1 self.load_args = (symbols, start, end) self.load_asset_type = asset_type return self.panel + def load_panel_for_backtest(self, symbols, start, end, feature_plan, asset_type="stock") -> pl.DataFrame: + return self.load_panel(symbols, start, end, columns=sorted(feature_plan.base_columns), asset_type=asset_type) + + def load_market_data_matrix_for_backtest( + self, + symbols, + start, + end, + feature_plan, + asset_type="stock", + **kwargs, + ): + panel = self.load_panel_for_backtest( + symbols, + start, + end, + feature_plan, + asset_type=asset_type, + ) + field_columns = ( + set(feature_plan.base_columns) + | set(feature_plan.instrument_columns) + | set(feature_plan.matrix_columns) + ) + return build_market_data_matrix(panel, field_columns=field_columns) + def simulate_portfolio(self, panel, entries, exits, config, progress_cb=None, cancel_event=None, entry_signal_ids=None, exit_signal_ids=None) -> SimResult: self.sim_panel = panel self.sim_entries = entries @@ -69,6 +100,23 @@ class _EngineStub: stats={"total_return": 0.0, "n_trades": 0}, ) + def simulate_market_matrix( + self, + matrix, + config, + progress_cb=None, + cancel_event=None, + options=None, + ) -> SimResult: + self.sim_matrix = matrix + return SimResult( + equity_curve=[{"date": "2024-01-01", "value": config.initial_capital}], + drawdown_curve=[{"date": "2024-01-01", "value": 0.0}], + trades=[], + per_symbol_stats=[], + stats={"total_return": 0.0, "n_trades": 0}, + ) + def test_basic_filter_only_limits_entries_not_panel_rows(): start = date(2024, 1, 1) @@ -101,11 +149,9 @@ def test_basic_filter_only_limits_entries_not_panel_rows(): )) assert result.error is None - assert engine.sim_panel is not None - assert engine.sim_panel.height == 3 - assert engine.sim_panel.filter(pl.col("amount") == 0.0).height == 1 - assert engine.sim_entries is not None - assert engine.sim_entries.to_list() == [True, False, True] + assert engine.sim_matrix is not None + assert engine.sim_matrix.shape == (3, 1) + assert engine.sim_matrix.entry[:, 0].tolist() == [1, 0, 1] assert engine.load_args is not None assert engine.load_args[1] < start # warmup 只用于计算, 不参与正式交易 @@ -136,7 +182,7 @@ def test_full_mode_executes_every_candidate_with_strategy_rules(): ]).sort(["symbol", "date"]) engine = BacktestEngine(repo=None) # type: ignore[arg-type] - engine.load_panel = lambda symbols, s, e, columns=None, asset_type="stock": panel # type: ignore[method-assign] + engine.load_panel_for_backtest = lambda symbols, s, e, plan, asset_type="stock": panel # type: ignore[method-assign] strategy = _strategy( filter_fn=lambda df, params: pl.col("date") == start, max_hold_days=1, @@ -162,3 +208,249 @@ def test_full_mode_executes_every_candidate_with_strategy_rules(): assert result.trades[0]["entry_date"] == str(start + timedelta(days=1)) assert result.trades[0]["exit_reason"] == "max_hold" assert result.stats["avg_return"] == round(20 / 11 - 1, 4) + + +def test_matrix_native_strategy_uses_shared_orchestrator_path(): + start = date(2024, 1, 1) + panel = pl.DataFrame([ + {"symbol": "A", "name": "A", "date": start, "open": 10.0, "high": 10.0, "low": 10.0, "close": 10.0, "volume": 1.0, "amount": 1000.0, "raw_close": 10.0, "raw_high": 10.0, "signal_limit_up": False, "signal_limit_down": False}, + {"symbol": "A", "name": "A", "date": start + timedelta(days=1), "open": 11.0, "high": 11.0, "low": 11.0, "close": 11.0, "volume": 1.0, "amount": 1000.0, "raw_close": 11.0, "raw_high": 11.0, "signal_limit_up": False, "signal_limit_down": False}, + ]) + + class NativeStrategy: + def required_fields(self): + return frozenset({"open", "high", "low", "close", "volume"}) + + def required_warmup_bars(self, params): + return 1 + + def compute_signals(self, market, params): + return make_signal_matrix( + market.shape, + entry=np.ones(market.shape, dtype=np.uint8), + ) + + engine = _EngineStub(panel) + strategy = _strategy( + meta={"id": "native", "name": "native", "scoring": {}, "params": [], "limit": 100}, + basic_filter={"enabled": True, "amount_min": 100.0}, + filter_fn=None, + execution_backend="matrix_native", + matrix_strategy=NativeStrategy(), + required_features=frozenset({"amount"}), + ) + service = StrategyBacktestService(engine=engine, strategy_engine=_StrategyEngineStub(strategy)) + + result = service.run(StrategyBacktestConfig( + strategy_id="native", + symbols=None, + start=start, + end=start + timedelta(days=1), + matching="close_t", + mode="position", + )) + + assert result.error is None + assert engine.sim_matrix is not None + assert engine.sim_matrix.entry[:, 0].tolist() == [1, 1] + assert result.stats["execution_backend"] == "matrix_native" + + +def test_matrix_native_accepts_legacy_default_signal_overrides_but_rejects_replacements(): + start = date(2024, 1, 1) + panel = pl.DataFrame([ + {"symbol": "A", "name": "A", "date": start, "open": 10.0, "high": 10.0, "low": 10.0, "close": 10.0, "volume": 1.0, "amount": 1000.0, "raw_close": 10.0, "raw_high": 10.0, "signal_limit_up": False, "signal_limit_down": False}, + {"symbol": "A", "name": "A", "date": start + timedelta(days=1), "open": 11.0, "high": 11.0, "low": 11.0, "close": 11.0, "volume": 1.0, "amount": 1000.0, "raw_close": 11.0, "raw_high": 11.0, "signal_limit_up": False, "signal_limit_down": False}, + ]) + + class NativeStrategy: + def required_fields(self): + return frozenset({"open", "high", "low", "close", "volume"}) + + def required_warmup_bars(self, params): + return 1 + + def compute_signals(self, market, params): + return make_signal_matrix(market.shape, entry=np.ones(market.shape, dtype=np.uint8)) + + engine = _EngineStub(panel) + strategy = _strategy( + meta={"id": "native_defaults", "name": "native", "scoring": {}, "params": [], "limit": 100}, + filter_fn=None, + execution_backend="matrix_native", + matrix_strategy=NativeStrategy(), + entry_signals=["signal_custom_entry"], + exit_signals=["signal_custom_exit"], + required_features=frozenset(), + ) + service = StrategyBacktestService(engine=engine, strategy_engine=_StrategyEngineStub(strategy)) + + accepted = service.run(StrategyBacktestConfig( + strategy_id="native_defaults", + symbols=None, + start=start, + end=start + timedelta(days=1), + matching="close_t", + overrides={ + "entry_signals": ["signal_custom_entry"], + "exit_signals": ["signal_custom_exit"], + }, + )) + assert accepted.error is None + + rejected = service.run(StrategyBacktestConfig( + strategy_id="native_defaults", + symbols=None, + start=start, + end=start + timedelta(days=1), + matching="close_t", + overrides={"entry_signals": ["signal_other"]}, + )) + assert rejected.error == "matrix_native 策略的进出场信号由策略协议生成，不支持列信号覆盖" + + +def test_matrix_optimizer_preparation_loads_and_builds_base_data_once(): + start = date(2024, 1, 1) + panel = pl.DataFrame([ + {"symbol": "A", "name": "A", "date": start, "open": 10.0, "high": 10.0, "low": 10.0, "close": 10.0, "volume": 1.0, "amount": 1000.0, "raw_close": 10.0, "raw_high": 10.0, "signal_limit_up": False, "signal_limit_down": False}, + {"symbol": "A", "name": "A", "date": start + timedelta(days=1), "open": 11.0, "high": 11.0, "low": 11.0, "close": 11.0, "volume": 1.0, "amount": 1000.0, "raw_close": 11.0, "raw_high": 11.0, "signal_limit_up": False, "signal_limit_down": False}, + ]) + + class NativeStrategy: + def required_fields(self): + return frozenset({"open", "high", "low", "close", "volume"}) + + def required_warmup_bars(self, params): + return int(params.get("warmup", 1)) + + def compute_signals(self, market, params): + return make_signal_matrix( + market.shape, + entry=np.ones(market.shape, dtype=np.uint8), + ) + + engine = _EngineStub(panel) + strategy = _strategy( + meta={ + "id": "native", + "name": "native", + "scoring": {}, + "params": [{"id": "warmup", "type": "int", "default": 1, "min": 1, "max": 10}], + "limit": 100, + }, + basic_filter={"enabled": False}, + filter_fn=None, + execution_backend="matrix_native", + matrix_strategy=NativeStrategy(), + required_features=frozenset({"amount"}), + ) + service = StrategyBacktestService(engine=engine, strategy_engine=_StrategyEngineStub(strategy)) + configs = [ + StrategyBacktestConfig( + strategy_id="native", + symbols=None, + start=start, + end=start + timedelta(days=1), + params={"warmup": warmup}, + matching="close_t", + ) + for warmup in (1, 10) + ] + + prepared = service.prepare_matrix_optimization(configs) + results = [service.run(config, prepared=prepared) for config in configs] + + assert engine.load_count == 1 + assert prepared.market_data.nbytes > 0 + assert all(result.error is None for result in results) + assert all(result.stats["shared_market_data"] is True for result in results) + assert all(result.stats["shared_market_data_bytes"] == prepared.market_data.nbytes for result in results) + + +def test_matrix_cache_preserves_trades_daily_equity_and_core_stats(): + start = date(2024, 1, 1) + panel = pl.DataFrame([ + { + "symbol": "A", + "name": "A", + "date": start + timedelta(days=offset), + "open": close, + "high": close, + "low": close, + "close": close, + "volume": 1000.0, + "amount": close * 1000.0, + "raw_close": close, + "raw_high": close, + "signal_limit_up": False, + "signal_limit_down": False, + } + for offset, close in enumerate((10.0, 11.0, 12.0, 11.0, 13.0, 12.0)) + ]) + + class RollingEntry: + def required_fields(self): + return frozenset({"close"}) + + def required_warmup_bars(self, params): + return 2 + + def compute_signals(self, market, params): + entry = market.close >= rolling_mean(market.close, 2) + return make_signal_matrix(market.shape, entry=entry.astype(np.uint8)) + + engine = BacktestEngine(repo=None) # type: ignore[arg-type] + engine.load_market_data_matrix_for_backtest = ( # type: ignore[method-assign] + lambda symbols, s, e, plan, asset_type="stock", **kwargs: build_market_data_matrix( + panel, + field_columns=( + set(plan.base_columns) + | set(plan.instrument_columns) + | set(plan.matrix_columns) + ), + ) + ) + strategy = _strategy( + meta={"id": "rolling", "name": "rolling", "scoring": {}, "params": [], "limit": 100}, + basic_filter={"enabled": False}, + filter_fn=None, + execution_backend="matrix_native", + matrix_strategy=RollingEntry(), + required_features=frozenset({"amount"}), + max_hold_days=1, + ) + service = StrategyBacktestService(engine=engine, strategy_engine=_StrategyEngineStub(strategy)) + config = StrategyBacktestConfig( + strategy_id="rolling", + symbols=None, + start=start, + end=start + timedelta(days=5), + matching="close_t", + fees_pct=0, + slippage_bps=0, + max_positions=1, + ) + + uncached = service.run(config) + prepared = service.prepare_matrix_optimization([config]) + cached = service.run(config, prepared=prepared) + cached_again = service.run(config, prepared=prepared) + prepared.compute_cache.close() + + assert uncached.error is None + assert cached.error is None + assert cached_again.error is None + assert cached.trades == uncached.trades + assert cached.equity_curve == uncached.equity_curve + assert cached.drawdown_curve == uncached.drawdown_curve + for name in ( + "total_return", + "annual_return", + "max_drawdown", + "sharpe", + "sortino", + "n_trades", + ): + assert cached.stats[name] == uncached.stats[name] + assert cached_again.stats[name] == uncached.stats[name] + assert cached_again.stats["matrix_compute_cache"]["hits"] > 0 diff --git a/backend/tests/backtest/test_worker_process.py b/backend/tests/backtest/test_worker_process.py new file mode 100644 index 0000000..e3c5d97 --- /dev/null +++ b/backend/tests/backtest/test_worker_process.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +from datetime import date, timedelta + +import polars as pl + +from app.backtest.optimizer import OptimizeConfig +from app.backtest.strategy import StrategyBacktestConfig +from app.backtest.walkforward import WalkForwardConfig +from app.backtest.worker import make_worker_task, run_worker_task + + +def _write_worker_strategy(data_dir) -> None: + strategy_dir = data_dir / "strategies" / "custom" + strategy_dir.mkdir(parents=True) + (strategy_dir / "worker_always_entry.py").write_text( + """import numpy as np +from app.backtest.matrix import make_signal_matrix + +META = { + "id": "worker_always_entry", + "name": "worker", + "asset_types": ["stock"], + "timeframes": ["1d"], + "params": [ + {"id": "gate", "type": "int", "default": 1, "min": 1, "max": 2, "step": 1}, + ], + "scoring": {}, + "required_features": ["open", "high", "low", "close", "volume"], +} +EXECUTION_BACKEND = "matrix_native" +ENTRY_SIGNALS = [] +EXIT_SIGNALS = [] +STOP_LOSS = None +MAX_HOLD_DAYS = 1 +ALERTS = [] + +class AlwaysEntry: + def required_fields(self): + return frozenset({"open", "high", "low", "close", "volume"}) + + def required_warmup_bars(self, params): + return 1 + + def compute_signals(self, market, params): + return make_signal_matrix( + market.shape, + entry=np.ones(market.shape, dtype=np.uint8), + ) + +MATRIX_STRATEGY = AlwaysEntry() +""", + encoding="utf-8", + ) + + +def _write_market_data(data_dir, start: date, days: int = 3) -> None: + rows = [] + for offset in range(days): + close = 10.0 + offset + current = start + timedelta(days=offset) + rows.append({ + "symbol": "600000.SH", + "date": current, + "open": close, + "high": close, + "low": close, + "close": close, + "volume": 1000.0, + "amount": close * 100000.0, + "raw_close": close, + "raw_high": close, + "raw_low": close, + "turnover_rate": 1.0, + "consecutive_limit_ups": 0, + "consecutive_limit_downs": 0, + }) + partition = data_dir / "kline_daily_enriched" / f"date={current.isoformat()}" + partition.mkdir(parents=True) + pl.DataFrame([rows[-1]]).write_parquet(partition / "part.parquet") + + instruments_dir = data_dir / "instruments" + instruments_dir.mkdir(parents=True) + pl.DataFrame({ + "symbol": ["600000.SH"], + "name": ["浦发银行"], + "total_shares": [1_000_000_000.0], + "float_shares": [1_000_000_000.0], + }).write_parquet(instruments_dir / "part.parquet") + + +def test_spawn_worker_returns_compact_result_and_memory_metrics(tmp_path): + start = date(2024, 1, 1) + data_dir = tmp_path / "data" + _write_worker_strategy(data_dir) + _write_market_data(data_dir, start) + config = StrategyBacktestConfig( + strategy_id="worker_always_entry", + symbols=["600000.SH"], + start=start, + end=start + timedelta(days=2), + overrides={"basic_filter": {"enabled": False}}, + matching="close_t", + fees_pct=0, + slippage_bps=0, + max_positions=1, + ) + + result = run_worker_task(make_worker_task("backtest", data_dir, config)) + + assert result["error"] is None + assert result["stats"]["execution_backend"] == "matrix_native" + worker = result["stats"]["worker"] + assert worker["peak_rss_bytes"] > 0 + assert worker["serialized_result_bytes"] > 0 + assert worker["worker_exitcode"] == 0 + assert worker["parent_rss_after_worker_exit_bytes"] > 0 + + +def test_spawn_optimizer_reuses_one_matrix_and_exits(tmp_path): + start = date(2024, 1, 1) + data_dir = tmp_path / "data" + _write_worker_strategy(data_dir) + _write_market_data(data_dir, start) + config = OptimizeConfig( + strategy_id="worker_always_entry", + symbols=["600000.SH"], + start=start, + end=start + timedelta(days=2), + param_grid={"gate": [1, 2]}, + objective="total_return", + max_workers=4, + overrides={"basic_filter": {"enabled": False}}, + backtest_kwargs={ + "matching": "close_t", + "fees_pct": 0, + "slippage_bps": 0, + "max_positions": 1, + }, + ) + + result = run_worker_task(make_worker_task("optimize", data_dir, config)) + + assert result["n_completed"] == 2 + assert result["effective_workers"] == 1 + assert result["shared_market_data"] is True + assert result["shared_market_data_bytes"] > 0 + assert result["best_backtest"]["equity_curve"] + assert result["best_backtest"]["trades"] + assert "mc_maxdd_p50" in result["best_backtest"]["stats"] + assert result["matrix_compute_cache"]["released"] is True + assert result["performance"]["trial_peak_rss_bytes"] > 0 + assert result["performance"]["best_backtest_peak_rss_bytes"] > 0 + assert result["performance"]["trials_per_second"] > 0 + assert result["worker"]["worker_exitcode"] == 0 + + +def test_spawn_walkforward_reuses_shared_matrix_across_folds(tmp_path): + start = date(2024, 1, 1) + data_dir = tmp_path / "data" + _write_worker_strategy(data_dir) + _write_market_data(data_dir, start, days=8) + config = WalkForwardConfig( + strategy_id="worker_always_entry", + symbols=["600000.SH"], + start=start, + end=start + timedelta(days=7), + param_grid={"gate": [1, 2]}, + objective="total_return", + train_days=2, + test_days=1, + step_days=2, + overrides={"basic_filter": {"enabled": False}}, + backtest_kwargs={ + "matching": "close_t", + "fees_pct": 0, + "slippage_bps": 0, + "max_positions": 1, + }, + ) + + result = run_worker_task(make_worker_task("walkforward", data_dir, config)) + + assert result["n_folds"] == 2 + assert result["shared_market_data"] is True + assert result["shared_market_data_bytes"] > 0 + assert all(fold["oos_stats"]["shared_market_data"] for fold in result["folds"]) + assert result["worker"]["worker_exitcode"] == 0 diff --git a/backend/tests/test_ai_generator_prompt.py b/backend/tests/test_ai_generator_prompt.py index c4e91c7..17d82b5 100644 --- a/backend/tests/test_ai_generator_prompt.py +++ b/backend/tests/test_ai_generator_prompt.py @@ -27,3 +27,20 @@ def test_build_step1_keeps_user_prompt_compact(): assert "模式 A 框架" not in prompt assert "策略ID（必须使用此ID）：ai_test" in prompt assert len(prompt) < 1000 + + +def test_matrix_backend_prompt_and_imports_are_supported(): + prompt = build_step1( + "矩阵策略", + "矩阵原生示例", + "long", + "收盘价站上 MA20", + "ai_matrix", + "matrix_native", + ) + assert "执行后端：matrix_native" in prompt + + AIStrategyGenerator._validate_safety( + "import numpy as np\n" + "from app.backtest.matrix import MarketDataMatrix, SignalMatrix, make_signal_matrix\n" + ) diff --git a/backend/tests/test_high_turnover_strategy.py b/backend/tests/test_high_turnover_strategy.py index c330cfb..bb4bace 100644 --- a/backend/tests/test_high_turnover_strategy.py +++ b/backend/tests/test_high_turnover_strategy.py @@ -1,18 +1,33 @@ from __future__ import annotations +from datetime import date + import polars as pl +from app.backtest.matrix import build_market_data_matrix from app.strategy.builtin import high_turnover_surge def test_high_turnover_surge_uses_percent_value_turnover_rate(): - df = pl.DataFrame({ - "symbol": ["low", "hit"], - "turnover_rate": [4.9, 5.1], - "change_pct": [0.04, 0.04], + panel = pl.DataFrame({ + "symbol": ["low", "hit", "low", "hit"], + "date": [date(2024, 1, 2), date(2024, 1, 2), date(2024, 1, 3), date(2024, 1, 3)], + "open": [100.0, 100.0, 104.0, 104.0], + "high": [100.0, 100.0, 104.0, 104.0], + "low": [100.0, 100.0, 104.0, 104.0], + "close": [100.0, 100.0, 104.0, 104.0], + "volume": [1000.0, 1000.0, 1000.0, 1000.0], + "turnover_rate": [4.9, 5.1, 4.9, 5.1], }) + market = build_market_data_matrix(panel, field_columns={"turnover_rate"}) + signals = high_turnover_surge.MATRIX_STRATEGY.compute_signals( + market, + {"min_turnover": 5.0, "min_change": 3.0}, + ) - expr = high_turnover_surge.filter(df, {"min_turnover": 5.0, "min_change": 3.0}) - out = df.filter(expr) - - assert out["symbol"].to_list() == ["hit"] + selected = [ + symbol + for symbol, hit in zip(market.symbols, signals.entry[-1], strict=True) + if hit + ] + assert selected == ["hit"] diff --git a/backend/tests/test_indicator_needed.py b/backend/tests/test_indicator_needed.py new file mode 100644 index 0000000..fe4d7ba --- /dev/null +++ b/backend/tests/test_indicator_needed.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from datetime import date, timedelta + +import polars as pl + +from app.indicators.pipeline import compute_indicators, compute_limit_signals, compute_signals + + +def _bars(n: int = 90) -> pl.DataFrame: + rows = [] + for symbol, offset in (("600000", 0.0), ("300001", 2.0)): + for i in range(n): + close = 10.0 + offset + i * 0.03 + ((i % 7) - 3) * 0.04 + rows.append({ + "symbol": symbol, + "date": date(2024, 1, 1) + timedelta(days=i), + "open": close - 0.02, + "high": close + 0.10, + "low": close - 0.10, + "close": close, + "volume": 1000 + i * 10, + "amount": close * (1000 + i * 10), + "raw_close": close, + "raw_high": close + 0.10, + "raw_low": close - 0.10, + }) + return pl.DataFrame(rows) + + +def test_compute_signals_subset_matches_full_values(): + indicators = compute_indicators(_bars()) + full = compute_signals(indicators) + subset = compute_signals(indicators, needed={"signal_macd_golden", "signal_volume_surge"}) + + assert "signal_macd_golden" in subset.columns + assert "signal_volume_surge" in subset.columns + assert "signal_ma20_breakout" not in subset.columns + assert subset["signal_macd_golden"].equals(full["signal_macd_golden"]) + assert subset["signal_volume_surge"].equals(full["signal_volume_surge"]) + + +def test_compute_signals_empty_needed_adds_no_signals(): + indicators = compute_indicators(_bars(), needed={"ma20"}) + result = compute_signals(indicators, needed=set()) + assert not any(col.startswith(("signal_", "csg_")) for col in result.columns) + + +def test_compute_limit_signals_subset_matches_full_and_prunes_other_outputs(): + bars = compute_indicators(_bars(), needed={"change_pct"}) + instruments = pl.DataFrame({ + "symbol": ["600000", "300001"], + "name": ["浦发银行", "测试股份"], + "float_shares": [1_000_000_000.0, 500_000_000.0], + }) + full = compute_limit_signals(bars, instruments) + subset = compute_limit_signals(bars, instruments, needed={"signal_limit_up"}) + + assert "signal_limit_up" in subset.columns + assert "signal_limit_down" not in subset.columns + assert "signal_broken_limit_up" not in subset.columns + assert subset["signal_limit_up"].equals(full["signal_limit_up"]) diff --git a/backend/tests/test_screener_builtin_params.py b/backend/tests/test_screener_builtin_params.py index c4aa425..ea734e3 100644 --- a/backend/tests/test_screener_builtin_params.py +++ b/backend/tests/test_screener_builtin_params.py @@ -1,96 +1,11 @@ from __future__ import annotations -import random import types from datetime import date -from pathlib import Path from typing import ClassVar -import polars as pl - from app.api import screener as screener_api -from app.services.screener import PRESET_STRATEGIES, ScreenerResult, ScreenerService -from app.strategy.engine import StrategyEngine - - -def _builtin_engine() -> StrategyEngine: - builtin_dir = Path(__file__).parents[1] / "app" / "strategy" / "builtin" - return StrategyEngine( - enriched_loader=lambda _as_of: pl.DataFrame(), - strategy_dirs=[builtin_dir], - ) - - -def _comparison_frame() -> pl.DataFrame: - rng = random.Random(20260715) - size = 200 - return pl.DataFrame({ - "symbol": [f"S{i:03d}" for i in range(size)], - "close": [rng.uniform(5, 100) for _ in range(size)], - "open": [rng.uniform(5, 100) for _ in range(size)], - "ma5": [rng.uniform(5, 100) for _ in range(size)], - "ma10": [rng.uniform(5, 100) for _ in range(size)], - "ma20": [rng.uniform(5, 100) for _ in range(size)], - "ma60": [rng.uniform(5, 100) for _ in range(size)], - "vol_ratio_5d": [rng.uniform(0, 4) for _ in range(size)], - "momentum_20d": [rng.uniform(-0.5, 0.5) for _ in range(size)], - "momentum_60d": [rng.uniform(-0.5, 0.5) for _ in range(size)], - "annual_vol_20d": [rng.uniform(0, 0.6) for _ in range(size)], - "change_pct": [rng.uniform(-0.1, 0.1) for _ in range(size)], - "rsi_14": [rng.uniform(0, 100) for _ in range(size)], - "consecutive_limit_ups": [rng.randrange(0, 5) for _ in range(size)], - "signal_n_day_high": [rng.choice([True, False, None]) for _ in range(size)], - "signal_ma_golden_5_20": [rng.choice([True, False, None]) for _ in range(size)], - "signal_macd_golden": [rng.choice([True, False, None]) for _ in range(size)], - "signal_ma20_breakout": [rng.choice([True, False, None]) for _ in range(size)], - "signal_limit_up": [rng.choice([True, False, None]) for _ in range(size)], - "signal_boll_breakout_upper": [rng.choice([True, False, None]) for _ in range(size)], - "signal_n_day_low": [rng.choice([True, False, None]) for _ in range(size)], - }) - - -def test_builtin_default_filters_match_legacy_presets(): - engine = _builtin_engine() - df = _comparison_frame() - - for strategy_id, preset in PRESET_STRATEGIES.items(): - strategy = engine.get(strategy_id) - defaults = {item["id"]: item["default"] for item in strategy.meta["params"]} - expected = df.filter(preset["filter"])["symbol"].to_list() - actual = df.filter(strategy.filter_fn(df, defaults))["symbol"].to_list() - assert actual == expected, strategy_id - - -def test_run_preset_applies_numeric_and_boolean_strategy_params(): - engine = _builtin_engine() - strategy = engine.get("trend_breakout") - df = pl.DataFrame({ - "symbol": ["A", "B", "C"], - "close": [10.0, 10.0, 10.0], - "ma60": [9.0, 9.0, 9.0], - "signal_n_day_high": [True, True, True], - "vol_ratio_5d": [1.0, 2.5, 3.5], - "momentum_60d": [0.1, 0.2, 0.3], - }) - service = ScreenerService(types.SimpleNamespace()) - - strict = service.run_preset( - "trend_breakout", - as_of=date(2026, 7, 15), - precomputed=df, - filter_fn=strategy.filter_fn, - strategy_params={"vol_ratio_min": 3.0}, - ) - volume_disabled = service.run_preset( - "trend_breakout", - as_of=date(2026, 7, 15), - precomputed=df, - filter_fn=strategy.filter_fn, - strategy_params={"use_volume_filter": False}, - ) - - assert [row["symbol"] for row in strict.rows] == ["C"] - assert [row["symbol"] for row in volume_disabled.rows] == ["C", "B", "A"] +from app.services.screener import ScreenerResult class _CapturingScreenerService: @@ -103,12 +18,53 @@ class _CapturingScreenerService: def latest_date(self): return date(2026, 7, 15) - def _load_enriched_for_date(self, _as_of): - return pl.DataFrame({"symbol": ["A"]}) + def build_strategy_context( + self, + engine, + as_of, + strategy_ids, + *, + timeframe="1d", + params_map=None, + overrides_map=None, + ): + self.calls.append({ + "kind": "context", + "strategy_ids": strategy_ids, + "timeframe": timeframe, + "params_map": params_map, + "overrides_map": overrides_map, + }) + return types.SimpleNamespace(as_of=as_of) - def run_preset(self, strategy_id, as_of, **kwargs): - self.calls.append({"strategy_id": strategy_id, **kwargs}) - return ScreenerResult(as_of=as_of, strategy=strategy_id) + +class _CapturingStrategyEngine: + calls: ClassVar[list[dict]] = [] + + def has(self, strategy_id): + return strategy_id == "builtin_strategy" + + def run(self, strategy_id, context, *, pool=None, params=None, overrides=None): + self.calls.append({ + "kind": "run", + "strategy_id": strategy_id, + "pool": pool, + "params": params, + "overrides": overrides, + }) + return ScreenerResult(as_of=context.as_of, strategy=strategy_id) + + def run_all(self, context, *, params_map=None, overrides_map=None, strategy_ids=None): + self.calls.append({ + "kind": "run_all", + "params_map": params_map, + "overrides_map": overrides_map, + "strategy_ids": strategy_ids, + }) + return { + strategy_id: ScreenerResult(as_of=context.as_of, strategy=strategy_id) + for strategy_id in strategy_ids or [] + } def _api_request(tmp_path, engine): @@ -117,50 +73,59 @@ def _api_request(tmp_path, engine): return types.SimpleNamespace(app=types.SimpleNamespace(state=state)) -def test_single_run_passes_saved_params_to_builtin_filter(monkeypatch, tmp_path): - engine = _builtin_engine() - request = _api_request(tmp_path, engine) +def _install_api_fakes(monkeypatch): _CapturingScreenerService.calls = [] + _CapturingStrategyEngine.calls = [] monkeypatch.setattr(screener_api, "ScreenerService", _CapturingScreenerService) - monkeypatch.setattr( - screener_api.strategy_config, - "load_override", - lambda *_args: {"params": {"vol_ratio_min": 3.0}}, - ) monkeypatch.setattr(screener_api, "_load_ext_value_maps", lambda *_args: {}) monkeypatch.setattr(screener_api, "_update_cache_strategy", lambda *_args: None) + monkeypatch.setattr(screener_api.strategy_cache, "write_cache", lambda *_args: None) + + +def test_single_run_passes_saved_params_to_strategy_engine(monkeypatch, tmp_path): + engine = _CapturingStrategyEngine() + request = _api_request(tmp_path, engine) + _install_api_fakes(monkeypatch) + saved = {"params": {"threshold": 3.0, "enabled": False}} + monkeypatch.setattr(screener_api.strategy_config, "load_override", lambda *_args: saved) screener_api.run_preset( screener_api.PresetRequest( - strategy_id="trend_breakout", + strategy_id="builtin_strategy", as_of=date(2026, 7, 15), ), request, ) - call = _CapturingScreenerService.calls[0] - assert call["filter_fn"] is not None - assert call["strategy_params"] == {"vol_ratio_min": 3.0} + context_call = _CapturingScreenerService.calls[0] + run_call = _CapturingStrategyEngine.calls[0] + assert context_call["params_map"] == {"builtin_strategy": saved["params"]} + assert context_call["overrides_map"] == {"builtin_strategy": saved} + assert run_call["params"] == saved["params"] + assert run_call["overrides"] == saved -def test_batch_run_passes_saved_params_to_builtin_filter(monkeypatch, tmp_path): - engine = _builtin_engine() +def test_batch_run_passes_saved_params_to_strategy_engine(monkeypatch, tmp_path): + engine = _CapturingStrategyEngine() request = _api_request(tmp_path, engine) - _CapturingScreenerService.calls = [] - monkeypatch.setattr(screener_api, "ScreenerService", _CapturingScreenerService) + _install_api_fakes(monkeypatch) + saved = {"params": {"threshold": 3.0, "enabled": False}} monkeypatch.setattr( screener_api.strategy_config, "list_overrides", - lambda *_args: {"trend_breakout": {"params": {"vol_ratio_min": 3.0}}}, + lambda *_args: {"builtin_strategy": saved}, ) - monkeypatch.setattr(screener_api.strategy_cache, "write_cache", lambda *_args: None) - monkeypatch.setattr(screener_api, "_load_ext_value_maps", lambda *_args: {}) screener_api.run_all( request, - body={"as_of": "2026-07-15", "strategy_ids": ["trend_breakout"]}, + body={"as_of": "2026-07-15", "strategy_ids": ["builtin_strategy"]}, ) - call = _CapturingScreenerService.calls[0] - assert call["filter_fn"] is not None - assert call["strategy_params"] == {"vol_ratio_min": 3.0} + context_call = _CapturingScreenerService.calls[0] + run_all_call = _CapturingStrategyEngine.calls[0] + expected_params = {"builtin_strategy": saved["params"]} + expected_overrides = {"builtin_strategy": saved} + assert context_call["params_map"] == expected_params + assert context_call["overrides_map"] == expected_overrides + assert run_all_call["params_map"] == expected_params + assert run_all_call["overrides_map"] == expected_overrides diff --git a/backend/tests/test_screener_etf.py b/backend/tests/test_screener_etf.py index 18d10be..c201f40 100644 --- a/backend/tests/test_screener_etf.py +++ b/backend/tests/test_screener_etf.py @@ -1,46 +1,20 @@ -from app.services.screener import ( - PRESET_STRATEGIES, - strategy_supports_asset, -) - - -def test_all_presets_have_asset_types(): - for sid, strat in PRESET_STRATEGIES.items(): - assert "asset_types" in strat, f"{sid} 缺 asset_types" - assert "stock" in strat["asset_types"], f"{sid} 必须支持 stock" - - -def test_limit_up_strategies_are_stock_only(): - for sid in ("broken_board_recovery", "consecutive_limit_ups"): - assert PRESET_STRATEGIES[sid]["asset_types"] == ["stock"] - - -def test_pure_technical_strategies_support_etf(): - for sid in ( - "trend_breakout", "ma_golden_cross", "macd_golden", - "volume_price_surge", "low_volatility_leader", "oversold_bounce", - "boll_breakout", "bullish_alignment", "pullback_to_support", - "n_day_low_reversal", - ): - assert "etf" in PRESET_STRATEGIES[sid]["asset_types"], sid - - -def test_strategy_supports_asset_defaults_to_stock(): - assert strategy_supports_asset({}, "stock") is True - assert strategy_supports_asset({}, "etf") is False - assert strategy_supports_asset({"asset_types": ["stock", "etf"]}, "etf") is True - +from __future__ import annotations import types -from datetime import date +from datetime import date, timedelta +from pathlib import Path import polars as pl +import pytest from app.services.screener import ScreenerService +from app.strategy.engine import StrategyDataContext, StrategyEngine + +BUILTIN_DIR = Path(__file__).resolve().parents[1] / "app" / "strategy" / "builtin" class _FakeRepo: - """最小 repo 桩：只实现 screener 用到的 _asset 取数接口。""" + """最小 repo 桩: 只实现 screener 数据上下文需要的资产取数接口。""" def __init__(self, data_dir, enriched=None, instruments=None, latest=None): self.store = types.SimpleNamespace(data_dir=data_dir) @@ -55,7 +29,93 @@ class _FakeRepo: return self._instruments def get_enriched_history(self, target_date, lookback_days): - return None # stock 缓存；ETF 分支不应调用它 + return None + + +def _engine() -> StrategyEngine: + return StrategyEngine(strategy_dirs=[BUILTIN_DIR]) + + +def test_all_builtin_strategies_declare_asset_types_and_timeframes(): + engine = _engine() + assert engine.load_errors() == [] + for meta in engine.list_strategies(): + assert meta["asset_types"] + assert meta["timeframes"] == ["1d"] + + +def test_all_builtin_strategies_use_matrix_backend_only(): + engine = _engine() + assert engine.load_errors() == [] + strategies = [engine.get(meta["id"]) for meta in engine.list_strategies()] + assert len(strategies) == 18 + assert all(strategy.execution_backend == "matrix_native" for strategy in strategies) + assert all(strategy.matrix_strategy is not None for strategy in strategies) + assert all(strategy.filter_fn is None for strategy in strategies) + assert all(strategy.filter_history_fn is None for strategy in strategies) + + +def test_all_builtin_matrix_formulas_accept_base_market_matrix(): + rows = [] + start = date(2024, 1, 1) + for offset in range(80): + close = 10.0 + offset * 0.04 + rows.append({ + "symbol": "000001.SZ", + "name": "测试股票", + "date": start + timedelta(days=offset), + "open": close - 0.05, + "high": close + 0.15, + "low": close - 0.15, + "close": close, + "volume": 1000.0 + offset * 5.0, + "amount": 100000.0, + "raw_close": close, + "turnover_rate": 5.0, + "consecutive_limit_ups": 0, + }) + panel = pl.DataFrame(rows) + engine = _engine() + from app.backtest.matrix import build_market_data_matrix + + fields = set() + for strategy in (engine.get(meta["id"]) for meta in engine.list_strategies()): + fields.update(engine._matrix_field_columns(strategy)) + market = build_market_data_matrix(panel, field_columns=fields) + for meta in engine.list_strategies(): + strategy = engine.get(meta["id"]) + signals = strategy.matrix_strategy.compute_signals(market, {}) + assert signals.shape == market.shape, meta["id"] + + +def test_limit_up_strategies_are_stock_only(): + engine = _engine() + for sid in ("broken_board_recovery", "consecutive_limit_ups"): + assert engine.get(sid).meta["asset_types"] == ["stock"] + + +def test_pure_technical_strategies_support_etf(): + engine = _engine() + for sid in ( + "trend_breakout", "ma_golden_cross", "macd_golden", + "volume_price_surge", "low_volatility_leader", "oversold_bounce", + "boll_breakout", "bullish_alignment", "pullback_to_support", + "n_day_low_reversal", + ): + assert "etf" in engine.get(sid).meta["asset_types"], sid + + +def test_custom_strategy_defaults_to_stock_and_daily(tmp_path): + path = tmp_path / "custom_default.py" + path.write_text( + 'import polars as pl\n' + 'META = {"id": "custom_default", "name": "x"}\n' + 'def filter(df, params):\n return pl.lit(True)\n', + encoding="utf-8", + ) + strategy = StrategyEngine._load_file(path) + assert strategy.meta["asset_types"] == ["stock"] + assert strategy.meta["timeframes"] == ["1d"] def test_service_defaults_to_stock_dir(tmp_path): @@ -70,50 +130,54 @@ def test_service_etf_uses_etf_dir(tmp_path): assert svc._enriched_dirname == "kline_etf_enriched" -def test_etf_run_preset_empty_data_degrades(tmp_path): - """ETF enriched 为空时，run_preset 返回空结果而非抛错。""" - svc = ScreenerService(_FakeRepo(tmp_path), asset_type="etf") - result = svc.run_preset("trend_breakout", as_of=date(2026, 1, 2)) - assert result.total == 0 - assert result.rows == [] - - -def test_etf_run_preset_filters_rows(tmp_path): - """给一份含技术列的 ETF enriched，趋势突破策略能选出命中行。""" - enriched = pl.DataFrame({ - "symbol": ["510300", "159915"], - "name": ["沪深300ETF", "创业板ETF"], - "date": [date(2026, 1, 2), date(2026, 1, 2)], - "close": [4.0, 2.0], - "open": [3.9, 2.1], - "ma60": [3.5, 2.5], - "signal_n_day_high": [True, False], - "vol_ratio_5d": [2.5, 0.5], - "momentum_60d": [0.2, -0.1], - }) - repo = _FakeRepo(tmp_path, enriched=enriched, latest=date(2026, 1, 2)) - svc = ScreenerService(repo, asset_type="etf") - result = svc.run_preset("trend_breakout", as_of=date(2026, 1, 2)) +def test_etf_strategy_runs_through_engine_context(tmp_path): + rows = [] + for offset in range(61): + trade_date = date(2025, 11, 3) + timedelta(days=offset) + leader_close = 3.0 + offset / 60.0 + weak_close = 3.0 - offset / 60.0 + rows.extend([ + { + "symbol": "510300", "name": "沪深300ETF", "date": trade_date, + "open": leader_close - 0.01, "high": leader_close + 0.01, + "low": leader_close - 0.02, "close": leader_close, + "volume": 300.0 if offset == 60 else 100.0, + }, + { + "symbol": "159915", "name": "创业板ETF", "date": trade_date, + "open": weak_close + 0.01, "high": weak_close + 0.02, + "low": weak_close - 0.01, "close": weak_close, + "volume": 50.0 if offset == 60 else 100.0, + }, + ]) + history = pl.DataFrame(rows) + target_date = history["date"].max() + current = history.filter(pl.col("date") == target_date) + engine = _engine() + result = engine.run( + "trend_breakout", + StrategyDataContext( + asset_type="etf", + timeframe="1d", + as_of=target_date, + current=current, + history=history, + ), + overrides={"basic_filter": {"enabled": False}}, + ) assert result.total == 1 assert result.rows[0]["symbol"] == "510300" -def test_strategies_filtered_for_etf(): - etf_ids = [sid for sid, s in PRESET_STRATEGIES.items() - if strategy_supports_asset(s, "etf")] - assert "trend_breakout" in etf_ids - assert "consecutive_limit_ups" not in etf_ids - assert len(etf_ids) == 10 - - -def test_run_preset_stock_only_strategy_on_etf_returns_empty(tmp_path): - """对 ETF 跑股票专有策略（连板）应返回空结果，而非误命中或抛错。""" - enriched = pl.DataFrame({ - "symbol": ["510300"], - "date": [date(2026, 1, 2)], - "close": [4.0], - }) - repo = _FakeRepo(tmp_path, enriched=enriched, latest=date(2026, 1, 2)) - svc = ScreenerService(repo, asset_type="etf") - result = svc.run_preset("consecutive_limit_ups", as_of=date(2026, 1, 2)) - assert result.total == 0 +def test_stock_only_strategy_on_etf_fails_explicitly(): + engine = _engine() + with pytest.raises(ValueError, match="does not support asset_type"): + engine.run( + "consecutive_limit_ups", + StrategyDataContext( + asset_type="etf", + timeframe="1d", + as_of=date(2026, 1, 2), + current=pl.DataFrame({"symbol": ["510300"], "close": [4.0]}), + ), + ) diff --git a/backend/tests/test_st_limit_and_sharpe.py b/backend/tests/test_st_limit_and_sharpe.py index 55722a0..78502c9 100644 --- a/backend/tests/test_st_limit_and_sharpe.py +++ b/backend/tests/test_st_limit_and_sharpe.py @@ -10,23 +10,32 @@ from __future__ import annotations from datetime import date import polars as pl +import pytest from app.backtest.factor import FactorBacktestService +from app.backtest.matrix import build_market_data_matrix from app.indicators.pipeline import compute_limit_signals -from app.strategy.builtin.near_limit_up import _limit_pct +from app.strategy.builtin.near_limit_up import MATRIX_STRATEGY def test_near_limit_pct_st_only_on_main_board(): df = pl.DataFrame({ "symbol": ["300001", "688001", "600001", "000001", "830001.BJ"], "name": ["*ST创业", "科创ST", "*ST主板", "平安银行", "北交ST"], + "date": [date(2024, 1, 2)] * 5, + "open": [10.0] * 5, + "high": [10.0] * 5, + "low": [10.0] * 5, + "close": [10.0] * 5, + "volume": [1000.0] * 5, }) - lp = df.with_columns(_limit_pct().alias("lp"))["lp"].to_list() - assert lp[0] == 0.20 # 创业板 ST → 20% (不再是 5%) - assert lp[1] == 0.20 # 科创板 ST → 20% - assert lp[2] == 0.05 # 主板 ST → 5% - assert lp[3] == 0.10 # 主板普通 → 10% - assert lp[4] == 0.30 # 北交所 → 30% + market = build_market_data_matrix(df) + limit_by_symbol = dict(zip(market.symbols, MATRIX_STRATEGY._limit_pct(market), strict=True)) + assert limit_by_symbol["300001"] == pytest.approx(0.20) # 创业板 ST → 20% + assert limit_by_symbol["688001"] == pytest.approx(0.20) # 科创板 ST → 20% + assert limit_by_symbol["600001"] == pytest.approx(0.05) # 主板 ST → 5% + assert limit_by_symbol["000001"] == pytest.approx(0.10) # 主板普通 → 10% + assert limit_by_symbol["830001.BJ"] == pytest.approx(0.30) # 北交所 → 30% def _two_day(symbol: str, prev_close: float, today_close: float) -> pl.DataFrame: diff --git a/backend/tests/test_strategy_code_save.py b/backend/tests/test_strategy_code_save.py index dd2e419..dc1c2aa 100644 --- a/backend/tests/test_strategy_code_save.py +++ b/backend/tests/test_strategy_code_save.py @@ -2,7 +2,6 @@ from __future__ import annotations from types import SimpleNamespace -import polars as pl import pytest from app.api.strategy import ( @@ -47,10 +46,7 @@ def filter(df: pl.DataFrame, params: dict) -> pl.Expr: def _request(tmp_path): ai_dir = tmp_path / "strategies" / "ai" custom_dir = tmp_path / "strategies" / "custom" - engine = StrategyEngine( - enriched_loader=lambda _date: pl.DataFrame(), - strategy_dirs=[custom_dir, ai_dir], - ) + engine = StrategyEngine(strategy_dirs=[custom_dir, ai_dir]) repo = SimpleNamespace(store=SimpleNamespace(data_dir=tmp_path)) return SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(repo=repo, strategy_engine=engine))) diff --git a/backend/tests/test_strategy_realtime_refresh.py b/backend/tests/test_strategy_realtime_refresh.py index 6da0a78..abdda3f 100644 --- a/backend/tests/test_strategy_realtime_refresh.py +++ b/backend/tests/test_strategy_realtime_refresh.py @@ -1,13 +1,17 @@ """策略页实时结果刷新 SSE 回归测试。""" from __future__ import annotations +from datetime import timedelta +from pathlib import Path from types import SimpleNamespace from unittest.mock import patch import polars as pl +from app.market_time import cn_today from app.services import quote_service from app.services.quote_service import QuoteService, QuoteSubscriber +from app.strategy.engine import StrategyEngine from app.strategy.monitor import MonitorRuleEngine @@ -60,16 +64,16 @@ def test_strategy_result_notification_fans_out_to_all_subscribers(): class _EmptyResultStrategyEngine: def get(self, strategy_id: str): assert strategy_id == "strategy_1" - return SimpleNamespace(filter_history_fn=None) + return SimpleNamespace(filter_history_fn=None, execution_backend="polars_expr") - def run(self, strategy_id: str, **kwargs): + def run(self, strategy_id: str, context, **kwargs): assert strategy_id == "strategy_1" - assert kwargs["precomputed"].height == 1 + assert context.current.height == 1 return SimpleNamespace(total=0, rows=[]) class _FailingStrategyEngine(_EmptyResultStrategyEngine): - def run(self, strategy_id: str, **kwargs): + def run(self, strategy_id: str, context, **kwargs): raise RuntimeError("strategy failed") @@ -102,6 +106,72 @@ def test_failed_or_skipped_strategy_does_not_mark_result_refresh(): assert skipped.consume_strategy_result_updates() is False +def test_matrix_strategy_monitor_reuses_live_matrix_and_updates_last_row(): + target = cn_today() + start = target - timedelta(days=61) + rows = [] + for offset in range(62): + for symbol, base in (("000001.SZ", 10.0), ("600000.SH", 20.0)): + close = base + offset * 0.05 + rows.append({ + "symbol": symbol, + "name": symbol, + "date": start + timedelta(days=offset), + "open": close, + "high": close * 1.01, + "low": close * 0.99, + "close": close, + "volume": 1_000_000.0, + "amount": 100_000_000.0, + "total_shares": 1_000_000_000.0, + "float_shares": 800_000_000.0, + }) + panel = pl.DataFrame(rows) + history = panel.filter(pl.col("date") < target) + current = panel.filter(pl.col("date") == target) + load_calls = [] + + def load_history(as_of, lookback): + load_calls.append((as_of, lookback)) + return history + + strategy_engine = StrategyEngine( + strategy_dirs=[Path(__file__).resolve().parents[1] / "app" / "strategy" / "builtin"], + ) + overrides = { + "params": {"require_macd_golden": False, "use_volume_filter": False}, + "basic_filter": {"enabled": False}, + } + monitor = MonitorRuleEngine() + monitor.set_strategy_engine(strategy_engine) + monitor.set_data_dir(Path("test-data")) + monitor.set_history_loader(load_history) + monitor.set_rules([{ + "id": "matrix_macd", + "name": "MACD", + "type": "strategy", + "asset_type": "stock", + "strategy_id": "macd_golden", + "scope": "all", + "symbols": [], + "cooldown_seconds": 0, + }]) + + with patch("app.strategy.monitor._strategy_config.load_override", return_value=overrides): + assert monitor.evaluate(current) == [] + assert monitor.latest_strategy_results()["macd_golden"]["total"] == 2 + first_stats = strategy_engine.realtime_matrix_stats("monitor:stock") + assert first_stats["build_count"] == 1 + assert len(load_calls) == 1 + + updated = current.with_columns((pl.col("close") + 1.0).alias("close")) + assert monitor.evaluate(updated) == [] + second_stats = strategy_engine.realtime_matrix_stats("monitor:stock") + assert second_stats["build_count"] == 1 + assert second_stats["update_count"] == 1 + assert len(load_calls) == 1 + + class _MonitorWithUpdate: rule_count = 1 diff --git a/backend/tests/test_strategy_registry.py b/backend/tests/test_strategy_registry.py new file mode 100644 index 0000000..3799aa8 --- /dev/null +++ b/backend/tests/test_strategy_registry.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +from datetime import date + +import polars as pl +import pytest + +from app.strategy.engine import StrategyDataContext, StrategyEngine + + +def _strategy_code(strategy_id: str, *, body: str = "return pl.lit(True)") -> str: + return f'''import polars as pl +META = {{ + "id": "{strategy_id}", + "name": "{strategy_id}", + "asset_types": ["stock"], + "timeframes": ["1d"], +}} +EXECUTION_BACKEND = "polars_expr" +def filter(df, params): + {body} +''' + + +def test_duplicate_strategy_id_reports_both_paths(tmp_path): + first = tmp_path / "first" + second = tmp_path / "second" + first.mkdir() + second.mkdir() + (first / "a.py").write_text(_strategy_code("duplicate"), encoding="utf-8") + (second / "b.py").write_text(_strategy_code("duplicate"), encoding="utf-8") + + engine = StrategyEngine(strategy_dirs=[first, second]) + + assert not engine.has("duplicate") + errors = engine.load_errors() + assert len(errors) == 2 + assert {item["file"] for item in errors} == {str(first / "a.py"), str(second / "b.py")} + assert all("duplicate strategy id" in item["error"] for item in errors) + + +def test_failed_reload_keeps_previous_registry(tmp_path): + path = tmp_path / "stable.py" + path.write_text(_strategy_code("stable"), encoding="utf-8") + engine = StrategyEngine(strategy_dirs=[tmp_path]) + previous = engine.get("stable") + + path.write_text("this is not valid python", encoding="utf-8") + + with pytest.raises(ValueError, match="strategy reload failed"): + engine.reload() + + assert engine.get("stable") is previous + assert engine.load_errors() + + +def test_context_rejects_unsupported_timeframe(tmp_path): + path = tmp_path / "daily.py" + path.write_text(_strategy_code("daily"), encoding="utf-8") + engine = StrategyEngine(strategy_dirs=[tmp_path]) + + with pytest.raises(ValueError, match="does not support timeframe"): + engine.run( + "daily", + StrategyDataContext( + asset_type="stock", + timeframe="5m", + as_of=date(2026, 1, 2), + current=pl.DataFrame({"symbol": ["000001.SZ"]}), + ), + ) + + +def test_run_all_respects_explicit_empty_strategy_ids(tmp_path): + path = tmp_path / "daily.py" + path.write_text(_strategy_code("daily"), encoding="utf-8") + engine = StrategyEngine(strategy_dirs=[tmp_path]) + context = StrategyDataContext( + asset_type="stock", + timeframe="1d", + as_of=date(2026, 1, 2), + current=pl.DataFrame({"symbol": ["000001.SZ"]}), + ) + + assert engine.run_all(context, strategy_ids=[]) == {} + + +def test_builtin_custom_and_ai_files_share_one_registry_and_run_path(tmp_path): + strategy_ids = { + "builtin": "builtin_plugin", + "custom": "custom_plugin", + "ai": "ai_plugin", + } + dirs = [] + for source, strategy_id in strategy_ids.items(): + directory = tmp_path / "strategies" / source + directory.mkdir(parents=True) + (directory / f"{strategy_id}.py").write_text( + _strategy_code(strategy_id), + encoding="utf-8", + ) + dirs.append(directory) + + engine = StrategyEngine(strategy_dirs=dirs) + sources = {meta["id"]: meta["source"] for meta in engine.list_strategies()} + assert sources == {strategy_id: source for source, strategy_id in strategy_ids.items()} + + context = StrategyDataContext( + asset_type="stock", + timeframe="1d", + as_of=date(2026, 1, 2), + current=pl.DataFrame({"symbol": ["000001.SZ"]}), + ) + overrides = { + strategy_id: {"basic_filter": {"enabled": False}} + for strategy_id in strategy_ids.values() + } + results = engine.run_all(context, overrides_map=overrides) + assert set(results) == set(strategy_ids.values()) + assert all(result.total == 1 for result in results.values()) diff --git a/backend/tests/test_strategy_saved_params.py b/backend/tests/test_strategy_saved_params.py index 2e1468c..e2ccf89 100644 --- a/backend/tests/test_strategy_saved_params.py +++ b/backend/tests/test_strategy_saved_params.py @@ -2,12 +2,12 @@ from datetime import date import polars as pl -from app.strategy.engine import StrategyDef, StrategyEngine +from app.strategy.engine import StrategyDataContext, StrategyDef, StrategyEngine -def _make_engine() -> StrategyEngine: +def _make_engine() -> tuple[StrategyEngine, StrategyDataContext]: df = pl.DataFrame({"symbol": ["A", "B", "C"], "value": [1, 2, 3]}) - engine = StrategyEngine(enriched_loader=lambda _as_of: df) + engine = StrategyEngine(strategy_dirs=[]) engine._strategies["saved_params"] = StrategyDef( meta={"id": "saved_params", "scoring": {}, "limit": 100}, basic_filter={"enabled": False}, @@ -24,13 +24,19 @@ def _make_engine() -> StrategyEngine: lookback_days=1, source="custom", ) - return engine + return engine, StrategyDataContext( + asset_type="stock", + timeframe="1d", + as_of=date(2026, 7, 15), + current=df, + ) def test_run_applies_saved_strategy_params(): - result = _make_engine().run( + engine, context = _make_engine() + result = engine.run( "saved_params", - date(2026, 7, 15), + context, overrides={"params": {"min_value": 2}}, ) @@ -38,9 +44,10 @@ def test_run_applies_saved_strategy_params(): def test_explicit_params_override_saved_strategy_params(): - result = _make_engine().run( + engine, context = _make_engine() + result = engine.run( "saved_params", - date(2026, 7, 15), + context, params={"min_value": 3}, overrides={"params": {"min_value": 2}}, ) diff --git a/backend/uv.lock b/backend/uv.lock index 3b8411a..63d96a7 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -2512,12 +2512,14 @@ dependencies = [ { name = "fastapi" }, { name = "fastexcel" }, { name = "httpx" }, + { name = "numba" }, { name = "openai" }, { name = "pandas" }, { name = "pillow" }, { name = "platformdirs" }, { name = "plyer" }, { name = "polars" }, + { name = "psutil" }, { name = "pyarrow" }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -2556,6 +2558,7 @@ requires-dist = [ { name = "fastexcel", specifier = ">=0.10" }, { name = "httpx", specifier = ">=0.27" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" }, + { name = "numba", specifier = ">=0.65.1" }, { name = "openai", specifier = ">=1.40" }, { name = "pandas", specifier = ">=2.2" }, { name = "pillow", specifier = ">=10.0" }, @@ -2563,6 +2566,7 @@ requires-dist = [ { name = "plyer", specifier = ">=2.1" }, { name = "polars", specifier = ">=1.0" }, { name = "polars", extras = ["rtcompat"], marker = "extra == 'legacy-cpu'", specifier = ">=1.0" }, + { name = "psutil", specifier = ">=5.9" }, { name = "pyarrow", specifier = ">=16.0" }, { name = "pydantic", specifier = ">=2.7" }, { name = "pydantic-settings", specifier = ">=2.4" }, diff --git a/frontend/src/components/screener/StrategyBuilderDialog.tsx b/frontend/src/components/screener/StrategyBuilderDialog.tsx index 39566c9..f607d4d 100644 --- a/frontend/src/components/screener/StrategyBuilderDialog.tsx +++ b/frontend/src/components/screener/StrategyBuilderDialog.tsx @@ -91,6 +91,8 @@ META = { "name": "我的策略", "description": "策略描述", "tags": ["自定义"], + "asset_types": ["stock"], + "timeframes": ["1d"], "basic_filter": { "price_min": 3, "price_max": 200, "market_cap_min": 10e8, "amount_min": 0.5e8, @@ -105,6 +107,7 @@ META = { "limit": 100, } +EXECUTION_BACKEND = "polars_expr" ENTRY_SIGNALS = ["signal_n_day_high"] EXIT_SIGNALS = ["signal_ma20_breakdown"] STOP_LOSS = -0.05 @@ -124,6 +127,45 @@ def filter(df: pl.DataFrame, params: dict) -> pl.Expr: ) ` +const MATRIX_TEMPLATE = `"""矩阵原生策略示例""" +import numpy as np +from app.backtest.matrix import MarketDataMatrix, SignalMatrix, make_signal_matrix, matrix_feature + +META = { + "id": "custom_matrix_strategy", + "name": "矩阵策略", + "description": "收盘价站上 MA20", + "tags": ["自定义", "矩阵"], + "asset_types": ["stock"], + "timeframes": ["1d"], + "params": [], + "scoring": {}, + "order_by": "score", + "descending": True, + "limit": 100, +} + +EXECUTION_BACKEND = "matrix_native" +ENTRY_SIGNALS = [] +EXIT_SIGNALS = [] +STOP_LOSS = -0.05 +MAX_HOLD_DAYS = 20 +ALERTS = [] + +class CustomMatrixStrategy: + def required_fields(self) -> frozenset[str]: + return frozenset({"close", "ma20"}) + + def required_warmup_bars(self, params: dict) -> int: + return 60 + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + entry = market.close > matrix_feature(market, "ma20") + return make_signal_matrix(market.shape, entry=entry.astype(np.uint8)) + +MATRIX_STRATEGY = CustomMatrixStrategy() +` + interface Props { open: boolean; onClose: () => void; onSavedId?: (id: string) => void | Promise; mode?: 'create' | 'modify' } export function StrategyBuilderDialog({ open, onClose, onSavedId, mode = 'create' }: Props) { @@ -135,6 +177,7 @@ export function StrategyBuilderDialog({ open, onClose, onSavedId, mode = 'create const [name, setName] = useState('') const [description, setDescription] = useState('') const [direction, setDirection] = useState('long') + const [executionBackend, setExecutionBackend] = useState<'polars_expr' | 'matrix_native'>('polars_expr') const [rules, setRules] = useState('') const [code, setCode] = useState('') const [instruction, setInstruction] = useState('') @@ -157,6 +200,10 @@ export function StrategyBuilderDialog({ open, onClose, onSavedId, mode = 'create if (d) { setStep(d.step ?? 1); setName(d.name ?? ''); setDescription(d.description ?? '') setDirection(d.direction ?? 'long') + setExecutionBackend( + (d as any).executionBackend + ?? (String(d.code ?? '').includes('matrix_native') ? 'matrix_native' : 'polars_expr'), + ) setRules(d.rules ?? ''); setCode(d.code ?? ''); setStrategyId(d.strategyId ?? '') setSource((d as any).source ?? (d.strategyId?.startsWith('custom_') ? 'custom' : 'ai')) if (mode === 'modify') setTab('custom') @@ -175,14 +222,14 @@ export function StrategyBuilderDialog({ open, onClose, onSavedId, mode = 'create if (!name && !rules && !code) { draftStore.set(null) } else { - draftStore.set({ name, description, direction, rules, code, step, strategyId, source } as any) + draftStore.set({ name, description, direction, executionBackend, rules, code, step, strategyId, source } as any) } - }, [name, description, direction, rules, code, step, strategyId, source]) + }, [name, description, direction, executionBackend, rules, code, step, strategyId, source]) useEffect(() => { if (loaded) persist() }, [loaded, persist]) const clearDraft = () => { draftStore.set(null) - setName(''); setDescription(''); setDirection('long') + setName(''); setDescription(''); setDirection('long'); setExecutionBackend('polars_expr') setRules(''); setCode(''); setStep(1); setError(''); setInstruction('') setStrategyId(''); setSource('ai'); setValidated(false) } @@ -195,6 +242,13 @@ export function StrategyBuilderDialog({ open, onClose, onSavedId, mode = 'create return slugId(target) } + const selectExecutionBackend = (backend: 'polars_expr' | 'matrix_native') => { + setExecutionBackend(backend) + if (tab === 'custom' && (!code || code === CUSTOM_TEMPLATE || code === MATRIX_TEMPLATE)) { + setCode(backend === 'matrix_native' ? MATRIX_TEMPLATE : CUSTOM_TEMPLATE) + } + } + // Step 1: 生成 const handleGenerate = async () => { if (!name.trim() || !rules.trim()) return @@ -204,7 +258,7 @@ export function StrategyBuilderDialog({ open, onClose, onSavedId, mode = 'create const id = resolveStrategyId('ai') setStrategyId(id); setSource('ai'); setPreviewTab('code') let finalResult: any = null - for await (const evt of api.strategyBuildStream(1, { name: name.trim(), description: description.trim(), direction, rules: rules.trim(), strategy_id: id })) { + for await (const evt of api.strategyBuildStream(1, { name: name.trim(), description: description.trim(), direction, execution_backend: executionBackend, rules: rules.trim(), strategy_id: id })) { if (evt.type === 'delta') { setCode(prev => prev + evt.content) } else if (evt.type === 'error') { @@ -326,7 +380,7 @@ export function StrategyBuilderDialog({ open, onClose, onSavedId, mode = 'create - @@ -402,6 +456,13 @@ export function StrategyBuilderDialog({ open, onClose, onSavedId, mode = 'create ))} +
+ 执行后端 +
+ + +
+
策略规则