diff --git a/README.md b/README.md index c7e0b5e..718f332 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ | :--------------- | :--------------------------------------------------------------------- | :-------------------------------- | | 🔍 **选股引擎** | 18 个内置策略 + 自定义信号 + AI 生成 + 代码迁移,Polars 毫秒级扫全 A 股 | [strategy.md](./docs/strategy.md) | | 📊 **指标流水线** | MA/EMA/MACD/RSI/KDJ/布林/量比等,一次扫表落盘 enriched Parquet | [features.md](./docs/features.md) | -| 🧪 **回测引擎** | 三种模式(个股/策略组合/自由信号),T+1/手续费/滑点/止损,SSE 流式进度 | [features.md](./docs/features.md) | +| 🧪 **回测研究** | 因子/策略回测 + 嵌套样本外挖掘,T+1/费用/滑点约束,SSE 持久任务 | [mining.md](./docs/mining.md) | | 📡 **监控中心** | 四类监控(策略/个股信号/价格/异动),多条件 AND/OR + 语音播报 + 飞书推送 | [features.md](./docs/features.md) | | 📈 **个股分析** | 9 类关键价位 + AI 四维分析(技术/基本面/财务/消息面) | [features.md](./docs/features.md) | | 🏆 **连板梯队** | 连板层级统计 + 概念涨幅轮动 + 盘后 AI 复盘 + 炸板/翘板预警 | [features.md](./docs/features.md) | @@ -238,6 +238,7 @@ PORT=3018 # 服务端口 | [docs/features.md](./docs/features.md) | 各功能模块详细说明(选股/指标/回测/监控/个股分析/数据扩展) | | [docs/custom-data-source.md](./docs/custom-data-source.md) | 自定义数据源接入、YAML 配置与 mock 联调示例 | | [docs/strategy.md](./docs/strategy.md) | 策略体系(18 内置策略 + 三种扩展方式 + 文件结构) | +| [docs/mining.md](./docs/mining.md) | 因子与策略挖掘口径、防泄漏、任务隔离和发布边界 | | [docs/secondary-development.md](./docs/secondary-development.md) | 代码二次开发、前端插槽、后端策略接口与 AI 开发模板 | | [backend/app/strategy/prompts/strategy-guide.md](./backend/app/strategy/prompts/strategy-guide.md) | 策略开发完整规范(AI 生成与手写) | diff --git a/backend/app/api/backtest.py b/backend/app/api/backtest.py index d834f32..2605e73 100644 --- a/backend/app/api/backtest.py +++ b/backend/app/api/backtest.py @@ -381,7 +381,10 @@ def strategy_run(req: StrategyBacktestRequest, request: Request): regime_filter=req.regime_filter, ) task = make_worker_task("backtest", settings.data_dir, cfg) - return run_worker_task(task) + from app.services.heavy_job_limiter import shared_heavy_job_limiter + + with shared_heavy_job_limiter.slot("normal"): + return run_worker_task(task) # ── SSE 流式回测 (实时进度 + 可取消 + 支持重连) ─────────────────── @@ -409,10 +412,6 @@ _running_jobs: dict[str, _BacktestJob] = {} _jobs_lock = threading.Lock() _JOB_TTL = 300 # 完成后保留 5 分钟 -# 并发回测上限: 多个重回测同时跑会 OOM (服务器内存约 1.8GB)。用信号量限并发, -# 超出的任务在 _run_backtest 里排队, SSE 连接照常保持, run 一开始就有进度。 -_backtest_semaphore = threading.Semaphore(2) - def _cleanup_stale_jobs(): """清理过期任务 (完成超过 TTL 的)。全程持 _jobs_lock: 迭代+pop 与其他访问互斥。""" @@ -588,21 +587,27 @@ async def strategy_stream( ) def _run_backtest(): - # 信号量限并发: 超额任务在此阻塞排队, 不并发吃满内存 (等待期间 cancel_event - # 仍可置位, svc.run 会据此提前返回 cancelled)。持槽跑完在 finally 释放。 - _backtest_semaphore.acquire() + from app.services.heavy_job_limiter import ( + HeavyJobCancelledError, + shared_heavy_job_limiter, + ) + try: - task = make_worker_task("backtest", settings.data_dir, cfg) - result = run_worker_task( - task, - lambda d: job.progress.append(d), - job.cancel_event, - ) + with shared_heavy_job_limiter.slot( + "normal", + cancel_event=job.cancel_event, + ): + task = make_worker_task("backtest", settings.data_dir, cfg) + result = run_worker_task( + task, + lambda d: job.progress.append(d), + job.cancel_event, + ) _finish_job(job, result=result) + except HeavyJobCancelledError: + _finish_job(job, error="回测已取消") except Exception as e: _finish_job(job, error=str(e)) - finally: - _backtest_semaphore.release() # 启动后台线程 (不阻塞事件循环) threading.Thread(target=_run_backtest, daemon=True).start() @@ -886,14 +891,25 @@ async def optimize_stream( ) def _run_opt(): + from app.services.heavy_job_limiter import ( + HeavyJobCancelledError, + shared_heavy_job_limiter, + ) + try: - task = make_worker_task("optimize", settings.data_dir, ocfg) - result = run_worker_task( - task, - lambda d: job.progress.append(d), - job.cancel_event, - ) + with shared_heavy_job_limiter.slot( + "normal", + cancel_event=job.cancel_event, + ): + task = make_worker_task("optimize", settings.data_dir, ocfg) + result = run_worker_task( + task, + lambda d: job.progress.append(d), + job.cancel_event, + ) _finish_job(job, result=result) + except HeavyJobCancelledError: + _finish_job(job, error="优化已取消") except Exception as e: _finish_job(job, error=str(e)) @@ -1101,14 +1117,25 @@ async def walkforward_stream( ) def _run_wf(): + from app.services.heavy_job_limiter import ( + HeavyJobCancelledError, + shared_heavy_job_limiter, + ) + try: - task = make_worker_task("walkforward", settings.data_dir, wf_cfg) - result = run_worker_task( - task, - lambda d: job.progress.append(d), - job.cancel_event, - ) + with shared_heavy_job_limiter.slot( + "normal", + cancel_event=job.cancel_event, + ): + task = make_worker_task("walkforward", settings.data_dir, wf_cfg) + result = run_worker_task( + task, + lambda d: job.progress.append(d), + job.cancel_event, + ) _finish_job(job, result=result) + except HeavyJobCancelledError: + _finish_job(job, error="walk-forward 已取消") except Exception as e: _finish_job(job, error=str(e)) diff --git a/backend/app/api/data.py b/backend/app/api/data.py index ce3ec92..3526d7f 100644 --- a/backend/app/api/data.py +++ b/backend/app/api/data.py @@ -11,6 +11,7 @@ from typing import Any, Callable from fastapi import APIRouter, Request +from app.enriched_generation import EnrichedPublication from app.indicators.pipeline import ENRICHED_COLUMNS logger = logging.getLogger(__name__) @@ -504,26 +505,21 @@ def _compute_storage(data_dir: Path) -> dict: stats[f"{key}_files"] = fc stats[f"{key}_size_mb"] = sz - # total: 再加上其他零散文件(pools, financials, capabilities.json 等) - other_dirs = ["pools", "financials", "backtest_results", "screener_results", "ai_cache"] + # total: 再加上其他零散目录 (financials 有下方专属明细统计, 不在此列) + other_dirs = ["pools", "backtest_results", "screener_results", "ai_cache"] for name in other_dirs: d = data_dir / name if d.exists(): _, s = _scan_dir_stats(d) total_size += s - # financials 单独统计 + # financials 单独统计 (明细与 total 各计入一次, 不得与其他目录循环重复累加) fin_dir = data_dir / "financials" if fin_dir.exists(): fc, sz = _scan_dir_stats(fin_dir) stats["financials_files"] = fc stats["financials_size_mb"] = sz total_size += sz - for name in other_dirs: - d = data_dir / name - if d.exists(): - _, s = _scan_dir_stats(d) - total_size += s # 根目录散文件 for entry in os.scandir(data_dir): if entry.is_file(follow_symlinks=False): @@ -626,6 +622,10 @@ def clear_data(request: Request): repo = request.app.state.repo data_dir = repo.store.data_dir deleted = 0 + publications = { + "kline_daily_enriched": EnrichedPublication(data_dir, "stock"), + "kline_etf_enriched": EnrichedPublication(data_dir, "etf"), + } for sub in ( "kline_daily", "kline_daily_enriched", "kline_index_daily", "kline_index_enriched", @@ -634,15 +634,27 @@ def clear_data(request: Request): "backtest_results", "screener_results", "ai_cache", ): d = data_dir / sub - if d.exists(): - # 先删所有 parquet 文件 - for f in d.rglob("*.parquet"): + if not d.exists(): + continue + publication = publications.get(sub) + parquet_files = list(d.rglob("*.parquet")) + if publication is not None and parquet_files: + publication.begin() + try: + for f in parquet_files: f.unlink() deleted += 1 - # 再删除空的日期分区子目录（date=YYYY-MM-DD 等） + if publication is not None: + publication.mark_changed() for child in list(d.iterdir()): if child.is_dir(): shutil.rmtree(child, ignore_errors=True) + if publication is not None: + publication.commit() + except BaseException: + if publication is not None: + publication.abandon() + raise # 清除同步历史（内存 + 磁盘 job_store/ 文件夹） from app.services.pipeline_jobs import job_store diff --git a/backend/app/api/kline.py b/backend/app/api/kline.py index 15f7fea..e5fd5c0 100644 --- a/backend/app/api/kline.py +++ b/backend/app/api/kline.py @@ -12,7 +12,7 @@ from fastapi import APIRouter, HTTPException, Query, Request from app.indicators.pipeline import compute_enriched, compute_enriched_single from app.market_time import cn_now, cn_today from app.price_limits import is_risk_warning_name, price_limit_pct -from app.db_safe import is_valid_ext_ident, quote_ident +from app.db_safe import is_valid_ext_ident from app.services import kline_sync logger = logging.getLogger(__name__) @@ -175,21 +175,28 @@ def instruments_names(request: Request, symbols: list[str]): def _get_stock_info(repo, symbol: str) -> dict: - """从 instruments 视图查标的名称 + 股本。""" + """从 instruments 内存缓存查标的名称 + 股本。 + + 该接口在个股弹窗打开时每秒被调用 (SSE invalidate 触发重拉), 走 + repo.get_instruments() 的 Polars 内存缓存按 symbol 过滤, 不再每请求 + DuckDB 扫 instruments parquet。列缺失时返回空 dict, 与旧 SQL 报错路径一致。 + """ + import polars as pl try: - row = repo.execute_one( - "SELECT name, total_shares, float_shares FROM instruments WHERE symbol = ? LIMIT 1", - [symbol], - ) + df = repo.get_instruments() + needed = ("symbol", "name", "total_shares", "float_shares") + if df.is_empty() or not all(c in df.columns for c in needed): + return {} + hit = df.filter(pl.col("symbol") == symbol).head(1) + if hit.is_empty(): + return {} + return { + "name": hit["name"][0], + "total_shares": hit["total_shares"][0], + "float_shares": hit["float_shares"][0], + } except Exception: # noqa: BLE001 return {} - if not row: - return {} - return { - "name": row[0], - "total_shares": row[1], - "float_shares": row[2], - } def _get_asset_info(repo, symbol: str, asset_type: str) -> dict: @@ -377,7 +384,8 @@ def _attach_ext(resp: dict, repo, symbol: str, ext_columns: Optional[str]) -> di """按 ext_columns 规格为单只股票 LEFT JOIN 扩展数据，平铺到 stock_info['ext']。 key 形如 "{config_id}__{field_name}"，与自选列表 enriched 接口保持一致。 - JOIN 逻辑参考 watchlist.watchlist_enriched；任何 ext 表/字段缺失都静默跳过。 + 委托 screener._load_ext_value_maps 取值: 复用其 (路径,mtime) 签名缓存, + 个股弹窗每秒重拉时不再重复读 ext parquet; 任何 ext 表/字段缺失都静默跳过。 """ if not ext_columns or not ext_columns.strip(): return resp @@ -394,43 +402,17 @@ def _attach_ext(resp: dict, repo, symbol: str, ext_columns: Optional[str]) -> di if not specs: return resp - import polars as pl - data_dir = repo.store.data_dir try: - from app.services.ext_data import ExtConfigStore - from app.api.ext_data import _read_ext_dataframe - ext_store = ExtConfigStore(data_dir) - configs = {c.id: c for c in ext_store.load_all()} + from app.api.screener import _load_ext_value_maps + value_maps = _load_ext_value_maps(repo, ext_columns) except Exception: # noqa: BLE001 - configs = {} + value_maps = {} ext_values: dict = {} for config_id, field_name in specs: ext_col_name = f"{config_id}__{field_name}" - value = None - try: - cfg = configs.get(config_id) - if cfg: - ext_df, _ = _read_ext_dataframe(cfg, data_dir) - else: - ext_df = pl.from_arrow( - repo.store.db.query( - f"SELECT symbol, {quote_ident(field_name)} FROM ext_{config_id}" - ).arrow() - ) - if not ext_df.is_empty() and "symbol" in ext_df.columns and field_name in ext_df.columns: - # 时序表取最新分区，避免一个 symbol 多行 - row = ( - ext_df - .select(["symbol", field_name]) - .unique(subset=["symbol"], keep="last") - .filter(pl.col("symbol") == symbol) - ) - if not row.is_empty(): - value = row[field_name][0] - except Exception as e: # noqa: BLE001 - logger.debug("kline ext join failed for %s.%s: %s", config_id, field_name, e) - ext_values[ext_col_name] = value + vmap = value_maps.get(ext_col_name) or {} + ext_values[ext_col_name] = vmap.get(symbol) stock_info = dict(resp.get("stock_info") or {}) stock_info["ext"] = ext_values @@ -575,12 +557,13 @@ def get_daily_batch(request: Request, body: dict): return {"data": {}} df = pl.concat(frames, how="diagonal_relaxed") - # 按 symbol 分组, 每只取最近 N 条 + # 按 symbol 分组, 每只取最近 N 条。 + # partition_by 一次切分, 避免 N 只自选时对同一批数据做 N 次全帧过滤。 result: dict[str, list[dict]] = {} - for sym in symbols: - sub = df.filter(pl.col("symbol") == sym).sort("date").tail(days) + for part in df.partition_by("symbol", maintain_order=True): + sub = part.sort("date").tail(days) if not sub.is_empty(): - result[sym] = sub.to_dicts() + result[sub["symbol"][0]] = sub.to_dicts() return {"data": result} @@ -662,14 +645,15 @@ def get_minute_batch(request: Request, body: dict): else: expected = 240 - # 按 symbol 分组, 判定哪些不完整需要补拉 + # 按 symbol 分组, 判定哪些不完整需要补拉 (partition_by 一次切分, 同 daily-batch) result: dict[str, list[dict]] = {} incomplete: list[str] = [] + local_parts: dict[str, pl.DataFrame] = {} + if not df_local.is_empty(): + for part in df_local.partition_by("symbol", maintain_order=True): + local_parts[part["symbol"][0]] = part.sort("datetime") for sym in symbols: - if df_local.is_empty(): - sub = pl.DataFrame() - else: - sub = df_local.filter(pl.col("symbol") == sym).sort("datetime") + sub = local_parts.get(sym, pl.DataFrame()) if expected > 0 and (sub.is_empty() or len(sub) < expected * 0.9): incomplete.append(sym) elif not sub.is_empty(): @@ -712,9 +696,13 @@ def get_minute_batch(request: Request, body: dict): live_parts.append(df_e) if live_parts: live_df = pl.concat(live_parts, how="diagonal_relaxed") + live_map: dict[str, pl.DataFrame] = { + part["symbol"][0]: part.sort("datetime") + for part in live_df.partition_by("symbol", maintain_order=True) + } for sym in incomplete: - sub = live_df.filter(pl.col("symbol") == sym).sort("datetime") - if not sub.is_empty(): + sub = live_map.get(sym) + if sub is not None and not sub.is_empty(): result[sym] = sub.to_dicts() return {"data": result} diff --git a/backend/app/api/mining.py b/backend/app/api/mining.py new file mode 100644 index 0000000..b028f0e --- /dev/null +++ b/backend/app/api/mining.py @@ -0,0 +1,736 @@ +"""Persistent factor and strategy mining HTTP API.""" +from __future__ import annotations + +import asyncio +import json +import math +from collections.abc import AsyncIterator, Mapping, Sequence +from datetime import date +from typing import Annotated, Any, Literal + +import polars as pl +from fastapi import APIRouter, Header, HTTPException, Query, Request +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from sse_starlette.sse import EventSourceResponse + +from app.backtest.factor import FACTOR_COLUMNS +from app.backtest.mining import ( + MAX_BEAM_WIDTH, + MAX_COMBINATION_SIZE, + MAX_FINALISTS, + evaluate_candidate_gate, +) +from app.services import preferences +from app.services.mining_jobs import ( + RUN_STATUSES, + SUCCESS_RUN_STATUSES, + TERMINAL_RUN_STATUSES, + MiningRunStore, + MiningRunStoreError, + MiningRunValidationError, +) +from app.services.mining_preflight import ( + mining_availability, + require_mining_availability, +) +from app.services.mining_schedule import ( + MINING_ALGORITHM_VERSION, + build_data_fingerprint, +) + +router = APIRouter(prefix="/api/backtest/mining", tags=["backtest"]) +_FACTOR_IDS = frozenset(str(item["id"]) for item in FACTOR_COLUMNS) +_MAX_ARTIFACT_BYTES = 64 * 1024 * 1024 +_SSE_POLL_SECONDS = 0.5 +_SSE_HEARTBEAT_SECONDS = 15.0 + + +class MiningStartRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + factor_names: list[str] = Field(min_length=1, max_length=48) + strategy_ids: list[str] = Field(default_factory=list, max_length=8) + symbols: list[str] | None = None + asset_type: Literal["stock", "etf"] = "stock" + start: date | None = None + end: date | None = None + budget_profile: Literal["exploratory", "balanced", "strict"] = "balanced" + commission_pct: float = Field(0.0002, ge=0.0, le=0.05, allow_inf_nan=False) + stamp_tax_pct: float = Field(0.0005, ge=0.0, le=0.05, allow_inf_nan=False) + slippage_bps: float = Field(5.0, ge=0.0, le=1000.0, allow_inf_nan=False) + correlation_threshold: float = Field(0.75, gt=0.0, le=1.0, allow_inf_nan=False) + max_combination_factors: int = Field(4, ge=1, le=MAX_COMBINATION_SIZE) + beam_width: int = Field(12, ge=1, le=MAX_BEAM_WIDTH) + max_finalists: int = Field(MAX_FINALISTS, ge=1, le=MAX_FINALISTS) + force: bool = False + + @field_validator("start", "end", mode="before") + @classmethod + def _iso_dates(cls, value: Any) -> Any: + if isinstance(value, str): + try: + return date.fromisoformat(value) + except ValueError as exc: + raise ValueError("dates must use ISO YYYY-MM-DD format") from exc + return value + + @field_validator("factor_names", "strategy_ids") + @classmethod + def _unique_ids(cls, values: list[str]) -> list[str]: + if any(not value or len(value) > 120 for value in values): + raise ValueError("IDs must contain 1 to 120 characters") + if len(set(values)) != len(values): + raise ValueError("IDs must be unique") + return values + + @field_validator("factor_names") + @classmethod + def _known_factors(cls, values: list[str]) -> list[str]: + unknown = sorted(set(values) - _FACTOR_IDS) + if unknown: + raise ValueError(f"unknown mining factors: {unknown}") + return values + + @field_validator("symbols") + @classmethod + def _symbols(cls, values: list[str] | None) -> list[str] | None: + if values is None: + return None + cleaned = [value for value in values if value] + if len(cleaned) > 10_000: + raise ValueError("symbols contains more than 10000 entries") + if len(set(cleaned)) != len(cleaned): + raise ValueError("symbols must be unique") + return cleaned or None + + @model_validator(mode="after") + def _date_range(self) -> MiningStartRequest: + if self.start is not None and self.end is not None and self.start > self.end: + raise ValueError("start must not be after end") + return self + + +class MiningSchedulePatch(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + mining_schedule_enabled: bool | None = None + mining_schedule_weekday: int | None = Field(None, ge=0, le=4) + mining_budget_profile: Literal["balanced", "strict"] | None = None + + +@router.get("/availability") +def get_availability( + request: Request, + asset_type: Annotated[Literal["stock", "etf"], Query()] = "stock", + budget_profile: Annotated[ + Literal["exploratory", "balanced", "strict"], Query() + ] = "balanced", + start: Annotated[date | None, Query()] = None, + end: Annotated[date | None, Query()] = None, +) -> dict[str, Any]: + try: + return mining_availability( + request.app.state.repo.store.data_dir, + asset_type=asset_type, + budget_profile=budget_profile, + start=start, + end=end, + ).to_dict() + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/runs") +def list_runs( + request: Request, + limit: Annotated[int, Query(ge=1, le=200)] = 50, + status: Annotated[list[str] | None, Query()] = None, +) -> dict[str, Any]: + manager = _manager(request) + statuses = None + if status: + unknown = sorted(set(status) - RUN_STATUSES) + if unknown: + raise HTTPException(status_code=400, detail=f"unsupported mining statuses: {unknown}") + statuses = status + try: + manifests = manager.store.list_runs(limit=limit, statuses=statuses) + return {"items": [_project_run(manager.store, item) for item in manifests]} + except MiningRunValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except MiningRunStoreError as exc: + raise HTTPException(status_code=500, detail="failed to read mining runs") from exc + + +@router.post("/runs") +def start_run(payload: MiningStartRequest, request: Request) -> dict[str, Any]: + manager = _manager(request) + worker_request = payload.model_dump(mode="json", exclude={"force"}) + try: + _validate_selected_strategies( + request.app.state.strategy_engine, + payload.strategy_ids, + payload.asset_type, + ) + require_mining_availability( + request.app.state.repo.store.data_dir, + asset_type=payload.asset_type, + budget_profile=payload.budget_profile, + start=payload.start, + end=payload.end, + ) + fingerprint = build_data_fingerprint( + request.app.state.repo, + request.app.state, + worker_request, + ) + existing = None + if not payload.force: + from app.services.mining_jobs import ( + ACTIVE_RUN_STATUSES, + SUCCESS_RUN_STATUSES, + compute_run_signature, + ) + + signature = compute_run_signature(worker_request, fingerprint) + existing = manager.store.find_by_signature( + signature, + statuses=ACTIVE_RUN_STATUSES | SUCCESS_RUN_STATUSES, + ) + manifest = manager.start( + worker_request, + fingerprint, + force=payload.force, + source="manual", + ) + projected = _project_run(manager.store, manifest) + projected["reused"] = existing is not None + return projected + except (MiningRunValidationError, ValueError) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except MiningRunStoreError as exc: + raise HTTPException(status_code=500, detail="failed to persist mining run") from exc + + +@router.get("/runs/{run_id}") +def get_run(run_id: str, request: Request) -> dict[str, Any]: + store = _manager(request).store + return _project_run(store, _required_manifest(store, run_id)) + + +@router.post("/runs/{run_id}/cancel") +def cancel_run(run_id: str, request: Request) -> dict[str, Any]: + manager = _manager(request) + try: + return _project_run(manager.store, manager.cancel(run_id)) + except KeyError as exc: + raise HTTPException(status_code=404, detail="mining run not found") from exc + except MiningRunValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/runs/{run_id}/result") +def get_result(run_id: str, request: Request) -> dict[str, Any]: + store = _manager(request).store + manifest = _required_manifest(store, run_id) + status = str(manifest["status"]) + if status not in SUCCESS_RUN_STATUSES: + status_code = 409 if status not in TERMINAL_RUN_STATUSES else 422 + raise HTTPException( + status_code=status_code, + detail=f"mining result is unavailable for status {status}", + ) + try: + summary = store.read_summary(run_id) + frames = { + name: _read_registered_artifact(store, manifest, name) + for name in ("factors", "correlation", "candidates", "folds") + } + return _project_result(manifest, summary, frames) + except ( + MiningRunStoreError, + OSError, + pl.exceptions.PolarsError, + ValueError, + ) as exc: + raise HTTPException( + status_code=500, + detail="mining result artifacts are unavailable", + ) from exc + + +@router.get("/runs/{run_id}/events") +def stream_events( + run_id: str, + request: Request, + last_event_id: str | None = Header(None, alias="Last-Event-ID"), +) -> EventSourceResponse: + store = _manager(request).store + _required_manifest(store, run_id) + cursor = _event_cursor(last_event_id) + + async def generate() -> AsyncIterator[dict[str, str]]: + nonlocal cursor + last_emit = asyncio.get_running_loop().time() + terminal_sent = False + first_batch = True + while not await request.is_disconnected(): + events = await asyncio.to_thread(store.read_events, run_id, after_id=cursor) + if first_batch and events and int(events[0]["id"]) > cursor + 1: + summary = await asyncio.to_thread(store.read_summary, run_id) + progress = summary.get("progress") + if isinstance(progress, Mapping): + yield { + "id": str(cursor), + "event": "progress", + "data": json.dumps(progress, ensure_ascii=False, allow_nan=False), + } + last_emit = asyncio.get_running_loop().time() + first_batch = False + for event in events: + cursor = int(event["id"]) + event_type = "failed" if event.get("type") == "error" else str(event["type"]) + payload = dict(event.get("payload") or {}) + if event_type in TERMINAL_RUN_STATUSES: + payload.setdefault("status", event_type) + terminal_sent = True + yield { + "id": str(cursor), + "event": event_type, + "data": json.dumps(payload, ensure_ascii=False, allow_nan=False), + } + last_emit = asyncio.get_running_loop().time() + manifest = await asyncio.to_thread(store.get, run_id) + if manifest is None: + return + status = str(manifest["status"]) + if status in TERMINAL_RUN_STATUSES: + if not terminal_sent: + event_type = "failed" if status == "failed" else status + yield { + "id": str(cursor), + "event": event_type, + "data": json.dumps( + {"status": status, "message": manifest.get("error")}, + ensure_ascii=False, + ), + } + return + now = asyncio.get_running_loop().time() + if now - last_emit >= _SSE_HEARTBEAT_SECONDS: + yield {"event": "heartbeat", "data": "{}"} + last_emit = now + await asyncio.sleep(_SSE_POLL_SECONDS) + + return EventSourceResponse(generate(), ping=_SSE_HEARTBEAT_SECONDS) + + +@router.post("/runs/{run_id}/candidates/{signature}/promote") +def promote_candidate(run_id: str, signature: str, request: Request) -> dict[str, Any]: + service = _candidate_service(request) + try: + return service.promote(run_id, signature) + except KeyError as exc: + raise HTTPException(status_code=404, detail="mining run or candidate not found") from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + + +@router.post("/runs/{run_id}/candidates/{signature}/publish") +def publish_candidate(run_id: str, signature: str, request: Request) -> dict[str, Any]: + service = _candidate_service(request) + try: + return service.publish(run_id, signature) + except KeyError as exc: + raise HTTPException(status_code=404, detail="mining run or candidate not found") from exc + except FileExistsError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + + +@router.get("/config") +def get_config() -> dict[str, Any]: + return preferences.get_mining_schedule() + + +@router.patch("/config") +def update_config(payload: MiningSchedulePatch) -> dict[str, Any]: + current = preferences.get_mining_schedule() + updates = payload.model_dump(exclude_none=True) + if not updates: + raise HTTPException(status_code=400, detail="at least one mining config field is required") + merged = {**current, **updates} + try: + return preferences.set_mining_schedule( + merged["mining_schedule_enabled"], + merged["mining_schedule_weekday"], + merged["mining_budget_profile"], + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +def _manager(request: Request): + manager = getattr(request.app.state, "mining_manager", None) + if manager is None: + raise HTTPException(status_code=503, detail="mining manager is unavailable") + return manager + + +def _candidate_service(request: Request): + service = getattr(request.app.state, "mining_candidate_service", None) + if service is not None: + return service + from app.backtest.candidates import CandidateStore + from app.services.mining_candidates import MiningCandidateService + + manager = _manager(request) + data_dir = request.app.state.repo.store.data_dir + monitor_engine = getattr(request.app.state, "monitor_engine", None) + service = MiningCandidateService( + data_dir, + manager.store, + CandidateStore(data_dir), + request.app.state.strategy_engine, + monitor_state_invalidator=( + monitor_engine.invalidate_strategy_state + if monitor_engine is not None + else None + ), + ) + request.app.state.mining_candidate_service = service + return service + + +def _required_manifest(store: MiningRunStore, run_id: str) -> dict[str, Any]: + try: + manifest = store.get(run_id) + except MiningRunValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except MiningRunStoreError as exc: + raise HTTPException(status_code=500, detail="failed to read mining run") from exc + if manifest is None: + raise HTTPException(status_code=404, detail="mining run not found") + return manifest + + +def _project_run(store: MiningRunStore, manifest: Mapping[str, Any]) -> dict[str, Any]: + run_id = str(manifest["run_id"]) + summary = store.read_summary(run_id) + events = store.read_events(run_id) + source = next( + ( + event.get("payload", {}).get("source") + for event in events + if event.get("type") == "queued" and event.get("payload", {}).get("source") + ), + None, + ) + if source is None and isinstance(manifest.get("data_fingerprint"), Mapping): + source = manifest["data_fingerprint"].get("source") + compact = _summary_projection(summary) if manifest["status"] in SUCCESS_RUN_STATUSES else None + return { + "run_id": run_id, + "signature": manifest["run_signature"], + "status": manifest["status"], + "request": manifest.get("request") or {}, + "source": source or "manual", + "created_at": manifest.get("created_at"), + "updated_at": manifest.get("updated_at"), + "started_at": manifest.get("started_at"), + "finished_at": manifest.get("finished_at"), + "data_as_of": summary.get("data_as_of"), + "progress": ( + summary.get("progress") + if isinstance(summary.get("progress"), Mapping) + else None + ), + "error": manifest.get("error"), + "summary": compact, + } + + +def _request_summary(manifest: Mapping[str, Any]) -> dict[str, Any]: + request = manifest.get("request") or {} + factor_names = request.get("factor_names") + strategy_ids = request.get("strategy_ids") + return { + "asset_type": request.get("asset_type") or "stock", + "budget_profile": request.get("budget_profile") or "balanced", + "start": request.get("start"), + "end": request.get("end"), + "factor_count": len(factor_names) if isinstance(factor_names, list) else 0, + "strategy_count": len(strategy_ids) if isinstance(strategy_ids, list) else 0, + "commission_pct": _finite(request.get("commission_pct")), + "stamp_tax_pct": _finite(request.get("stamp_tax_pct")), + "slippage_bps": _finite(request.get("slippage_bps")), + "correlation_threshold": _finite(request.get("correlation_threshold")), + } + + +def _summary_projection(summary: Mapping[str, Any]) -> dict[str, Any]: + worker = summary.get("worker") if isinstance(summary.get("worker"), Mapping) else {} + return { + "factor_count": int(summary.get("factor_count") or 0), + "selected_factor_count": int(summary.get("selected_factor_count") or 0), + "candidate_count": int(summary.get("candidate_count") or 0), + "valid_fold_count": int(summary.get("valid_fold_count") or 0), + "skipped_fold_count": int(summary.get("skipped_fold_count") or 0), + "confidence": summary.get("confidence") or "low", + "budget_exhausted": bool(summary.get("budget_exhausted", False)), + "elapsed_ms": _finite(summary.get("elapsed_ms")), + "peak_rss_bytes": _optional_int(worker.get("peak_rss_bytes")), + } + + +def _read_registered_artifact( + store: MiningRunStore, + manifest: Mapping[str, Any], + name: str, +) -> pl.DataFrame: + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, Mapping) or name not in artifacts: + raise ValueError(f"mining artifact is not registered: {name}") + raw_path = artifacts[name] + if not isinstance(raw_path, str): + raise ValueError(f"mining artifact registration is invalid: {name}") + run_dir = store.artifact_path(str(manifest["run_id"]), name).parent # type: ignore[arg-type] + registered = (run_dir / raw_path).resolve() + if not registered.is_relative_to(run_dir.resolve()): + raise ValueError(f"mining artifact escapes its run directory: {name}") + if registered.suffix.lower() != ".parquet" or not registered.is_file(): + raise ValueError(f"mining artifact is unavailable: {name}") + if registered.stat().st_size > _MAX_ARTIFACT_BYTES: + raise ValueError(f"mining artifact exceeds size limit: {name}") + return pl.read_parquet(registered) + + +def _project_result( + manifest: Mapping[str, Any], + summary: Mapping[str, Any], + frames: Mapping[str, pl.DataFrame], +) -> dict[str, Any]: + factors = [_clean_record(row) for row in frames["factors"].to_dicts()] + correlation = _project_correlation(frames["correlation"]) + fold_records = [_project_fold(row) for row in frames["folds"].to_dicts()] + candidates = _project_candidates(frames["candidates"], fold_records) + selected_signature = candidates[0]["signature"] if candidates else None + folds = [ + _public_fold(row) + for row in fold_records + if row["regime_state"] == "overall" + and (selected_signature is None or row["candidate_signature"] == selected_signature) + ] + regimes = _project_regimes(fold_records, selected_signature) + worker = summary.get("worker") if isinstance(summary.get("worker"), Mapping) else {} + threshold = _finite((manifest.get("request") or {}).get("correlation_threshold")) + correlation["threshold"] = threshold if threshold is not None else 0.75 + return { + "run_id": manifest["run_id"], + "methodology_version": summary.get("methodology_version") or "factor_v2", + "algorithm_version": summary.get("algorithm_version") or MINING_ALGORITHM_VERSION, + "data_as_of": summary.get("data_as_of"), + "request_summary": _request_summary(manifest), + "summary": _summary_projection(summary), + "factors": factors, + "correlation": correlation, + "regimes": regimes, + "candidates": candidates, + "folds": folds, + "telemetry": { + "elapsed_ms": _finite(summary.get("elapsed_ms")), + "peak_rss_bytes": _optional_int(worker.get("peak_rss_bytes")), + "panel_scans": _optional_int(summary.get("panel_scans")), + "matrix_bytes": _optional_int(summary.get("matrix_bytes")), + "serialized_result_bytes": _optional_int(worker.get("serialized_result_bytes")), + "phase_ms": _finite_mapping(summary.get("phase_ms")), + }, + } + + +def _project_correlation(frame: pl.DataFrame) -> dict[str, Any]: + required = {"factor_x", "factor_y", "rho", "pair_count"} + if not required.issubset(frame.columns): + raise ValueError("correlation artifact schema is invalid") + labels = sorted(set(frame["factor_x"].to_list()) | set(frame["factor_y"].to_list())) + positions = {str(label): index for index, label in enumerate(labels)} + matrix: list[list[float | None]] = [[None for _ in labels] for _ in labels] + counts: list[list[int | None]] = [[None for _ in labels] for _ in labels] + for row in frame.iter_rows(named=True): + left = positions[str(row["factor_x"])] + right = positions[str(row["factor_y"])] + matrix[left][right] = _finite(row["rho"]) + counts[left][right] = _optional_int(row["pair_count"]) + return {"labels": labels, "matrix": matrix, "pair_counts": counts} + + +def _project_fold(row: Mapping[str, Any]) -> dict[str, Any]: + projected = _clean_record(row) + projected["selected_factors"] = _json_string_list(row.get("selected_factors_json")) + projected["candidate_signature"] = row.get("candidate_signature") + projected["regime_state"] = str(row.get("regime_state") or "overall") + projected["n_dates"] = int(row.get("n_dates") or 0) + return projected + + +def _public_fold(row: Mapping[str, Any]) -> dict[str, Any]: + return { + key: row.get(key) + for key in ( + "fold", "label", "train_start", "train_end", "test_start", "test_end", + "selected_factors", "total_return", "sharpe", "max_drawdown", "n_trades", + "skipped", "reason", "evaluation_kind", + ) + } + + +def _project_candidates( + frame: pl.DataFrame, + folds: Sequence[Mapping[str, Any]], +) -> list[dict[str, Any]]: + required = {"signature", "name", "kind", "factor_names_json", "confidence"} + if not required.issubset(frame.columns): + raise ValueError("candidates artifact schema is invalid") + candidates = [] + for row in frame.to_dicts(): + candidate = _clean_record(row) + candidate.pop("definition_json", None) + candidate.pop("factor_names_json", None) + candidate["factor_names"] = _json_string_list(row.get("factor_names_json")) + signature = str(row["signature"]) + candidate["folds"] = [ + _public_fold(fold) + for fold in folds + if fold["regime_state"] == "overall" + and fold["candidate_signature"] == signature + ] + gate = evaluate_candidate_gate( + confidence=row.get("confidence"), + valid_folds=row.get("valid_folds"), + positive_fold_ratio=row.get("oos_positive_fold_ratio"), + sharpe=row.get("oos_sharpe"), + max_drawdown=row.get("oos_max_drawdown"), + n_trades=row.get("oos_n_trades"), + ) + candidate["gate"] = { + "qualified": gate.qualified, + "reasons": list(gate.reasons), + } + candidates.append(candidate) + candidates.sort( + key=lambda item: ( + -(item.get("oos_sharpe") if item.get("oos_sharpe") is not None else -math.inf), + str(item["signature"]), + ) + ) + return candidates + + +def _project_regimes( + folds: Sequence[Mapping[str, Any]], + signature: str | None, +) -> list[dict[str, Any]]: + labels = {"overall": "整体", "strong": "强势", "range": "震荡", "weak": "弱势"} + result = [] + for state in ("overall", "strong", "range", "weak"): + rows = [ + row + for row in folds + if row["regime_state"] == state + and (signature is None or row["candidate_signature"] == signature) + and not row.get("skipped") + ] + result.append({ + "state": state, + "label": labels[state], + "n_dates": sum(int(row.get("n_dates") or 0) for row in rows), + "total_return": _mean(row.get("total_return") for row in rows), + "sharpe": _mean(row.get("sharpe") for row in rows), + "max_drawdown": _minimum(row.get("max_drawdown") for row in rows), + }) + return result + + +def _validate_selected_strategies( + strategy_engine: Any, + strategy_ids: Sequence[str], + asset_type: str, +) -> None: + for strategy_id in strategy_ids: + strategy = strategy_engine.get(strategy_id) + if strategy.meta.get("research_only"): + raise ValueError(f"research template cannot be mined as existing: {strategy_id}") + if strategy.execution_backend != "matrix_native": + raise ValueError(f"mining strategy is not matrix-native: {strategy_id}") + if "1d" not in strategy.meta.get("timeframes", ["1d"]): + raise ValueError(f"mining strategy is not daily-compatible: {strategy_id}") + if asset_type not in strategy.meta.get("asset_types", ["stock"]): + raise ValueError(f"mining strategy does not support {asset_type}: {strategy_id}") + + +def _event_cursor(value: str | None) -> int: + if value in (None, ""): + return 0 + try: + cursor = int(value) + except ValueError as exc: + raise HTTPException(status_code=400, detail="Last-Event-ID must be an integer") from exc + if cursor < 0: + raise HTTPException(status_code=400, detail="Last-Event-ID must be non-negative") + return cursor + + +def _json_string_list(value: Any) -> list[str]: + if not isinstance(value, str): + return [] + parsed = json.loads(value) + if not isinstance(parsed, list) or any(not isinstance(item, str) for item in parsed): + raise ValueError("artifact JSON list is invalid") + return parsed + + +def _clean_record(row: Mapping[str, Any]) -> dict[str, Any]: + return { + str(key): (_finite(value) if isinstance(value, float) else value) + for key, value in row.items() + } + + +def _finite(value: Any) -> float | None: + if value is None: + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None + + +def _optional_int(value: Any) -> int | None: + number = _finite(value) + return int(number) if number is not None else None + + +def _finite_mapping(value: Any) -> dict[str, float] | None: + if not isinstance(value, Mapping): + return None + return { + str(key): number + for key, item in value.items() + if (number := _finite(item)) is not None + } + + +def _mean(values: Sequence[Any] | Any) -> float | None: + finite = [number for value in values if (number := _finite(value)) is not None] + return sum(finite) / len(finite) if finite else None + + +def _minimum(values: Sequence[Any] | Any) -> float | None: + finite = [number for value in values if (number := _finite(value)) is not None] + return min(finite) if finite else None diff --git a/backend/app/api/regime.py b/backend/app/api/regime.py index f484033..57b9bd7 100644 --- a/backend/app/api/regime.py +++ b/backend/app/api/regime.py @@ -7,8 +7,9 @@ from __future__ import annotations import threading import time from datetime import date -from typing import Any +from typing import Annotated, Any +import polars as pl from fastapi import APIRouter, Query, Request from app.services import regime_builder @@ -142,6 +143,8 @@ def regime_recompute(request: Request, start: date | None = None, end: date | No 与 daily_pipeline 的增量补差(compute_regime_incremental)不同 —— 此接口面向 人工「我要重新算一遍」的预期, 必须真正重算而非增量补缺口。 - 传 start: 仅重算 [start, end] 区间。 + - 重算后统一重标情绪周期阶段(refresh_phase_labels)并回填主线 + (概念+行业, 概念成分为当前快照回看历史, 早年有归属漂移)。 """ repo = request.app.state.repo data_dir = _data_dir(request) @@ -156,5 +159,201 @@ def regime_recompute(request: Request, start: date | None = None, end: date | No new_rows = regime_builder.run_regime_batch(repo, start=start, end=end) if not new_rows.is_empty(): regime_builder.upsert_regime_history(data_dir, new_rows) + phase_days = regime_builder.refresh_phase_labels(data_dir) + + from app.services import market_mainline + + mainline_rows = 0 + for kind in ("concept", "industry"): + rows = market_mainline.compute_mainline_range(repo, data_dir, start, end, kind=kind) + if not rows.is_empty(): + market_mainline.upsert_mainline_history(data_dir, rows) + mainline_rows += rows.height + invalidate_regime_cache() - return {"ok": True, "computed": new_rows.height if not new_rows.is_empty() else 0} + return { + "ok": True, + "computed": new_rows.height if not new_rows.is_empty() else 0, + "phase_days": phase_days, + "mainline_rows": mainline_rows, + } + + +@router.get("/phases") +def regime_phases( + request: Request, + start: date | None = None, + end: date | None = None, +): + """情绪周期阶段段列表: 连续同阶段合段, 附段内均值指标与主导主线。 + + 直接回答「什么阶段走什么主升」: 主升/高潮段的主导主线即该段行情主线。 + 主线按段内进入当日 top5 的天数与累计分排序, 取前 3。 + """ + from app.services.market_mainline import load_mainline_history + from app.services.market_phase import PHASE_LABELS + + data_dir = _data_dir(request) + df = regime_builder.load_regime_history(data_dir) + if df.is_empty() or "phase" not in df.columns: + return {"segments": [], "total": 0} + if start: + df = df.filter(pl_col_date(df, ">=", start)) + if end: + df = df.filter(pl_col_date(df, "<=", end)) + df = df.sort("date") + if df.is_empty(): + return {"segments": [], "total": 0} + + mainline = load_mainline_history(data_dir, "concept") + + segments: list[dict] = [] + cur: dict | None = None + for r in df.iter_rows(named=True): + phase = r.get("phase") + if cur is None or cur["phase"] != phase: + cur = { + "phase": phase, + "label": PHASE_LABELS.get(phase, phase), + "start": str(r["date"]), + "end": str(r["date"]), + "days": 0, + "_height": 0.0, + "_first_board": 0.0, + "_ge2": 0.0, + "_promo_sum": 0.0, + "_promo_n": 0, + "_seal": 0.0, + } + segments.append(cur) + cur["end"] = str(r["date"]) + cur["days"] += 1 + cur["_height"] += float(r.get("max_consecutive") or 0) + cur["_first_board"] += float(r.get("first_board") or 0) + cur["_ge2"] += float(r.get("ge2_count") or 0) + promo = r.get("promo_rate") + if promo is not None: + cur["_promo_sum"] += float(promo) + cur["_promo_n"] += 1 + cur["_seal"] += float(r.get("seal_rate") or 0) + + for seg in segments: + n = seg["days"] + seg["avg_height"] = round(seg.pop("_height") / n, 1) + seg["avg_first_board"] = round(seg.pop("_first_board") / n, 1) + seg["avg_ge2"] = round(seg.pop("_ge2") / n, 1) + seg["avg_promo"] = ( + round(seg.pop("_promo_sum") / seg["_promo_n"], 3) if seg["_promo_n"] else None + ) + seg.pop("_promo_n") + seg["avg_seal_rate"] = round(seg.pop("_seal") / n, 3) + seg["top_mainlines"] = _segment_mainlines( + mainline, date.fromisoformat(seg["start"]), date.fromisoformat(seg["end"]) + ) + + return {"segments": segments, "total": len(segments)} + + +def _segment_mainlines(mainline: pl.DataFrame, start: date, end: date, top: int = 3) -> list[dict]: + """段内主导主线: 按进入当日 top5 的天数与累计分排序。""" + if mainline.is_empty(): + return [] + seg = mainline.filter( + (pl.col("date") >= start) & (pl.col("date") <= end) & (pl.col("rank") <= 5) + ) + if seg.is_empty(): + return [] + ranked = ( + seg.group_by("member") + .agg( + pl.col("date").n_unique().alias("top5_days"), + pl.col("score").sum().alias("score_sum"), + pl.col("max_boards").max().alias("max_boards"), + pl.col("leader_symbol").first().alias("leader_symbol"), + ) + .sort(["top5_days", "score_sum"], descending=[True, True]) + .head(top) + ) + return [ + { + "member": r["member"], + "top5_days": r["top5_days"], + "score_sum": round(r["score_sum"], 1), + "max_boards": r["max_boards"], + "leader_symbol": r["leader_symbol"], + } + for r in ranked.to_dicts() + ] + + +@router.post("/mainline/recompute") +def mainline_recompute(request: Request): + """全量重算主线(概念+行业), 应用当前过滤配置。窄扫描, 秒级。 + + 修改过滤配置(preferences mainline-filter)后调用本接口生效, + 无需触发较重的 regime 全量重算。 + """ + from app.services import market_mainline + + repo = request.app.state.repo + data_dir = _data_dir(request) + earliest = regime_builder.earliest_enriched_date(repo) + if earliest is None: + return {"ok": True, "rows": 0} + rows = 0 + for kind in ("concept", "industry"): + computed = market_mainline.compute_mainline_range( + repo, data_dir, earliest, date.today(), kind=kind + ) + if not computed.is_empty(): + market_mainline.upsert_mainline_history(data_dir, computed) + rows += computed.height + return {"ok": True, "rows": rows} + + +@router.get("/mainline") +def regime_mainline( + request: Request, + start: date | None = None, + end: date | None = None, + top: Annotated[int, Query(ge=1, le=30)] = 10, + kind: Annotated[str, Query(pattern="^(concept|industry)$")] = "concept", +): + """每日主线排行(截 rank<=top) + 窗口内持续性汇总。 + + membership_note 说明概念成分口径(当前快照回看历史)。 + """ + from app.services.market_mainline import MEMBERSHIP_NOTE, load_mainline_history + + try: + from app.services import preferences + + filter_cfg = preferences.get_mainline_filter_config() + except Exception: + filter_cfg = {"min_members": 4, "max_members": 600, "blacklist": []} + df = load_mainline_history(_data_dir(request), kind) + if df.is_empty(): + return {"rows": [], "leaders": [], "membership_note": MEMBERSHIP_NOTE, "filter": filter_cfg} + if start: + df = df.filter(pl_col_date(df, ">=", start)) + if end: + df = df.filter(pl_col_date(df, "<=", end)) + df = df.sort(["date", "rank"]) + rows_df = df.filter(pl.col("rank") <= top) + leaders = ( + df.filter(pl.col("rank") == 1) + .group_by("member") + .agg( + pl.col("date").n_unique().alias("top1_days"), + pl.col("score").mean().round(1).alias("avg_score"), + pl.col("max_boards").max().alias("max_boards"), + ) + .sort(["top1_days", "avg_score"], descending=[True, True]) + .head(10) + ) + return { + "rows": _df_to_records(rows_df), + "leaders": leaders.to_dicts(), + "membership_note": MEMBERSHIP_NOTE, + "filter": filter_cfg, + } diff --git a/backend/app/api/screener.py b/backend/app/api/screener.py index b05a5f1..784d105 100644 --- a/backend/app/api/screener.py +++ b/backend/app/api/screener.py @@ -238,6 +238,8 @@ def strategies( raise HTTPException(status_code=503, detail="策略引擎未初始化") presets = [] for meta in engine.list_strategies(): + if meta.get("research_only"): + continue if asset_type not in meta.get("asset_types", ["stock"]): continue if timeframe not in meta.get("timeframes", ["1d"]): @@ -292,6 +294,8 @@ def run_preset(req: PresetRequest, request: Request): try: if not engine.has(req.strategy_id): raise ValueError(f"unknown strategy: {req.strategy_id}") + if engine.get(req.strategy_id).meta.get("research_only"): + raise ValueError(f"unknown strategy: {req.strategy_id}") params = dict(overrides.get("params") or {}) context = svc.build_strategy_context( engine, @@ -520,14 +524,19 @@ def run_all(request: Request, body: Optional[dict] = None): requested_ids = body.get("strategy_ids") if requested_ids and isinstance(requested_ids, list): all_ids = [str(sid) for sid in requested_ids] - unknown = [sid for sid in all_ids if not engine.has(sid)] + unknown = [ + sid + for sid in all_ids + if not engine.has(sid) or engine.get(sid).meta.get("research_only") + ] if unknown: raise HTTPException(status_code=404, detail=f"unknown strategies: {unknown}") else: all_ids = [ meta["id"] for meta in engine.list_strategies() - if asset_type in meta.get("asset_types", ["stock"]) + if not meta.get("research_only") + and asset_type in meta.get("asset_types", ["stock"]) and timeframe in meta.get("timeframes", ["1d"]) ] @@ -707,7 +716,9 @@ def limit_ladder( sealed_ready = False sealed_age: float | None = None if depth_svc: - sealed_map = depth_svc.get_sealed_map(as_of, is_down=is_down) + # 复用上方双方向计数已读取的 sealed map: 同一请求、同一 as_of、同一对象, + # 不再第三次读取 (内存路径含全量浅拷贝, parquet 路径含整文件读)。 + sealed_map = down_map if is_down else up_map sealed_ready = bool(sealed_map) and depth_svc.is_sealed_ready(as_of) sealed_age = depth_svc.get_sealed_age(as_of) if sealed_ready else None diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 6175b22..a106025 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -6,9 +6,10 @@ from __future__ import annotations import logging import time +from typing import Literal from fastapi import APIRouter, HTTPException, Request -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from app import secrets_store from app.data_providers.custom.config import MAX_TIMEOUT @@ -427,6 +428,14 @@ class CustomSourceTestIn(BaseModel): config: CustomSourceIn | None = None +class MiningSchedulePrefs(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + mining_schedule_enabled: bool + mining_schedule_weekday: int = Field(ge=0, le=4) + mining_budget_profile: Literal["balanced", "strict"] + + @router.get("/preferences") def get_preferences() -> dict: """返回用户偏好设置。""" @@ -485,6 +494,7 @@ def get_preferences() -> dict: "depth_finalize_time": preferences.get_depth_finalize_time(), "review_schedule": preferences.get_review_schedule(), "review_push_channels": preferences.get_review_push_channels(), + **preferences.get_mining_schedule(), } @@ -657,6 +667,18 @@ def update_data_source_job_timeouts(req: DataSourceJobTimeoutPrefs) -> dict: return req.model_dump() +@router.put("/preferences/mining-schedule") +def update_mining_schedule(req: MiningSchedulePrefs) -> dict: + """一次更新周度自动 mining 配置。""" + from app.services import preferences + + return preferences.set_mining_schedule( + req.mining_schedule_enabled, + req.mining_schedule_weekday, + req.mining_budget_profile, + ) + + @router.get("/preferences/watchlist-columns") def get_watchlist_columns() -> dict: """返回自选列表列配置。""" @@ -927,6 +949,23 @@ class PipelineIndexSymbolsIn(BaseModel): symbols: str = "" +class MainlineFilterIn(BaseModel): + """市场主线过滤配置(宽基/风格标签按成员数过滤 + 名称黑名单)。""" + + min_members: int | None = None + max_members: int | None = None + blacklist: list[str] | str | None = None + + +@router.put("/preferences/mainline-filter") +def update_mainline_filter(req: MainlineFilterIn) -> dict: + """更新市场主线过滤配置。部分更新; 修改后需重算主线(POST /api/regime/mainline/recompute)生效。""" + from app.services import preferences + + payload = req.model_dump() + return preferences.set_mainline_filter_config(payload) + + @router.put("/preferences/pipeline-index-symbols") def update_pipeline_index_symbols(req: PipelineIndexSymbolsIn) -> dict: """保存指数自定义拉取代码。""" diff --git a/backend/app/api/strategy.py b/backend/app/api/strategy.py index 48c5390..53803fc 100644 --- a/backend/app/api/strategy.py +++ b/backend/app/api/strategy.py @@ -43,6 +43,16 @@ def _get_engine(request: Request) -> StrategyEngine: return engine +def _get_public_strategy(engine: StrategyEngine, strategy_id: str) -> StrategyDef: + try: + strategy = engine.get(strategy_id) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + if strategy.meta.get("research_only"): + raise HTTPException(status_code=404, detail=f"unknown strategy: {strategy_id}") + return strategy + + def _get_monitor(request: Request) -> StrategyMonitorService: mon = getattr(request.app.state, "strategy_monitor", None) if not mon: @@ -288,6 +298,8 @@ def list_strategies( result = [] for meta in engine.list_strategies(): + if meta.get("research_only"): + continue if asset_type and asset_type not in meta.get("asset_types", ["stock"]): continue if timeframe and timeframe not in meta.get("timeframes", ["1d"]): @@ -302,10 +314,7 @@ def list_strategies( @router.get("/{strategy_id}") def get_strategy(strategy_id: str, request: Request): engine = _get_engine(request) - try: - s = engine.get(strategy_id) - except ValueError as e: - raise HTTPException(status_code=404, detail=str(e)) from e + s = _get_public_strategy(engine, strategy_id) overrides = strategy_config.load_override(_data_dir(request), strategy_id) return _strategy_detail(s, overrides or None, engine) @@ -316,6 +325,7 @@ def get_strategy(strategy_id: str, request: Request): @router.post("/run") def run_strategy(req: RunRequest, request: Request): engine = _get_engine(request) + _get_public_strategy(engine, req.strategy_id) data_dir = _data_dir(request) # 读取用户覆盖配置 @@ -377,7 +387,8 @@ def run_all(req: RunAllRequest, request: Request): strategy_ids = [ meta["id"] for meta in engine.list_strategies() - if req.asset_type in meta.get("asset_types", ["stock"]) + if not meta.get("research_only") + and req.asset_type in meta.get("asset_types", ["stock"]) and req.timeframe in meta.get("timeframes", ["1d"]) ] from app.services.screener import ScreenerService @@ -412,8 +423,7 @@ def run_all(req: RunAllRequest, request: Request): @router.post("/config") def save_config(req: SaveConfigRequest, request: Request): engine = _get_engine(request) - if not engine.has(req.strategy_id): - raise HTTPException(status_code=404, detail=f"策略 {req.strategy_id} 不存在") + _get_public_strategy(engine, req.strategy_id) _validate_scoring_config(req.overrides) # 剥离与策略默认值相同的字段，只保存用户真正修改过的值 @@ -426,8 +436,7 @@ def save_config(req: SaveConfigRequest, request: Request): @router.patch("/config") def patch_config(req: SaveConfigRequest, request: Request): engine = _get_engine(request) - if not engine.has(req.strategy_id): - raise HTTPException(status_code=404, detail=f"策略 {req.strategy_id} 不存在") + _get_public_strategy(engine, req.strategy_id) data_dir = _data_dir(request) overrides = strategy_config.load_override(data_dir, req.strategy_id) overrides.update(req.overrides) @@ -491,6 +500,7 @@ def _strip_defaults(strategy_id: str, overrides: dict, engine) -> dict: @router.delete("/config/{strategy_id}") def reset_config(strategy_id: str, request: Request): + _get_public_strategy(_get_engine(request), strategy_id) strategy_config.delete_override(_data_dir(request), strategy_id) return {"ok": True} @@ -755,10 +765,7 @@ def get_strategy_source(strategy_id: str, request: Request): # 先查 StrategyEngine 获取文件路径 engine = _get_engine(request) - try: - s = engine.get(strategy_id) - except ValueError: - raise HTTPException(status_code=404, detail=f"策略 {strategy_id} 不存在") + s = _get_public_strategy(engine, strategy_id) path = s.file_path if not path or not path.exists(): @@ -964,14 +971,12 @@ def _save_composite_strategy(req: StrategyCompositeSaveRequest, request: Request children = [{"strategy_id": c.strategy_id, "weight": c.weight} for c in req.children] # 子策略存在性预检(给出清晰错误, 而非等到 reload 后孤儿移除的笼统报错)。 for c in children: - if not engine.has(c["strategy_id"]): - raise ValueError(f"子策略 {c['strategy_id']!r} 不存在") try: - child_def = engine.get(c["strategy_id"]) - if child_def.execution_backend == "composite": - raise ValueError(f"子策略 {c['strategy_id']!r} 也是叠加策略; 首版禁止嵌套叠加") - except ValueError: - raise + child_def = _get_public_strategy(engine, c["strategy_id"]) + except HTTPException as exc: + raise ValueError(f"子策略 {c['strategy_id']!r} 不存在") from exc + if child_def.execution_backend == "composite": + raise ValueError(f"子策略 {c['strategy_id']!r} 也是叠加策略; 首版禁止嵌套叠加") code = _render_composite_code( sid, req.name, req.description, children, req.merge_mode, req.min_confirm diff --git a/backend/app/api/watchlist.py b/backend/app/api/watchlist.py index 9b29660..2ff2966 100644 --- a/backend/app/api/watchlist.py +++ b/backend/app/api/watchlist.py @@ -212,7 +212,7 @@ def clear_all(): # 自选页需要的列 _WATCHLIST_COLS = [ - "symbol", "close", "change_pct", "change_amount", "amount", + "symbol", "close", "open", "high", "low", "change_pct", "change_amount", "amount", "turnover_rate", "amplitude", "annual_vol_20d", "vol_ratio_5d", diff --git a/backend/app/backtest/candidates.py b/backend/app/backtest/candidates.py index 6af771d..52b55f2 100644 --- a/backend/app/backtest/candidates.py +++ b/backend/app/backtest/candidates.py @@ -1,4 +1,5 @@ """量化研究候选方案的轻量本地存储。""" + from __future__ import annotations import json @@ -17,28 +18,108 @@ MAX_NAME_LENGTH = 80 MAX_PAYLOAD_BYTES = 32 * 1024 MAX_FILE_BYTES = 2 * 1024 * 1024 +_MINING_SOURCE_CONFIG_FIELDS = frozenset( + { + "origin_run_id", + "candidate_signature", + "regime_state", + "algorithm_version", + "methodology_version", + } +) _CONFIG_FIELDS: dict[str, frozenset[str]] = { - "factor": frozenset({ - "factor_name", "symbols", "start", "end", "n_groups", "rebalance", "weight", - "fees_pct", "slippage_bps", "asset_type", - }), - "strategy": frozenset({ - "strategy_id", "symbols", "start", "end", "params", "overrides", "matching", - "entry_fill", "exit_fill", "fees_pct", "commission_pct", "stamp_tax_pct", - "slippage_bps", "max_positions", "max_exposure_pct", "initial_capital", - "position_sizing", "mode", "holding_days", "asset_type", "minute_fill", - "regime_filter", - }), + "factor": frozenset( + { + "factor_name", + "symbols", + "start", + "end", + "n_groups", + "rebalance", + "weight", + "fees_pct", + "slippage_bps", + "asset_type", + } + ) + | _MINING_SOURCE_CONFIG_FIELDS, + "strategy": frozenset( + { + "strategy_id", + "symbols", + "start", + "end", + "params", + "overrides", + "matching", + "entry_fill", + "exit_fill", + "fees_pct", + "commission_pct", + "stamp_tax_pct", + "slippage_bps", + "max_positions", + "max_exposure_pct", + "initial_capital", + "position_sizing", + "mode", + "holding_days", + "asset_type", + "minute_fill", + "regime_filter", + "factor_names", + "directions", + "weights", + } + ) + | _MINING_SOURCE_CONFIG_FIELDS, } +_MINING_METRIC_FIELDS = frozenset( + { + "oos_sharpe", + "oos_return", + "oos_max_drawdown", + "oos_positive_fold_ratio", + "oos_n_trades", + "valid_folds", + "skipped_folds", + "confidence", + "coverage", + "turnover", + "long_short_sharpe", + } +) _METRIC_FIELDS: dict[str, frozenset[str]] = { - "factor": frozenset({ - "ic_mean", "ic_std", "ir", "ic_win_rate", "long_short_return", - "long_short_max_drawdown", "n_symbols", "n_dates", "elapsed_ms", - }), - "strategy": frozenset({ - "total_return", "annual_return", "max_drawdown", "sharpe", "sortino", "win_rate", - "n_trades", "profit_factor", "avg_return", "median_return", "elapsed_ms", - }), + "factor": frozenset( + { + "ic_mean", + "ic_std", + "ir", + "ic_win_rate", + "long_short_return", + "long_short_max_drawdown", + "n_symbols", + "n_dates", + "elapsed_ms", + } + ) + | _MINING_METRIC_FIELDS, + "strategy": frozenset( + { + "total_return", + "annual_return", + "max_drawdown", + "sharpe", + "sortino", + "win_rate", + "n_trades", + "profit_factor", + "avg_return", + "median_return", + "elapsed_ms", + } + ) + | _MINING_METRIC_FIELDS, } _lock = threading.RLock() @@ -98,6 +179,70 @@ class CandidateStore: self._write(items) return item + def create_or_get_by_provenance( + self, + *, + origin_run_id: str, + candidate_signature: str, + kind: CandidateKind, + name: str, + source_id: str, + config: dict[str, Any], + metrics: dict[str, Any], + data_as_of: str | None, + status: CandidateStatus = "pending", + ) -> dict[str, Any]: + """Atomically return or create one item for a mining run candidate.""" + clean_name = self._validate_name(name) + clean_source_id = source_id.strip() + if not clean_source_id or len(clean_source_id) > 120: + raise CandidateValidationError("候选来源标识不能为空且不能超过 120 个字符") + clean_config = self._validate_config(kind, config) + clean_metrics = self._validate_metrics(kind, metrics) + if ( + clean_config.get("origin_run_id") != origin_run_id + or clean_config.get("candidate_signature") != candidate_signature + ): + raise CandidateValidationError("候选来源与配置中的挖掘溯源不一致") + + with _lock: + items = self._load() + for item in items: + item_config = item.get("config") or {} + if ( + item_config.get("origin_run_id") == origin_run_id + and item_config.get("candidate_signature") == candidate_signature + ): + if ( + item.get("kind") == kind + and item.get("source_id") == clean_source_id + and item_config == clean_config + and item.get("metrics") == clean_metrics + and item.get("data_as_of") == data_as_of + ): + return item + raise CandidateValidationError( + "相同挖掘溯源的候选内容冲突, 已停止覆盖" + ) + if len(items) >= MAX_CANDIDATES: + raise CandidateValidationError(f"候选方案最多保存 {MAX_CANDIDATES} 个") + now = datetime.now(UTC).isoformat() + item = { + "id": uuid.uuid4().hex, + "kind": kind, + "name": clean_name, + "source_id": clean_source_id, + "config": clean_config, + "metrics": clean_metrics, + "data_as_of": data_as_of, + "status": status, + "created_at": now, + "updated_at": now, + } + items.insert(0, item) + self._write(items) + return item + def update( self, candidate_id: str, @@ -203,7 +348,9 @@ class CandidateStore: def _validate_config(kind: CandidateKind, config: dict[str, Any]) -> dict[str, Any]: unknown = set(config) - _CONFIG_FIELDS[kind] if unknown: - raise CandidateValidationError(f"候选配置包含不允许的字段: {', '.join(sorted(unknown))}") + raise CandidateValidationError( + f"候选配置包含不允许的字段: {', '.join(sorted(unknown))}" + ) CandidateStore._check_json_size(config) return config @@ -211,7 +358,9 @@ class CandidateStore: def _validate_metrics(kind: CandidateKind, metrics: dict[str, Any]) -> dict[str, Any]: unknown = set(metrics) - _METRIC_FIELDS[kind] if unknown: - raise CandidateValidationError(f"候选指标包含不允许的字段: {', '.join(sorted(unknown))}") + raise CandidateValidationError( + f"候选指标包含不允许的字段: {', '.join(sorted(unknown))}" + ) if any(isinstance(value, (dict, list)) for value in metrics.values()): raise CandidateValidationError("候选指标只允许保存标量摘要") CandidateStore._check_json_size(metrics) diff --git a/backend/app/backtest/engine.py b/backend/app/backtest/engine.py index 7a6cdb7..b464462 100644 --- a/backend/app/backtest/engine.py +++ b/backend/app/backtest/engine.py @@ -26,6 +26,7 @@ from app.backtest.matrix import ( load_market_data_matrix_from_parquet, ) from app.config import settings +from app.enriched_generation import EnrichedGenerationUnavailableError from app.parquet import scan_enriched_parquet from app.tickflow.repository import KlineRepository @@ -213,8 +214,11 @@ class PanelCache: columns: list[str] | None, compute_fn, asset_type: str = "stock", + generation: str | None = None, ) -> pl.DataFrame: - key = self._make_key(symbols, start, end, columns, asset_type) + key = self._make_key( + symbols, start, end, columns, asset_type, generation + ) now = time.monotonic() with self._lock: @@ -280,13 +284,20 @@ class PanelCache: self._cache.clear() @staticmethod - def _make_key(symbols: list[str] | None, start: date, end: date, columns: list[str] | None, asset_type: str = "stock") -> str: + def _make_key( + symbols: list[str] | None, + start: date, + end: date, + columns: list[str] | None, + asset_type: str = "stock", + generation: str | None = None, + ) -> str: if symbols is None: h = "all" else: h = hashlib.md5(",".join(sorted(symbols)).encode()).hexdigest()[:12] cols = "all" if columns is None else hashlib.md5(",".join(sorted(columns)).encode()).hexdigest()[:8] - return f"{asset_type}:{h}:{start}:{end}:{cols}" + return f"{asset_type}:{generation or 'unmanaged'}:{h}:{start}:{end}:{cols}" # ================================================================ @@ -302,6 +313,23 @@ class BacktestEngine: # ── 数据加载 ────────────────────────────────────── + def data_generation(self, asset_type: str = "stock") -> str | None: + loader = getattr(self.repo, "get_matrix_data_generation", None) + return loader(asset_type) if callable(loader) else None + + def assert_data_generation( + self, + asset_type: str, + expected: str | None, + ) -> None: + if expected is None: + return + current = self.data_generation(asset_type) + if current != expected: + raise EnrichedGenerationUnavailableError( + "enriched data changed while the snapshot was being read" + ) + def load_panel( self, symbols: list[str] | None, @@ -309,9 +337,36 @@ class BacktestEngine: end: date, columns: list[str] | None = None, asset_type: str = "stock", + *, + expected_generation: str | None = None, ) -> pl.DataFrame: """加载 enriched 数据面板，带缓存。asset_type='etf' 时读 ETF enriched。""" - return self._cache.get_or_compute(symbols, start, end, columns, self._load_panel_inner, asset_type=asset_type) + attempts = 1 if expected_generation is not None else 2 + for attempt in range(attempts): + generation = ( + expected_generation + if expected_generation is not None + else self.data_generation(asset_type) + ) + panel = self._cache.get_or_compute( + symbols, + start, + end, + columns, + self._load_panel_inner, + asset_type=asset_type, + generation=generation, + ) + try: + self.assert_data_generation(asset_type, generation) + except EnrichedGenerationUnavailableError: + if attempt + 1 >= attempts: + raise + continue + return panel + raise EnrichedGenerationUnavailableError( + "unable to read a stable enriched data snapshot" + ) def load_panel_for_backtest( self, @@ -338,6 +393,25 @@ class BacktestEngine: if df.is_empty(): return df + from app.backtest.fundamentals import ( + attach_fundamental_factors, + load_fundamental_snapshot, + ) + + fundamental_names = sorted( + getattr(feature_plan, "fundamental_columns", frozenset()) + or frozenset() + ) + if fundamental_names: + # 财务因子列不落 enriched 存储, 在加载口按公告日门控并入。 + df = attach_fundamental_factors( + df, + load_fundamental_snapshot( + self.repo.store.data_dir if self.repo is not None else None + ), + fundamental_names, + ) + instruments = ( self.repo.get_instruments_asset(asset_type) if self.repo is not None @@ -402,6 +476,8 @@ class BacktestEngine: cache_profile: MatrixCacheProfile | None = None, coverage_start: date | None = None, coverage_end: date | None = None, + expected_generation: str | None = None, + cancel_event: threading.Event | None = None, ) -> MarketDataMatrix: """Load a matrix-native backtest directly from projected parquet batches.""" if feature_plan.execution_backend != "matrix_native": @@ -432,35 +508,67 @@ class BacktestEngine: ) generation_loader = getattr(self.repo, "get_matrix_data_generation", None) source_generation = ( - generation_loader(asset_type) - if cache_root is not None and callable(generation_loader) - else None - ) - try: - return load_market_data_matrix_from_parquet( - parquet_root, - start, - end, - field_columns=field_columns, - symbols=symbols, - instruments=instruments, - cache_root=cache_root, - coverage_start=coverage_start, - coverage_end=coverage_end, - cache_field_columns=cache_fields, - cache_max_bytes=cache_max_bytes, - profile_generation=( - cache_profile.generation if cache_profile is not None else "request" - ), - source_generation=source_generation, + expected_generation + if expected_generation is not None + else ( + generation_loader(asset_type) + if callable(generation_loader) + else None ) - except pa.ArrowException as exc: - raise ValueError(f"direct market matrix parquet scan failed: {exc}") from exc + ) + attempts = 1 if expected_generation is not None else 2 + for attempt in range(attempts): + try: + market = load_market_data_matrix_from_parquet( + parquet_root, + start, + end, + field_columns=field_columns, + symbols=symbols, + instruments=instruments, + cache_root=cache_root, + coverage_start=coverage_start, + coverage_end=coverage_end, + cache_field_columns=cache_fields, + cache_max_bytes=cache_max_bytes, + profile_generation=( + cache_profile.generation if cache_profile is not None else "request" + ), + source_generation=source_generation, + cancel_event=cancel_event, + ) + self.assert_data_generation(asset_type, source_generation) + from app.backtest.fundamentals import attach_matrix_fundamental_fields + + fundamental_names = sorted( + getattr(feature_plan, "fundamental_columns", frozenset()) + or frozenset() + ) + if fundamental_names: + # 财务因子不落 enriched 存储: 矩阵加载后按公告日门控附加字段。 + market = attach_matrix_fundamental_fields( + market, + self.repo.store.data_dir if self.repo is not None else None, + fundamental_names, + ) + return market + except EnrichedGenerationUnavailableError: + if attempt + 1 >= attempts: + raise + source_generation = self.data_generation(asset_type) + except pa.ArrowException as exc: + raise ValueError(f"direct market matrix parquet scan failed: {exc}") from exc + raise EnrichedGenerationUnavailableError( + "unable to read a stable enriched matrix snapshot" + ) def cache_stats(self) -> dict: """暴露 PanelCache 遥测快照 (扫盘耗时/次数/命中/复用), 供上层量化 IO 占比。""" return self._cache.stats() + def clear_panel_cache(self) -> None: + self._cache.invalidate() + def _load_panel_inner( self, symbols: list[str] | None, diff --git a/backend/app/backtest/factor.py b/backend/app/backtest/factor.py index 2b48e39..5ecfade 100644 --- a/backend/app/backtest/factor.py +++ b/backend/app/backtest/factor.py @@ -7,14 +7,22 @@ from __future__ import annotations import logging import time import uuid +from collections.abc import Mapping from dataclasses import dataclass, field from datetime import date, timedelta -from typing import Literal +from itertools import pairwise +from pathlib import Path +from typing import Any, Literal import numpy as np import polars as pl from app.backtest.engine import BacktestEngine +from app.backtest.fundamentals import ( + FUNDAMENTAL_FACTOR_NAMES, + attach_fundamental_factors, + load_fundamental_snapshot, +) from app.strategy.scoring import ( VIRTUAL_SCORING_DEPENDENCIES as DERIVED_FACTOR_DEPENDENCIES, ) @@ -76,9 +84,33 @@ FACTOR_COLUMNS: list[dict] = [ {"id": "close_position", "label": "收盘位置", "group": "价格位置", "desc": "收盘价在当日最低价到最高价之间的位置"}, {"id": "distance_to_high_60d", "label": "距60日高点", "group": "价格位置", "desc": "收盘价 / 60日最高收盘价 - 1"}, {"id": "distance_from_low_60d", "label": "距60日低点", "group": "价格位置", "desc": "收盘价 / 60日最低收盘价 - 1"}, + {"id": "vwap_bias", "label": "VWAP乖离", "group": "价格位置", "desc": "收盘价 / 当日成交均价 - 1, 成交均价 = 成交额 / (成交量x100)"}, + + {"id": "max_ret_20d", "label": "20日最大单日涨幅", "group": "收益形态", "desc": "近20个交易日单日涨幅最大值(彩票效应, 高值代表博彩型特征强)"}, + {"id": "ret_skew_20d", "label": "20日收益偏度", "group": "收益形态", "desc": "近20个交易日日收益偏度, 高值代表右偏(偶发大涨)"}, + {"id": "up_days_20d", "label": "20日上涨天数", "group": "收益形态", "desc": "近20个交易日中上涨天数(0~20)"}, + + {"id": "amihud_20d", "label": "20日Amihud非流动性", "group": "流动性", "desc": "近20日平均 |日涨跌幅| / 成交额(亿元), 高值代表流动性差"}, + {"id": "turnover_z_60d", "label": "换手率60日z分", "group": "流动性", "desc": "(当日换手率 - 前60日均值) / 前60日标准差, 衡量换手异动"}, + + {"id": "vol_price_corr_20d", "label": "20日量价相关", "group": "量价", "desc": "近20个交易日日涨跌幅与成交量的相关系数, 高值代表量价同向"}, + {"id": "vol_trend_5_60", "label": "量能趋势(5/60)", "group": "量价", "desc": "5日平均成交量 / 60日平均成交量 - 1"}, + + {"id": "limit_up_count_20d", "label": "涨停基因(20日)", "group": "涨停基因", "desc": "近20个交易日涨停次数"}, + {"id": "limit_up_count_60d", "label": "涨停基因(60日)", "group": "涨停基因", "desc": "近60个交易日涨停次数"}, + + {"id": "pb_latest", "label": "市净率(最新公告)", "group": "财务", "desc": "收盘价 / 最新已公告每股净资产; 无财务数据或公告前为空"}, + {"id": "roe_latest", "label": "ROE(最新公告)", "group": "财务", "desc": "最新已公告净资产收益率(%); 无财务数据或公告前为空"}, + {"id": "gross_margin_latest", "label": "毛利率(最新公告)", "group": "财务", "desc": "最新已公告销售毛利率(%)"}, + {"id": "net_margin_latest", "label": "净利率(最新公告)", "group": "财务", "desc": "最新已公告销售净利率(%)"}, + {"id": "revenue_yoy_latest", "label": "营收增速(最新公告)", "group": "财务", "desc": "最新已公告营业收入同比(%)"}, + {"id": "net_income_yoy_latest", "label": "净利增速(最新公告)", "group": "财务", "desc": "最新已公告归母净利润同比(%)"}, + {"id": "debt_ratio_latest", "label": "资产负债率(最新公告)", "group": "财务", "desc": "最新已公告资产负债率(%)"}, ] FACTOR_WARMUP_DAYS = 120 +FACTOR_METHODOLOGY_VERSION = "factor_v2" +_DAILY_FORWARD_HORIZONS = (1, 3, 5) @dataclass @@ -93,6 +125,8 @@ class FactorConfig: fees_pct: float = 0.0002 slippage_bps: float = 5.0 asset_type: str = "stock" + commission_pct: float | None = None + stamp_tax_pct: float | None = None @dataclass @@ -127,6 +161,14 @@ class FactorResult: n_symbols: int = 0 n_dates: int = 0 error: str | None = None + # factor_v2 兼容扩展字段必须追加在旧字段之后, 保留位置参数语义。 + methodology_version: str = FACTOR_METHODOLOGY_VERSION + coverage: float | None = None + turnover: float | None = None + long_short_sharpe: float | None = None + yearly_ic: list[dict] = field(default_factory=list) + ic_decay: list[dict] = field(default_factory=list) + regime_stats: list[dict] = field(default_factory=list) @dataclass @@ -141,6 +183,8 @@ class FactorBatchConfig: fees_pct: float = 0.0002 slippage_bps: float = 5.0 asset_type: str = "stock" + commission_pct: float | None = None + stamp_tax_pct: float | None = None @dataclass @@ -157,6 +201,13 @@ class FactorBatchItem: n_dates: int = 0 elapsed_ms: float = 0.0 error: str | None = None + methodology_version: str = FACTOR_METHODOLOGY_VERSION + coverage: float | None = None + turnover: float | None = None + long_short_sharpe: float | None = None + yearly_ic: list[dict] = field(default_factory=list) + ic_decay: list[dict] = field(default_factory=list) + regime_stats: list[dict] = field(default_factory=list) @dataclass @@ -174,16 +225,56 @@ class FactorBacktestService: def __init__(self, engine: BacktestEngine) -> None: self.engine = engine - def run(self, config: FactorConfig) -> FactorResult: + def run( + self, + config: FactorConfig, + *, + regime_by_date: Mapping[object, Any] | None = None, + ) -> FactorResult: t0 = time.perf_counter() run_id = uuid.uuid4().hex[:10] - panel = self._load_factor_panel(config, [config.factor_name]) + generation = self._data_generation(config.asset_type) + panel = self._load_factor_panel( + config, + [config.factor_name], + expected_generation=generation, + ) if panel.is_empty(): return self._error_result(config, run_id, t0, "无数据, 请检查日期范围或先运行盘后管道") - return self._evaluate_panel(panel, config, run_id, t0) + if config.factor_name in FUNDAMENTAL_FACTOR_NAMES and self._fundamentals_missing(): + return self._error_result( + config, run_id, t0, + "本地没有财务数据: 请先在数据页同步财务数据后再使用财务因子", + ) - def run_batch(self, config: FactorBatchConfig) -> FactorBatchResult: + trading_dates = self._global_trading_dates(config) + self._assert_data_generation(config.asset_type, generation) + panel = self._attach_shared_next_return( + panel, + config, + trading_dates=trading_dates, + ) + evaluate_kwargs = ( + {"regime_by_date": regime_by_date} + if regime_by_date is not None + else {} + ) + return self._evaluate_panel( + panel, + config, + run_id, + t0, + market_trading_dates=trading_dates, + **evaluate_kwargs, + ) + + def run_batch( + self, + config: FactorBatchConfig, + *, + regime_by_date: Mapping[object, Any] | None = None, + ) -> FactorBatchResult: """在同一份 Panel 上依次评估多个因子, 避免重复读取和计算指标。""" t0 = time.perf_counter() run_id = uuid.uuid4().hex[:10] @@ -196,7 +287,12 @@ class FactorBacktestService: error="至少选择一个因子", ) - panel = self._load_factor_panel(config, factor_names) + generation = self._data_generation(config.asset_type) + panel = self._load_factor_panel( + config, + factor_names, + expected_generation=generation, + ) if panel.is_empty(): return FactorBatchResult( run_id=run_id, @@ -206,9 +302,19 @@ class FactorBacktestService: ) # P1: 预计算共享下期收益 (仅依赖 close/date/symbol), 避免每个因子重复 shift/调仓日 JOIN。 - panel = self._attach_shared_next_return(panel, config) + trading_dates = self._global_trading_dates(config) + self._assert_data_generation(config.asset_type, generation) + panel = self._attach_shared_next_return( + panel, + config, + trading_dates=trading_dates, + ) metadata = {item["id"]: item for item in FACTOR_COLUMNS} + fundamentals_missing = ( + any(name in FUNDAMENTAL_FACTOR_NAMES for name in factor_names) + and self._fundamentals_missing() + ) items: list[FactorBatchItem] = [] for factor_name in factor_names: item_t0 = time.perf_counter() @@ -221,16 +327,34 @@ class FactorBacktestService: rebalance=config.rebalance, weight=config.weight, fees_pct=config.fees_pct, + commission_pct=config.commission_pct, + stamp_tax_pct=config.stamp_tax_pct, slippage_bps=config.slippage_bps, asset_type=config.asset_type, ) meta = metadata.get(factor_name, {}) + if factor_name in FUNDAMENTAL_FACTOR_NAMES and fundamentals_missing: + items.append(FactorBatchItem( + factor_name=factor_name, + label=str(meta.get("label", factor_name)), + group=str(meta.get("group", "")), + elapsed_ms=round((time.perf_counter() - item_t0) * 1000, 1), + error="本地没有财务数据: 请先在数据页同步财务数据后再使用财务因子", + )) + continue try: + evaluate_kwargs = ( + {"regime_by_date": regime_by_date} + if regime_by_date is not None + else {} + ) result = self._evaluate_panel( panel, factor_config, f"{run_id}-{len(items) + 1}", item_t0, + market_trading_dates=trading_dates, + **evaluate_kwargs, ) long_short = result.long_short_stats items.append(FactorBatchItem( @@ -242,6 +366,13 @@ class FactorBacktestService: ic_win_rate=result.ic_win_rate, long_short_return=long_short.get("total_return"), long_short_max_drawdown=long_short.get("max_drawdown"), + methodology_version=result.methodology_version, + coverage=result.coverage, + turnover=result.turnover, + long_short_sharpe=result.long_short_sharpe, + yearly_ic=result.yearly_ic, + ic_decay=result.ic_decay, + regime_stats=result.regime_stats, n_symbols=result.n_symbols, n_dates=result.n_dates, elapsed_ms=result.elapsed_ms, @@ -268,26 +399,57 @@ class FactorBacktestService: n_dates=n_dates, ) + def _data_generation(self, asset_type: str) -> str | None: + loader = getattr(self.engine, "data_generation", None) + return loader(asset_type) if callable(loader) else None + + def _fundamentals_missing(self) -> bool: + return load_fundamental_snapshot(self._fundamentals_data_dir()) is None + + def _fundamentals_data_dir(self) -> Path | None: + repo = getattr(self.engine, "repo", None) + return getattr(getattr(repo, "store", None), "data_dir", None) + + def _assert_data_generation( + self, + asset_type: str, + expected: str | None, + ) -> None: + verifier = getattr(self.engine, "assert_data_generation", None) + if callable(verifier): + verifier(asset_type, expected) + def _load_factor_panel( self, config: FactorConfig | FactorBatchConfig, factor_names: list[str], + *, + expected_generation: str | None = None, ) -> pl.DataFrame: panel_columns = [ "symbol", "date", "open", "high", "low", "close", "volume", "amount", "turnover_rate", ] - panel_columns.extend(name for name in factor_names if name not in panel_columns) + if any( + name in ("limit_up_count_20d", "limit_up_count_60d") + for name in factor_names + ): + panel_columns.append("consecutive_limit_ups") load_start = config.start if any(name != "turnover_rate" for name in factor_names): load_start = config.start - timedelta(days=FACTOR_WARMUP_DAYS) + load_kwargs = { + "columns": panel_columns, + "asset_type": config.asset_type, + } + if expected_generation is not None: + load_kwargs["expected_generation"] = expected_generation panel = self.engine.load_panel( config.symbols, load_start, config.end, - columns=panel_columns, - asset_type=config.asset_type, + **load_kwargs, ) if panel.is_empty(): return panel @@ -295,37 +457,129 @@ class FactorBacktestService: missing = set(factor_names) - set(panel.columns) if missing: panel = self._compute_missing_factors(panel, missing) + fundamental_names = [name for name in factor_names if name in FUNDAMENTAL_FACTOR_NAMES] + if fundamental_names: + # 点时财务因子: 公告日门控, 无数据标的保持 null (不参与该日截面)。 + panel = attach_fundamental_factors( + panel, + load_fundamental_snapshot(self._fundamentals_data_dir()), + fundamental_names, + ) return panel + def _global_trading_dates( + self, + config: FactorConfig | FactorBatchConfig, + ) -> list[date] | None: + """Read the market date axis from enriched partitions, independent of symbols.""" + repo = getattr(self.engine, "repo", None) + data_dir = getattr(getattr(repo, "store", None), "data_dir", None) + if data_dir is None: + return None + from app.tickflow.repository import enriched_dirname + + values: list[date] = [] + root = data_dir / enriched_dirname(config.asset_type) + for partition in root.glob("date=*"): + try: + value = date.fromisoformat(partition.name.removeprefix("date=")) + except ValueError: + continue + if value <= config.end and (partition / "part.parquet").is_file(): + values.append(value) + ordered = sorted(set(values)) + formal = [value for value in ordered if value >= config.start] + predecessor = next( + (value for value in reversed(ordered) if value < config.start), + None, + ) + if predecessor is not None: + formal.insert(0, predecessor) + return formal or None + @staticmethod def _attach_shared_next_return( - panel: pl.DataFrame, config: FactorBatchConfig, + panel: pl.DataFrame, + config: FactorConfig | FactorBatchConfig, + *, + trading_dates: list[date] | None = None, ) -> pl.DataFrame: - """对 [start, end] 内 close 有效序列计算一次 _next_return, 复用给批次内每个因子。 - - _next_return 仅依赖 (symbol, date, close, rebalance), 与具体因子无关; 因子空值 - 集中在预热期前缀, 过滤后剩余序列无内部空洞, 故与 _evaluate_panel 内逐因子在 - 过滤后面板上重算的结果完全等价。 - """ - if "_next_return" in panel.columns: + """Prepare returns once on the complete price axis before factor filtering.""" + forward_columns = [f"_forward_return_{horizon}d" for horizon in _DAILY_FORWARD_HORIZONS] + prepared_columns = ["_next_return", *forward_columns] + if all(column in panel.columns for column in prepared_columns): return panel + existing_columns = [column for column in prepared_columns if column in panel.columns] + if existing_columns: + panel = panel.drop(existing_columns) + base = ( panel.filter((pl.col("date") >= config.start) & (pl.col("date") <= config.end)) .filter(pl.col("close").is_not_null() & (pl.col("close") > 0)) .select(["symbol", "date", "close"]) + .unique(subset=["symbol", "date"], keep="last") .sort(["symbol", "date"]) ) if base.is_empty(): - return panel.with_columns(pl.lit(None).cast(pl.Float64).alias("_next_return")) + return panel.with_columns( + [pl.lit(None).cast(pl.Float64).alias(column) for column in forward_columns] + + [pl.lit(None).cast(pl.Float64).alias("_next_return")] + ) + + all_dates = sorted( + value + for value in ( + trading_dates if trading_dates is not None else base["date"].unique().to_list() + ) + if config.start <= value <= config.end + ) + date_dtype = base.schema["date"] + for horizon, return_column in zip( + _DAILY_FORWARD_HORIZONS, + forward_columns, + strict=True, + ): + if len(all_dates) <= horizon: + base = base.with_columns( + pl.lit(None).cast(pl.Float64).alias(return_column) + ) + continue + target_column = f"_target_date_{horizon}d" + target_close_column = f"_target_close_{horizon}d" + date_map = pl.DataFrame({ + "date": all_dates[:-horizon], + target_column: all_dates[horizon:], + }).with_columns( + pl.col("date").cast(date_dtype), + pl.col(target_column).cast(date_dtype), + ) + price_lookup = base.select( + "symbol", + pl.col("date").alias(target_column), + pl.col("close").alias(target_close_column), + ) + base = ( + base.join(date_map, on="date", how="left") + .join(price_lookup, on=["symbol", target_column], how="left") + .with_columns( + pl.when(pl.col(target_close_column).is_not_null()) + .then(pl.col(target_close_column) / pl.col("close") - 1.0) + .otherwise(None) + .cast(pl.Float64) + .alias(return_column) + ) + .drop([target_column, target_close_column]) + ) + if config.rebalance == "daily": base = base.with_columns( - (pl.col("close").shift(-1).over("symbol") / pl.col("close") - 1) - .alias("_next_return") + pl.col("_forward_return_1d").alias("_next_return") ) else: base = FactorBacktestService._calc_period_return(base, config.rebalance) + return panel.join( - base.select(["symbol", "date", "_next_return"]), + base.select(["symbol", "date", "_next_return", *forward_columns]), on=["symbol", "date"], how="left", ) @@ -336,6 +590,9 @@ class FactorBacktestService: config: FactorConfig, run_id: str, t0: float, + *, + regime_by_date: Mapping[object, Any] | None = None, + market_trading_dates: list[date] | None = None, ) -> FactorResult: def _err(msg: str) -> FactorResult: return self._error_result(config, run_id, t0, msg) @@ -345,62 +602,67 @@ class FactorBacktestService: return _err(f"因子列 '{factor_col}' 不存在于 enriched 数据中, 且无法从基础行情计算") if "close" not in source_panel.columns: return _err("enriched 数据缺少收盘价 close") + if "_next_return" not in source_panel.columns: + source_panel = self._attach_shared_next_return(source_panel, config) - # 批量模式由 run_batch 预计算 _next_return 并随 source_panel 传入, 直接复用; - # 单因子 run() 路径未预计算, 仍按原逻辑在此计算。 - select_cols = ["symbol", "date", "close", factor_col] - precomputed_return = "_next_return" in source_panel.columns - if precomputed_return: - select_cols.append("_next_return") - panel = source_panel.select(select_cols) - panel = panel.filter((pl.col("date") >= config.start) & (pl.col("date") <= config.end)) - - # 过滤有效行 - panel = panel.filter( - pl.col(factor_col).is_not_null() - & pl.col("close").is_not_null() - & (pl.col("close") > 0) + return_columns = [ + column + for column in ( + "_next_return", + *(f"_forward_return_{horizon}d" for horizon in _DAILY_FORWARD_HORIZONS), + ) + if column in source_panel.columns + ] + price_panel = ( + source_panel.select(["symbol", "date", "close", factor_col, *return_columns]) + .filter((pl.col("date") >= config.start) & (pl.col("date") <= config.end)) + .filter(pl.col("close").is_not_null() & (pl.col("close") > 0)) + ) + total_price_rows = price_panel.height + panel = price_panel.filter( + pl.col(factor_col).is_not_null() & pl.col(factor_col).is_finite() ) if panel.is_empty(): return _err("过滤后无有效数据") panel = panel.sort(["symbol", "date"]) - + coverage = panel.height / total_price_rows if total_price_rows else None n_symbols = panel["symbol"].n_unique() n_dates = panel["date"].n_unique() - # 计算下期收益 — _next_return 仅依赖 (symbol, date, close, rebalance), 与因子无关; - # 因子空值集中在预热期前缀, 过滤后剩余序列无内部空洞, 故预计算与逐因子重算结果等价。 - if not precomputed_return: - if config.rebalance == "daily": - panel = panel.with_columns( - (pl.col("close").shift(-1).over("symbol") / pl.col("close") - 1) - .alias("_next_return") - ) - else: - # weekly/monthly: 计算到下个调仓日的收益 - panel = self._calc_period_return(panel, config.rebalance) - # ── 1. IC 分析 ── ic_df = self._calc_ic(panel, factor_col) + valid_ic_df = ic_df.filter(pl.col("ic").is_not_null() & pl.col("ic").is_finite()) + ic_rows = valid_ic_df.iter_rows(named=True) ic_series = [ {"date": str(row["date"]), "ic": round(float(row["ic"]), 4)} - for row in ic_df.iter_rows(named=True) - if row["ic"] is not None and not np.isnan(float(row["ic"])) + for row in ic_rows ] - ic_values = [r["ic"] for r in ic_series] - ic_mean = float(np.mean(ic_values)) if ic_values else None - ic_std = float(np.std(ic_values)) if ic_values else None + ic_values = valid_ic_df["ic"].to_numpy() if not valid_ic_df.is_empty() else np.array([]) + ic_mean = float(np.mean(ic_values)) if ic_values.size else None + ic_std = float(np.std(ic_values)) if ic_values.size else None ir = (ic_mean / ic_std) if (ic_mean is not None and ic_std and ic_std > 1e-8) else None - ic_win_rate = (sum(1 for v in ic_values if v > 0) / len(ic_values)) if ic_values else None + ic_win_rate = float(np.mean(ic_values > 0)) if ic_values.size else None + yearly_ic = self._calc_yearly_ic(valid_ic_df) + ic_decay = self._calc_ic_decay(panel, factor_col) + regime_stats = self._calc_regime_stats( + valid_ic_df, + price_panel, + regime_by_date, + config.start, + config.end, + market_trading_dates=market_trading_dates, + ) # ── 2. 分层回测 ── panel = self._add_groups(panel, factor_col, config.n_groups) group_nav = self._calc_group_nav(panel, config) group_stats = self._calc_group_stats(group_nav, config.start, config.end, config.rebalance) + turnover = self._calc_turnover(panel, config) - # ── 3. 多空组合 ── + # ── 3. 理论因子多空组合 ── long_short_nav, long_short_stats = self._calc_long_short(group_nav, config) + long_short_sharpe = long_short_stats.get("sharpe") elapsed = (time.perf_counter() - t0) * 1000 return FactorResult( @@ -412,16 +674,27 @@ class FactorBacktestService: ic_win_rate=round(ic_win_rate, 4) if ic_win_rate is not None else None, ic_series=ic_series, group_stats=group_stats, - group_nav=group_nav, + group_nav=self._round_nav(group_nav), long_short_stats=long_short_stats, long_short_nav=long_short_nav, + coverage=round(coverage, 4) if coverage is not None else None, + turnover=round(turnover, 4) if turnover is not None else None, + long_short_sharpe=long_short_sharpe, + yearly_ic=yearly_ic, + ic_decay=ic_decay, + regime_stats=regime_stats, elapsed_ms=round(elapsed, 1), n_symbols=n_symbols, n_dates=n_dates, ) @staticmethod - def _compute_missing_factors(panel: pl.DataFrame, factor_cols: set[str]) -> pl.DataFrame: + def _compute_missing_factors( + panel: pl.DataFrame, + factor_cols: set[str], + *, + assume_sorted: bool = False, + ) -> pl.DataFrame: required = {"symbol", "date", "open", "high", "low", "close", "volume"} if not required.issubset(panel.columns): missing = sorted(required - set(panel.columns)) @@ -434,7 +707,11 @@ class FactorBacktestService: indicator_columns = factor_cols - derived for factor_name in derived: indicator_columns.update(DERIVED_FACTOR_DEPENDENCIES[factor_name]) - panel = compute_indicators(panel, needed=indicator_columns) + panel = compute_indicators( + panel, + needed=indicator_columns, + assume_sorted=assume_sorted, + ) return FactorBacktestService._compute_derived_factors(panel, derived) @staticmethod @@ -472,6 +749,154 @@ class FactorBacktestService: .sort("date") ) + @staticmethod + def _calc_yearly_ic(ic_df: pl.DataFrame) -> list[dict]: + if ic_df.is_empty(): + return [] + yearly = ( + ic_df.with_columns(pl.col("date").dt.year().alias("_year")) + .group_by("_year") + .agg( + pl.col("ic").mean().alias("ic_mean"), + pl.col("ic").std(ddof=0).alias("ic_std"), + (pl.col("ic") > 0).mean().alias("win_rate"), + pl.len().alias("n_dates"), + ) + .sort("_year") + ) + result: list[dict] = [] + for row in yearly.iter_rows(named=True): + mean = float(row["ic_mean"]) + std = float(row["ic_std"] or 0.0) + result.append({ + "year": int(row["_year"]), + "ic_mean": round(mean, 4), + "ir": round(mean / std, 4) if std > 1e-8 else None, + "win_rate": round(float(row["win_rate"]), 4), + "n_dates": int(row["n_dates"]), + }) + return result + + @staticmethod + def _calc_ic_decay(panel: pl.DataFrame, factor_col: str) -> list[dict]: + columns = [ + (horizon, f"_forward_return_{horizon}d") + for horizon in _DAILY_FORWARD_HORIZONS + if f"_forward_return_{horizon}d" in panel.columns + ] + if not columns: + return [] + decay_df = ( + panel.group_by("date") + .agg([ + pl.corr( + pl.col(factor_col).rank(method="average"), + pl.col(return_column).rank(method="average"), + ).alias(f"ic_{horizon}d") + for horizon, return_column in columns + ]) + .sort("date") + ) + result: list[dict] = [] + for horizon, _ in columns: + column = f"ic_{horizon}d" + values = decay_df.filter( + pl.col(column).is_not_null() & pl.col(column).is_finite() + )[column] + result.append({ + "horizon": horizon, + "ic_mean": round(float(values.mean()), 4) if len(values) else None, + "n_dates": len(values), + }) + return result + + @staticmethod + def _calc_regime_stats( + ic_df: pl.DataFrame, + price_panel: pl.DataFrame, + regime_by_date: Mapping[object, Any] | None, + required_start: date, + required_end: date, + *, + market_trading_dates: list[date] | None = None, + ) -> list[dict]: + if not regime_by_date: + return [] + + from app.backtest.regime_alignment import ( + align_regime_t_minus_one, + three_level_regime, + ) + + formal_labels = tuple( + str(value)[:10] for value in sorted(price_panel["date"].unique().to_list()) + ) + if market_trading_dates: + labels = tuple( + str(value)[:10] + for value in market_trading_dates + if value <= required_end + ) + else: + predecessor = max( + ( + str(value)[:10] + for value in regime_by_date + if str(value)[:10] < str(required_start) + ), + default=None, + ) + labels = ( + (predecessor, *formal_labels) + if predecessor is not None + else formal_labels + ) + aligned = align_regime_t_minus_one( + labels, + regime_by_date, + required_start, + required_end, + ) + ic_by_date = { + str(row["date"])[:10]: float(row["ic"]) + for row in ic_df.iter_rows(named=True) + } + grouped: dict[str, dict[str, list[float]]] = {} + for label, point in zip(labels, aligned, strict=True): + if label not in formal_labels or point is None: + continue + state, score = point + bucket = grouped.setdefault( + three_level_regime(state), + {"scores": [], "ics": [], "dates": []}, + ) + bucket["scores"].append(score) + bucket["dates"].append(label) + if label in ic_by_date: + bucket["ics"].append(ic_by_date[label]) + + order = {"strong": 0, "range": 1, "weak": 2} + result: list[dict] = [] + for state in sorted(grouped, key=lambda value: (order.get(value, 99), value)): + values = np.asarray(grouped[state]["ics"], dtype=np.float64) + scores = np.asarray(grouped[state]["scores"], dtype=np.float64) + mean = float(np.mean(values)) if values.size else None + std = float(np.std(values)) if values.size else None + result.append({ + "state": state, + "ic_mean": round(mean, 4) if mean is not None else None, + "ir": ( + round(mean / std, 4) + if mean is not None and std is not None and std > 1e-8 + else None + ), + "win_rate": round(float(np.mean(values > 0)), 4) if values.size else None, + "mean_score": round(float(np.mean(scores)), 2), + "n_dates": len(grouped[state]["dates"]), + "n_ic_dates": int(values.size), + }) + return result + # ── 调仓期收益 ── @staticmethod @@ -482,20 +907,21 @@ class FactorBacktestService: monthly: 下个月调仓日 close / 今日 close - 1 只在调仓日标记行有效，其他行为 null。 """ - import datetime as _dt - all_dates = sorted(panel["date"].unique().to_list()) - if rebalance == "weekly": - # 调仓日 = 每周一 rebalance_dates = set() - for d in all_dates: - if hasattr(d, "weekday"): - wd = d.weekday() - else: - wd = _dt.date.fromisoformat(str(d)).weekday() - if wd == 0: # Monday - rebalance_dates.add(d) + seen_weeks: set[tuple[int, int]] = set() + for current_date in all_dates: + normalized_date = ( + current_date + if hasattr(current_date, "isocalendar") + else date.fromisoformat(str(current_date)[:10]) + ) + iso_year, iso_week, _ = normalized_date.isocalendar() + week = (iso_year, iso_week) + if week not in seen_weeks: + seen_weeks.add(week) + rebalance_dates.add(current_date) else: # monthly # 调仓日 = 每月首个交易日 seen_months: set[str] = set() @@ -569,18 +995,21 @@ class FactorBacktestService: @staticmethod def _add_groups(panel: pl.DataFrame, factor_col: str, n_groups: int) -> pl.DataFrame: - """截面序号分桶，避免 qcut 在重复因子值截面上抛错。""" + """Tie-aware cross-sectional buckets; equal factor values never split.""" return ( - panel.sort(["date", factor_col, "symbol"]) - .with_columns( - (pl.cum_count("symbol").over("date") - 1).alias("_factor_ord"), + panel.with_columns( + pl.col(factor_col).rank(method="average").over("date").alias("_factor_rank"), pl.len().over("date").alias("_factor_count"), ) .with_columns( ( pl.lit("Q") + ( - ((pl.col("_factor_ord") * n_groups) / pl.col("_factor_count")) + ( + (pl.col("_factor_rank") - 1.0) + * n_groups + / pl.col("_factor_count") + ) .floor() .cast(pl.Int64) + 1 @@ -588,9 +1017,16 @@ class FactorBacktestService: .clip(1, n_groups) .cast(pl.Utf8) ) - .alias("_group") + .alias("_group"), + ( + pl.col("_factor_rank") + - (pl.col("_factor_count") + 1) / 2.0 + ) + .abs() + .add(0.5) + .alias("_factor_strength"), ) - .drop(["_factor_ord", "_factor_count"]) + .drop(["_factor_rank", "_factor_count"]) ) @staticmethod @@ -604,38 +1040,126 @@ class FactorBacktestService: # ── 分组净值 ── + @staticmethod + def _round_nav(group_nav: list[dict]) -> list[dict]: + return [ + { + key: value if key == "date" else round(float(value), 4) + for key, value in row.items() + } + for row in group_nav + ] + + @staticmethod + def _round_trip_cost(config: FactorConfig) -> float: + commission = ( + config.commission_pct + if config.commission_pct is not None + else config.fees_pct + ) + stamp_tax = config.stamp_tax_pct or 0.0 + slippage = config.slippage_bps / 10_000.0 + return 2.0 * commission + stamp_tax + 2.0 * slippage + @staticmethod def _calc_group_nav(panel: pl.DataFrame, config: FactorConfig) -> list[dict]: - """计算分组净值曲线 — 只在调仓日更新净值。""" - # 只保留有下期收益的行 (= 调仓日) - group_ret = ( - panel.filter(pl.col("_next_return").is_not_null() & pl.col("_group").is_not_null()) - .group_by(["date", "_group"]) - .agg(pl.col("_next_return").mean().alias("group_return")) + """Calculate group NAV with the configured cross-sectional weighting.""" + eligible = panel.filter( + pl.col("_next_return").is_not_null() & pl.col("_group").is_not_null() + ) + if config.weight == "factor_weight": + group_ret = ( + eligible.with_columns( + pl.when(pl.col("_factor_strength") > 0) + .then(pl.col("_factor_strength")) + .otherwise(1.0) + .alias("_weight") + ) + .group_by(["date", "_group"]) + .agg( + ( + (pl.col("_next_return") * pl.col("_weight")).sum() + / pl.col("_weight").sum() + ).alias("group_return") + ) + ) + else: + group_ret = eligible.group_by(["date", "_group"]).agg( + pl.col("_next_return").mean().alias("group_return") + ) + group_ret = group_ret.with_columns( + (pl.col("group_return") - FactorBacktestService._round_trip_cost(config)) + .alias("group_return") ) - # pivot: date × group - pivot = group_ret.pivot(index="date", columns="_group", values="group_return").sort("date") - + pivot = group_ret.pivot( + index="date", + on="_group", + values="group_return", + ).sort("date") if pivot.is_empty(): return [] - group_cols = sorted([c for c in pivot.columns if c != "date"], key=FactorBacktestService._group_sort_key) - - # 向量化累乘净值: null 视为 0 收益 (净值不变); 累乘保持全精度, 输出时 round(4), - # 等价于原 dict 累乘 `nav_values[c] *= (1+ret); entry[c] = round(nav_values[c], 4)`。 + group_cols = sorted( + [column for column in pivot.columns if column != "date"], + key=FactorBacktestService._group_sort_key, + ) nav_df = pivot.with_columns( - [(1.0 + pl.col(c).fill_null(0.0)).cum_prod().alias(c) for c in group_cols] + [(1.0 + pl.col(column).fill_null(0.0)).cum_prod().alias(column) for column in group_cols] ) result: list[dict] = [] for row in nav_df.iter_rows(named=True): entry: dict = {"date": str(row["date"])[:10]} - for c in group_cols: - entry[c] = round(float(row[c]), 4) + for column in group_cols: + entry[column] = float(row[column]) result.append(entry) - return result + @staticmethod + def _calc_turnover(panel: pl.DataFrame, config: FactorConfig) -> float | None: + eligible = panel.filter( + pl.col("_next_return").is_not_null() & pl.col("_group").is_not_null() + ) + if eligible.is_empty(): + return None + groups = sorted( + eligible["_group"].unique().to_list(), + key=FactorBacktestService._group_sort_key, + ) + if not groups: + return None + top_group = groups[-1] + weights = eligible.filter(pl.col("_group") == top_group) + if config.weight == "factor_weight": + weights = weights.with_columns( + pl.when(pl.col("_factor_strength") > 0) + .then(pl.col("_factor_strength")) + .otherwise(1.0) + .alias("_raw_weight") + ) + else: + weights = weights.with_columns(pl.lit(1.0).alias("_raw_weight")) + weights = weights.with_columns( + (pl.col("_raw_weight") / pl.col("_raw_weight").sum().over("date")) + .alias("_weight") + ) + + by_date: dict[object, dict[str, float]] = {} + for row in weights.select(["date", "symbol", "_weight"]).iter_rows(named=True): + by_date.setdefault(row["date"], {})[str(row["symbol"])] = float(row["_weight"]) + ordered_dates = sorted(by_date) + if len(ordered_dates) < 2: + return 0.0 + turnovers: list[float] = [] + for previous_date, current_date in pairwise(ordered_dates): + previous = by_date[previous_date] + current = by_date[current_date] + symbols = previous.keys() | current.keys() + turnovers.append( + 0.5 * sum(abs(current.get(symbol, 0.0) - previous.get(symbol, 0.0)) for symbol in symbols) + ) + return float(np.mean(turnovers)) + # ── 分组统计 ── @staticmethod @@ -732,13 +1256,26 @@ class FactorBacktestService: with np.errstate(divide="ignore", invalid="ignore"): top_ret = np.where(prev_top > 0, top / prev_top - 1.0, 0.0) bot_ret = np.where(prev_bot > 0, bot / prev_bot - 1.0, 0.0) - ls_ret = (top_ret - bot_ret) / 2.0 # 各分配 50% 资金 + ls_ret = ( + (top_ret - bot_ret) / 2.0 + - FactorBacktestService._round_trip_cost(config) + ) # 50/50 理论多空两腿 ls_value = np.cumprod(1.0 + ls_ret) # 最大回撤: 峰值 = max(1.0, 历史最高), 与原 peak 初值 1.0 一致 peak = np.maximum(np.maximum.accumulate(ls_value), 1.0) max_dd = float(np.min((ls_value - peak) / peak)) + ann_factor = {"daily": 252, "weekly": 52, "monthly": 12}.get( + config.rebalance, + 252, + ) + ls_std = float(np.std(ls_ret)) + sharpe = ( + float(np.mean(ls_ret) / ls_std) * np.sqrt(ann_factor) + if ls_std > 1e-8 + else 0.0 + ) ls_nav = [ {"date": group_nav[k]["date"], "value": round(float(ls_value[k]), 4)} for k in range(len(group_nav)) @@ -746,8 +1283,11 @@ class FactorBacktestService: ls_stats = { "total_return": round(float(ls_value[-1]) - 1.0, 4), "max_drawdown": round(max_dd, 4), + "sharpe": round(sharpe, 4), "top_group": top_col, "bottom_group": bottom_col, + "portfolio_type": "theoretical_factor_spread", + "executable_short": False, } return ls_nav, ls_stats @@ -763,6 +1303,8 @@ class FactorBacktestService: "rebalance": c.rebalance, "weight": c.weight, "fees_pct": c.fees_pct, + "commission_pct": c.commission_pct, + "stamp_tax_pct": c.stamp_tax_pct, "slippage_bps": c.slippage_bps, "asset_type": c.asset_type, } @@ -778,6 +1320,8 @@ class FactorBacktestService: "rebalance": c.rebalance, "weight": c.weight, "fees_pct": c.fees_pct, + "commission_pct": c.commission_pct, + "stamp_tax_pct": c.stamp_tax_pct, "slippage_bps": c.slippage_bps, "asset_type": c.asset_type, } diff --git a/backend/app/backtest/fundamentals.py b/backend/app/backtest/fundamentals.py new file mode 100644 index 0000000..d69f52a --- /dev/null +++ b/backend/app/backtest/fundamentals.py @@ -0,0 +1,209 @@ +"""财务因子: 基于本地财务快照的点时 (point-in-time) 无未来函数接入。 + +数据契约: +- 输入为 data/financials/metrics/part.parquet, 每行一份报告期指标; +- ``announce_date`` 是公告日。因子只在 **严格晚于公告日的交易日** 才有值 + (公告多在盘后发布, 保守取 T+1 生效), 此前保持 null; +- 财报历史按 (symbol, period_end) 累积 (见 services/financial_sync.py), + 同一期以最新公告为准; +- 无财务数据的标的/日期一律为 null, 绝不填 0 (填 0 会污染截面排名, + 例如资产负债率 0 会被当成最优杠杆)。下游 IC/分层/评分对 null 自动剔除。 + +性能: +- 财务表约数千行, join_asof 按 symbol 分组回填, 对百万行面板的代价是 + 毫秒级; 矩阵路径每个因子只物化一张 float32 TxN 矩阵 (T~900, N~5500 + 约 20MB), 且仅在策略/挖掘请求该因子时才构建。 +""" +from __future__ import annotations + +import logging +from pathlib import Path +from types import MappingProxyType +from typing import Any + +import numpy as np +import polars as pl + +logger = logging.getLogger(__name__) + +# 财务因子名 -> (metrics 表列名, 是否需要除以收盘价) +# pb_latest 单列声明为 bps 倒数口径: 因子值 = close / bps。 +FUNDAMENTAL_FACTORS: dict[str, dict[str, Any]] = { + "pb_latest": {"column": "bps", "price_ratio": True}, + "roe_latest": {"column": "roe", "price_ratio": False}, + "gross_margin_latest": {"column": "gross_margin", "price_ratio": False}, + "net_margin_latest": {"column": "net_margin", "price_ratio": False}, + "revenue_yoy_latest": {"column": "revenue_yoy", "price_ratio": False}, + "net_income_yoy_latest": {"column": "net_income_yoy", "price_ratio": False}, + "debt_ratio_latest": {"column": "debt_to_asset_ratio", "price_ratio": False}, +} + +FUNDAMENTAL_FACTOR_NAMES = frozenset(FUNDAMENTAL_FACTORS) + + +def load_fundamental_snapshot(data_dir: Path | None) -> pl.DataFrame | None: + """读取财务指标快照; 文件缺失或无有效行时返回 None。 + + 返回列: symbol, _announce (Date), 以及各因子对应的 metrics 列。 + """ + if data_dir is None: + return None + path = data_dir / "financials" / "metrics" / "part.parquet" + if not path.exists(): + return None + try: + frame = pl.read_parquet(path) + except Exception as exc: + logger.warning("读取财务指标快照失败: %s", exc) + return None + needed = {"symbol", "announce_date"} | { + spec["column"] for spec in FUNDAMENTAL_FACTORS.values() + } + if not needed.issubset(frame.columns): + logger.warning("财务指标快照缺少列: %s", sorted(needed - set(frame.columns))) + return None + snapshot = ( + frame.select(sorted(needed)) + .filter( + pl.col("symbol").is_not_null() + & pl.col("announce_date").is_not_null() + ) + .with_columns( + pl.col("announce_date").cast(pl.Utf8).str.slice(0, 10).str.to_date().alias("_announce") + ) + .sort(["symbol", "_announce"]) + ) + if snapshot.is_empty(): + return None + return snapshot + + +def attach_fundamental_factors( + panel: pl.DataFrame, + snapshot: pl.DataFrame | None, + names: Any, +) -> pl.DataFrame: + """把财务因子列按公告日门控地并入日频面板。 + + - snapshot 为 None (本地无财务数据): 产出全 null 列, 保持面板形状, + 由上层决定是否报"无财务数据"错误; + - 面板必须已按 (symbol, date) 排序 (存储与挖掘路径均满足)。 + """ + requested = [str(name) for name in names if str(name) in FUNDAMENTAL_FACTOR_NAMES] + missing_columns = [name for name in requested if name not in panel.columns] + if not missing_columns: + return panel + + if snapshot is None: + return panel.with_columns([ + pl.lit(None, dtype=pl.Float64).alias(name) + for name in missing_columns + ]) + + columns = sorted( + {FUNDAMENTAL_FACTORS[name]["column"] for name in missing_columns} + ) + right = snapshot.select(["symbol", "_announce", *columns]).sort(["symbol", "_announce"]) + joined = panel.join_asof( + right, + left_on="date", + right_on="_announce", + by="symbol", + strategy="backward", + check_sortedness=False, # 双侧均已按 (symbol, key) 排序, 免除逐组检查开销 + ) + announced = pl.col("_announce").is_not_null() & (pl.col("date") > pl.col("_announce")) + expressions = [] + for name in missing_columns: + spec = FUNDAMENTAL_FACTORS[name] + source = pl.col(spec["column"]) + if spec["price_ratio"]: + value = ( + pl.when(source > 0) + .then(pl.col("close") / source) + .otherwise(None) + ) + else: + value = source + expressions.append( + pl.when(announced).then(value).otherwise(None).alias(name) + ) + return joined.with_columns(expressions) + + +def build_fundamental_matrices( + market: Any, + snapshot: pl.DataFrame | None, + names: Any, +) -> dict[str, np.ndarray]: + """为 MarketDataMatrix 构建财务因子 TxN float32 字段。 + + 与 attach_fundamental_factors 同一口径: 公告日次一交易日起前向填充, + 无数据为 NaN。pb 类因子在矩阵侧用 close / bps 现算。 + """ + requested = [str(name) for name in names if str(name) in FUNDAMENTAL_FACTOR_NAMES] + if not requested: + return {} + + shape = market.shape + result: dict[str, np.ndarray] = {} + if snapshot is None: + for name in requested: + result[name] = np.full(shape, np.nan, dtype=np.float32) + return result + + asset_index = {symbol: index for index, symbol in enumerate(market.symbols)} + labels = market.timestamp_labels + label_dates = np.array([label[:10] for label in labels], dtype="datetime64[D]") + + raw_columns = { + FUNDAMENTAL_FACTORS[name]["column"]: np.full(shape, np.nan, dtype=np.float32) + for name in requested + } + announce_text = snapshot["announce_date"].str.slice(0, 10) + for row_index, symbol in enumerate(snapshot["symbol"].to_list()): + column_index = asset_index.get(symbol) + if column_index is None: + continue + announce = announce_text[row_index] + if announce is None: + continue + # 公告日之后 (严格大于) 的首个时间行索引 + start = int(np.searchsorted(label_dates, np.datetime64(announce, "D"), side="right")) + if start >= shape[0]: + continue + for column, target in raw_columns.items(): + value = snapshot[column][row_index] + if value is None or not np.isfinite(float(value)): + continue + target[start:, column_index] = float(value) + + for name in requested: + spec = FUNDAMENTAL_FACTORS[name] + source = raw_columns[spec["column"]] + if spec["price_ratio"]: + with np.errstate(divide="ignore", invalid="ignore"): + matrix = (market.close / source).astype(np.float32) + matrix[~(source > 0)] = np.nan + matrix[np.isinf(matrix)] = np.nan + else: + matrix = source + result[name] = matrix + return result + + +def attach_matrix_fundamental_fields(market: Any, data_dir: Path | None, names: Any) -> Any: + """把财务因子作为 matrix fields 附加到 (frozen) MarketDataMatrix 副本。""" + import dataclasses + + requested = [str(name) for name in names if str(name) in FUNDAMENTAL_FACTOR_NAMES] + if not requested: + return market + snapshot = load_fundamental_snapshot(data_dir) + extra = build_fundamental_matrices(market, snapshot, requested) + if not extra: + return market + merged = {**dict(market.fields), **extra} + for array in extra.values(): + array.flags.writeable = False + return dataclasses.replace(market, fields=MappingProxyType(merged)) diff --git a/backend/app/backtest/matrix.py b/backend/app/backtest/matrix.py index 5b8625f..d9cfda8 100644 --- a/backend/app/backtest/matrix.py +++ b/backend/app/backtest/matrix.py @@ -59,6 +59,17 @@ _ROLLING_MATERIALIZED_WINDOW_BUDGET_BYTES = 32 * 1024 * 1024 _MATRIX_DISK_CACHE_DEFAULT_MAX_BYTES = 512 * 1024 * 1024 logger = logging.getLogger(__name__) + + +class MatrixPrewarmCancelledError(RuntimeError): + """A matrix cache prewarm was cancelled during application shutdown.""" + + +def _raise_if_matrix_cancelled(cancel_event: threading.Event | None) -> None: + if cancel_event is not None and cancel_event.is_set(): + raise MatrixPrewarmCancelledError("matrix cache prewarm cancelled") + + _MATRIX_DISK_CACHE_LOCK = threading.RLock() _MATRIX_DISK_CACHE_LEASES: dict[str, int] = {} _MATRIX_DISK_CACHE_PENDING_DELETE: set[str] = set() @@ -679,8 +690,10 @@ def load_market_data_matrix_from_parquet( cache_max_bytes: int = _MATRIX_DISK_CACHE_DEFAULT_MAX_BYTES, profile_generation: str = "default", source_generation: str | None = None, + cancel_event: threading.Event | None = None, ) -> MarketDataMatrix: """Load a daily market matrix, reusing a covering read-only mmap when possible.""" + _raise_if_matrix_cancelled(cancel_event) if start > end: raise ValueError("matrix parquet range start must not exceed end") root = Path(parquet_root) @@ -729,6 +742,7 @@ def load_market_data_matrix_from_parquet( instruments, batch_size=batch_size, cache_status="disabled", + cancel_event=cancel_event, ) cache_dir = Path(cache_root) @@ -807,7 +821,9 @@ def load_market_data_matrix_from_parquet( source_generation, batch_size=batch_size, axis_cache_root=cache_dir, + cancel_event=cancel_event, ) + _raise_if_matrix_cancelled(cancel_event) _prune_matrix_disk_cache( cache_dir, keep=cache_path, @@ -916,12 +932,15 @@ def _build_market_data_matrix_from_dataset( *, batch_size: int, cache_status: str, + cancel_event: threading.Event | None = None, ) -> MarketDataMatrix: + _raise_if_matrix_cancelled(cancel_event) filter_expr = _matrix_filter_expression(start, end, symbols) actual_dates, actual_symbols = _collect_parquet_axes( dataset, filter_expr, batch_size=batch_size, + cancel_event=cancel_event, ) if not actual_dates or not actual_symbols: raise ValueError("matrix parquet range contains no market data") @@ -953,7 +972,9 @@ def _build_market_data_matrix_from_dataset( parquet_fields, seen, batch_size=batch_size, + cancel_event=cancel_event, ) + _raise_if_matrix_cancelled(cancel_event) names, latest_limits = _populate_matrix_derived_arrays( actual_symbols, arrays, @@ -1037,7 +1058,9 @@ def _build_market_data_matrix_cache_from_dataset( *, batch_size: int, axis_cache_root: Path, + cancel_event: threading.Event | None = None, ) -> None: + _raise_if_matrix_cancelled(cancel_event) build_started = time.perf_counter() timing_ms: dict[str, float] = {} cache_path.parent.mkdir(parents=True, exist_ok=True) @@ -1057,7 +1080,9 @@ def _build_market_data_matrix_cache_from_dataset( filter_expr, batch_size=batch_size, cache_root=axis_cache_root, + cancel_event=cancel_event, ) + _raise_if_matrix_cancelled(cancel_event) if not actual_dates or not actual_symbols: raise ValueError("matrix parquet range contains no market data") timing_ms["axes"] = round((time.perf_counter() - stage_started) * 1000, 1) @@ -1101,7 +1126,9 @@ def _build_market_data_matrix_cache_from_dataset( parquet_fields, seen, batch_size=batch_size, + cancel_event=cancel_event, ) + _raise_if_matrix_cancelled(cancel_event) if not seen.any(): raise ValueError("matrix parquet range contains no requested market data") timing_ms["scan"] = round((time.perf_counter() - stage_started) * 1000, 1) @@ -1118,6 +1145,7 @@ def _build_market_data_matrix_cache_from_dataset( vector_fields=vector_fields, ) _mask_unseen_staging_fields(fields, seen) + _raise_if_matrix_cancelled(cancel_event) if "price_limit_pct" in fields: write_numpy_price_limit_matrix( fields["price_limit_pct"], @@ -1147,6 +1175,7 @@ def _build_market_data_matrix_cache_from_dataset( apply_latest_limits=actual_dates[-1] == _latest_partition_date(root), ) timing_ms["derived"] = round((time.perf_counter() - stage_started) * 1000, 1) + _raise_if_matrix_cancelled(cancel_event) stage_started = time.perf_counter() for values in mapped: values.flush() @@ -1181,6 +1210,7 @@ def _build_market_data_matrix_cache_from_dataset( json.dumps(manifest, ensure_ascii=False, separators=(",", ":")), encoding="utf-8", ) + _raise_if_matrix_cancelled(cancel_event) try: os.replace(temporary, cache_path) except OSError: @@ -1313,6 +1343,7 @@ def _scan_matrix_values( seen: np.ndarray, *, batch_size: int, + cancel_event: threading.Event | None = None, ) -> None: date_to_id = {value: index for index, value in enumerate(actual_dates)} symbol_to_id = {value: index for index, value in enumerate(actual_symbols)} @@ -1343,6 +1374,7 @@ def _scan_matrix_values( **{name: fields[name] for name in parquet_fields}, } for batch in scanner.to_batches(): + _raise_if_matrix_cancelled(cancel_event) time_ids = _arrow_axis_ids(_batch_column(batch, "date"), date_to_id) asset_ids = _arrow_axis_ids(_batch_column(batch, "symbol"), symbol_to_id) flat_ids = time_ids.astype(np.int64) * asset_count + asset_ids @@ -1898,7 +1930,9 @@ def _load_or_build_matrix_axes( *, batch_size: int, cache_root: Path, + cancel_event: threading.Event | None = None, ) -> tuple[list[date], list[str]]: + _raise_if_matrix_cancelled(cancel_event) path = _matrix_axis_cache_path(cache_root, parquet_root, start, end, symbols) previous: dict[str, Any] | None = None if path.exists(): @@ -1931,6 +1965,7 @@ def _load_or_build_matrix_axes( dataset, filter_expr, batch_size=batch_size, + cancel_event=cancel_event, ) changed_labels = set() retained_dates = {value.isoformat() for value in actual_dates} @@ -1956,6 +1991,7 @@ def _load_or_build_matrix_axes( ) symbols_set = set(actual_symbols) for batch in scanner.to_batches(): + _raise_if_matrix_cancelled(cancel_event) retained_dates.update( value.isoformat() for value in pc.unique(_batch_column(batch, "date")).to_pylist() @@ -1971,8 +2007,10 @@ def _load_or_build_matrix_axes( dataset, filter_expr, batch_size=batch_size, + cancel_event=cancel_event, ) + _raise_if_matrix_cancelled(cancel_event) payload = { "version": _MATRIX_AXIS_INDEX_VERSION, "source_partitions": dict(source_partitions), @@ -1993,6 +2031,7 @@ def _collect_parquet_axes( filter_expr, *, batch_size: int, + cancel_event: threading.Event | None = None, ) -> tuple[list[date], list[str]]: dates: set[date] = set() symbols: set[str] = set() @@ -2003,6 +2042,7 @@ def _collect_parquet_axes( use_threads=True, ) for batch in scanner.to_batches(): + _raise_if_matrix_cancelled(cancel_event) dates.update(pc.unique(_batch_column(batch, "date")).to_pylist()) symbols.update( str(value) @@ -2998,6 +3038,7 @@ _VALID_REDUCE_MIN = 0 _VALID_REDUCE_MAX = 1 _VALID_REDUCE_MEAN = 2 _VALID_REDUCE_STD = 3 +_VALID_REDUCE_SUM = 4 @njit(cache=True, nogil=True, parallel=True) @@ -3049,6 +3090,8 @@ def _valid_rolling_kernel( mean = total / window_value if operation == _VALID_REDUCE_MEAN: out[row, asset_id] = mean + elif operation == _VALID_REDUCE_SUM: + out[row, asset_id] = total else: squared = 0.0 for offset in range(window): @@ -3187,6 +3230,30 @@ def valid_rolling_std( ) +def valid_rolling_sum( + values: np.ndarray, + valid_mask: np.ndarray, + window: int, + *, + bar_index: ValidBarIndex | None = None, +) -> np.ndarray: + source = np.asarray(values, dtype=np.float32) + valid = np.asarray(valid_mask, dtype=bool) & np.isfinite(source) + index = _resolve_valid_bar_index(source, valid, bar_index) + return _cached_matrix_operation( + "valid_rolling_sum", + (source, valid, index.offsets, index.rows), + {"window": int(window)}, + lambda: _valid_rolling_reduce( + source, + valid, + window, + _VALID_REDUCE_SUM, + bar_index=index, + ), + ) + + def rolling_quantile(values: np.ndarray, window: int, quantile: float) -> np.ndarray: source = np.asarray(values, dtype=np.float32) q = float(quantile) @@ -3675,6 +3742,10 @@ _MATRIX_COMPUTED_FEATURES = frozenset({ "turnover_ratio_5d", "log_amount", "amount_ratio_5d", "gap_return", "intraday_return", "close_position", "distance_to_high_60d", "distance_from_low_60d", + "max_ret_20d", "ret_skew_20d", "up_days_20d", + "amihud_20d", "turnover_z_60d", "vol_price_corr_20d", + "vwap_bias", "vol_trend_5_60", + "limit_up_count_20d", "limit_up_count_60d", }) @@ -3872,9 +3943,106 @@ def _compute_matrix_feature(market: MarketDataMatrix, name: str) -> np.ndarray: return _matrix_relative(market.close, matrix_feature(market, "high_60d")) if name == "distance_from_low_60d": return _matrix_relative(market.close, matrix_feature(market, "low_60d")) + if name == "max_ret_20d": + daily = matrix_feature(market, "change_pct") + return valid_rolling_max(daily, np.isfinite(daily), 20) + if name == "ret_skew_20d": + return _matrix_rolling_skew(matrix_feature(market, "change_pct"), 20) + if name == "up_days_20d": + daily = matrix_feature(market, "change_pct") + up = np.where(daily > 0, np.float32(1.0), np.float32(0.0)).astype(np.float32) + up[~np.isfinite(daily)] = np.nan + return valid_rolling_sum(up, np.isfinite(up), 20) + if name == "amihud_20d": + daily = matrix_feature(market, "change_pct") + amount = market.field("amount") + amount_yi = amount / np.float32(1e8) + illiquidity = _matrix_ratio(np.abs(daily), amount_yi) + return valid_rolling_mean( + illiquidity, + close_valid & np.isfinite(illiquidity), + 20, + ) + if name == "turnover_z_60d": + turnover = market.field("turnover_rate") + valid = close_valid & np.isfinite(turnover) + previous = valid_shift(turnover, 1, valid) + baseline_valid = np.isfinite(previous) + mean = valid_rolling_mean(previous, baseline_valid, 60) + std = valid_rolling_std(previous, baseline_valid, 60, ddof=1) + deviation = _matrix_ratio(turnover - mean, std) + deviation[np.isfinite(std) & (std <= 0)] = np.nan + return deviation + if name == "vol_price_corr_20d": + daily = matrix_feature(market, "change_pct") + return _matrix_rolling_corr(daily, market.volume, close_valid, 20) + if name == "vwap_bias": + amount = market.field("amount") + shares = market.volume * np.float32(100.0) + valid = close_valid & np.isfinite(amount) & (market.volume > 0) & (amount > 0) + vwap = np.full(market.shape, np.nan, dtype=np.float32) + np.divide(amount, shares, out=vwap, where=valid) + return _matrix_relative(market.close, vwap) + if name == "vol_trend_5_60": + volume_valid = close_valid & np.isfinite(market.volume) + fast = valid_rolling_mean(market.volume, volume_valid, 5) + slow = valid_rolling_mean(market.volume, volume_valid, 60) + return _matrix_relative(fast, slow) + if name in {"limit_up_count_20d", "limit_up_count_60d"}: + window = 20 if name == "limit_up_count_20d" else 60 + consecutive = market.field("consecutive_limit_ups") + hits = np.where(np.isfinite(consecutive) & (consecutive > 0), np.float32(1.0), np.float32(0.0)) + hits = hits.astype(np.float32) + return valid_rolling_sum(hits, close_valid, window) raise ValueError(f"unsupported matrix feature: {name}") +def _matrix_rolling_skew(values: np.ndarray, window: int) -> np.ndarray: + valid = np.isfinite(values) + first = valid_rolling_mean(values, valid, window) + second = valid_rolling_mean(np.square(values, dtype=np.float32), valid, window) + third = valid_rolling_mean( + (values * values * values).astype(np.float32), valid, window + ) + variance = second - np.square(first, dtype=np.float32) + central_third = ( + third + - np.float32(3.0) * first * second + + np.float32(2.0) * np.power(first, 3) + ) + out = _matrix_ratio(central_third, np.sqrt(np.power(variance, 3))) + out[np.isfinite(variance) & (variance <= 0)] = np.nan + return out + + +def _matrix_rolling_corr( + left: np.ndarray, right: np.ndarray, valid_mask: np.ndarray, window: int +) -> np.ndarray: + valid = valid_mask & np.isfinite(left) & np.isfinite(right) + product = (left * right).astype(np.float32) + mean_left = valid_rolling_mean(left, valid, window) + mean_right = valid_rolling_mean(right, valid, window) + mean_product = valid_rolling_mean(product, valid, window) + mean_left_sq = valid_rolling_mean( + np.square(left, dtype=np.float32), valid, window + ) + mean_right_sq = valid_rolling_mean( + np.square(right, dtype=np.float32), valid, window + ) + covariance = mean_product - mean_left * mean_right + variance_left = mean_left_sq - np.square(mean_left, dtype=np.float32) + variance_right = mean_right_sq - np.square(mean_right, dtype=np.float32) + denominator = np.sqrt(variance_left * variance_right) + out = _matrix_ratio(covariance, denominator) + degenerate = ( + np.isfinite(variance_left) + & np.isfinite(variance_right) + & ((variance_left <= 0) | (variance_right <= 0)) + ) + out[degenerate] = np.nan + return out + + def _matrix_ema(values: np.ndarray, valid: np.ndarray, period: int) -> np.ndarray: return valid_ewm_adjust_false(values, valid, alpha=2.0 / (period + 1.0)) diff --git a/backend/app/backtest/mining.py b/backend/app/backtest/mining.py new file mode 100644 index 0000000..a4a23fb --- /dev/null +++ b/backend/app/backtest/mining.py @@ -0,0 +1,1660 @@ +"""Pure factor-mining algorithms and callback-driven nested validation.""" +from __future__ import annotations + +import json +import logging +import math +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, dataclass, field, replace +from datetime import date +from typing import Any, Literal, Protocol + +import numpy as np +import polars as pl + +logger = logging.getLogger(__name__) + +MAX_MINING_FACTORS = 48 +MAX_EXISTING_STRATEGIES = 8 +MAX_COMBINATION_SIZE = 4 +MAX_BEAM_WIDTH = 32 +MAX_FINALISTS = 8 +MAX_REAL_TRIALS = 256 + +# Evidence-based promotion gate documented in docs/mining.md. A candidate must +# clear every threshold before it may be published as an independent strategy. +GATE_MIN_VALID_FOLDS = 2 +GATE_MIN_POSITIVE_FOLD_RATIO = 2.0 / 3.0 +GATE_MIN_OOS_SHARPE = 0.5 +GATE_MAX_DRAWDOWN = -0.25 +GATE_MIN_TRADES = 60 + +MiningProfile = Literal["exploratory", "balanced", "strict"] + + +class JsonDataclassMixin: + """Provide JSON output without coupling pure models to Pydantic.""" + + def to_json(self) -> str: + return json.dumps(asdict(self), ensure_ascii=False, allow_nan=False) + + +@dataclass(frozen=True) +class MiningBudget(JsonDataclassMixin): + max_factors: int = MAX_MINING_FACTORS + max_existing_strategies: int = MAX_EXISTING_STRATEGIES + max_combination_size: int = MAX_COMBINATION_SIZE + beam_width: int = 16 + max_proxy_trials: int = 256 + max_trials: int = 96 + + def __post_init__(self) -> None: + if not 1 <= self.max_factors <= MAX_MINING_FACTORS: + raise ValueError(f"max_factors must be between 1 and {MAX_MINING_FACTORS}") + if not 0 <= self.max_existing_strategies <= MAX_EXISTING_STRATEGIES: + raise ValueError( + "max_existing_strategies must be between 0 and " + f"{MAX_EXISTING_STRATEGIES}" + ) + if not 1 <= self.max_combination_size <= MAX_COMBINATION_SIZE: + raise ValueError( + "max_combination_size must be between 1 and " + f"{MAX_COMBINATION_SIZE}" + ) + if not 1 <= self.beam_width <= MAX_BEAM_WIDTH: + raise ValueError(f"beam_width must be between 1 and {MAX_BEAM_WIDTH}") + if self.max_proxy_trials <= 0: + raise ValueError("max_proxy_trials must be positive") + if not 1 <= self.max_trials <= MAX_REAL_TRIALS: + raise ValueError(f"max_trials must be between 1 and {MAX_REAL_TRIALS}") + + @classmethod + def exploratory(cls) -> MiningBudget: + return cls( + max_combination_size=3, + beam_width=8, + max_proxy_trials=96, + max_trials=32, + ) + + @classmethod + def balanced(cls) -> MiningBudget: + return cls() + + @classmethod + def strict(cls) -> MiningBudget: + return cls(beam_width=32, max_proxy_trials=512, max_trials=256) + + +@dataclass(frozen=True) +class NestedValidationConfig(JsonDataclassMixin): + outer_train_bars: int = 504 + outer_test_bars: int = 126 + outer_step_bars: int = 63 + inner_train_bars: int = 252 + inner_test_bars: int = 63 + inner_step_bars: int = 63 + purge_bars: int = 30 + embargo_bars: int = 5 + min_train_bars: int = 126 + + def __post_init__(self) -> None: + positive = { + "outer_train_bars": self.outer_train_bars, + "outer_test_bars": self.outer_test_bars, + "outer_step_bars": self.outer_step_bars, + "inner_train_bars": self.inner_train_bars, + "inner_test_bars": self.inner_test_bars, + "inner_step_bars": self.inner_step_bars, + "min_train_bars": self.min_train_bars, + } + invalid = [name for name, value in positive.items() if value <= 0] + if invalid: + raise ValueError(f"nested validation bars must be positive: {invalid}") + if self.purge_bars < 0 or self.embargo_bars < 0: + raise ValueError("purge_bars and embargo_bars must not be negative") + if self.outer_train_bars < self.min_train_bars: + raise ValueError("outer_train_bars is smaller than min_train_bars") + if self.inner_train_bars < self.min_train_bars: + raise ValueError("inner_train_bars is smaller than min_train_bars") + + @classmethod + def exploratory(cls) -> NestedValidationConfig: + return cls( + outer_train_bars=126, + outer_test_bars=63, + outer_step_bars=63, + inner_train_bars=63, + inner_test_bars=21, + inner_step_bars=21, + purge_bars=30, + embargo_bars=5, + min_train_bars=63, + ) + + @classmethod + def balanced(cls) -> NestedValidationConfig: + return cls() + + @classmethod + def strict(cls) -> NestedValidationConfig: + return cls( + outer_train_bars=756, + outer_test_bars=126, + outer_step_bars=126, + inner_train_bars=504, + inner_test_bars=63, + inner_step_bars=63, + purge_bars=30, + embargo_bars=5, + min_train_bars=126, + ) + + +def validation_config_for_profile(profile: str) -> NestedValidationConfig: + if profile not in {"exploratory", "balanced", "strict"}: + raise ValueError(f"unknown mining profile: {profile}") + return getattr(NestedValidationConfig, profile)() + + +def required_outer_folds(profile: str) -> int: + validation_config_for_profile(profile) + return 1 if profile == "exploratory" else 3 + + +def required_trading_bars( + config: NestedValidationConfig, + outer_folds: int, +) -> int: + if outer_folds <= 0: + raise ValueError("outer_folds must be positive") + return ( + config.outer_train_bars + + config.purge_bars + + config.outer_test_bars + + (outer_folds - 1) * config.outer_step_bars + ) + + +def nested_fold_count( + trading_bars: int, + config: NestedValidationConfig, +) -> int: + if trading_bars < 0: + raise ValueError("trading_bars must not be negative") + one_fold_bars = required_trading_bars(config, 1) + if trading_bars < one_fold_bars: + return 0 + return 1 + (trading_bars - one_fold_bars) // config.outer_step_bars + + +@dataclass(frozen=True) +class CandidateGateResult(JsonDataclassMixin): + qualified: bool + reasons: tuple[str, ...] + + +def evaluate_candidate_gate( + *, + confidence: str | None, + valid_folds: int | None, + positive_fold_ratio: float | None, + sharpe: float | None, + max_drawdown: float | None, + n_trades: int | None, +) -> CandidateGateResult: + """Check the documented promotion thresholds against real candidate evidence.""" + reasons: list[str] = [] + if confidence == "low": + reasons.append("exploratory results can only be saved as pending candidates") + if valid_folds is None or valid_folds < GATE_MIN_VALID_FOLDS: + reasons.append( + "requires at least " + f"{GATE_MIN_VALID_FOLDS} valid outer folds (got " + f"{'none' if valid_folds is None else valid_folds})" + ) + if positive_fold_ratio is None or positive_fold_ratio < GATE_MIN_POSITIVE_FOLD_RATIO: + reasons.append( + "requires a positive-return fold ratio of at least " + f"{GATE_MIN_POSITIVE_FOLD_RATIO:.2f}" + ) + if sharpe is None or sharpe < GATE_MIN_OOS_SHARPE: + reasons.append(f"requires an OOS Sharpe of at least {GATE_MIN_OOS_SHARPE}") + if max_drawdown is None or max_drawdown < GATE_MAX_DRAWDOWN: + reasons.append( + f"requires a max drawdown no worse than {abs(GATE_MAX_DRAWDOWN):.0%}" + ) + if n_trades is None or n_trades < GATE_MIN_TRADES: + reasons.append(f"requires at least {GATE_MIN_TRADES} OOS trades") + return CandidateGateResult(qualified=not reasons, reasons=tuple(reasons)) + + +@dataclass(frozen=True) +class MiningRequest(JsonDataclassMixin): + factor_names: tuple[str, ...] + existing_strategy_ids: tuple[str, ...] = () + correlation_threshold: float = 0.8 + date_column: str = "date" + target_column: str = "_next_return" + budget: MiningBudget = field(default_factory=MiningBudget.balanced) + validation: NestedValidationConfig = field( + default_factory=NestedValidationConfig.balanced + ) + profile: Literal["exploratory", "balanced", "strict"] = "balanced" + + def __post_init__(self) -> None: + if not self.factor_names: + raise ValueError("factor_names must not be empty") + if len(set(self.factor_names)) != len(self.factor_names): + raise ValueError("factor_names must not contain duplicates") + if len(self.factor_names) > self.budget.max_factors: + raise ValueError( + f"factor count {len(self.factor_names)} exceeds budget " + f"{self.budget.max_factors}" + ) + if len(set(self.existing_strategy_ids)) != len(self.existing_strategy_ids): + raise ValueError("existing_strategy_ids must not contain duplicates") + if len(self.existing_strategy_ids) > self.budget.max_existing_strategies: + raise ValueError( + f"existing strategy count {len(self.existing_strategy_ids)} exceeds " + f"budget {self.budget.max_existing_strategies}" + ) + if not 0.0 < self.correlation_threshold <= 1.0: + raise ValueError("correlation_threshold must be in (0, 1]") + if not self.date_column or not self.target_column: + raise ValueError("date_column and target_column must not be empty") + + @classmethod + def for_profile( + cls, + profile: Literal["exploratory", "balanced", "strict"], + factor_names: Sequence[str], + existing_strategy_ids: Sequence[str] = (), + **kwargs: Any, + ) -> MiningRequest: + if profile not in {"exploratory", "balanced", "strict"}: + raise ValueError(f"unknown mining profile: {profile}") + return cls( + factor_names=tuple(factor_names), + existing_strategy_ids=tuple(existing_strategy_ids), + budget=getattr(MiningBudget, profile)(), + validation=getattr(NestedValidationConfig, profile)(), + profile=profile, + **kwargs, + ) + + +@dataclass(frozen=True) +class FactorMetric(JsonDataclassMixin): + factor_id: str + composite_score: float + ir: float + coverage: float + turnover: float + rank_ic: float = 0.0 + + +@dataclass(frozen=True) +class CorrelationResult(JsonDataclassMixin): + factor_names: tuple[str, ...] + matrix: tuple[tuple[float, ...], ...] + pair_counts: tuple[tuple[int, ...], ...] + elapsed_ms: float + n_dates: int + n_rows: int + timing_ms: dict[str, float] = field(default_factory=dict) + + +@dataclass(frozen=True) +class FactorExclusion(JsonDataclassMixin): + factor_id: str + reason: str + representative: str + rho: float + + +@dataclass(frozen=True) +class PruneResult(JsonDataclassMixin): + selected: tuple[str, ...] + excluded: tuple[FactorExclusion, ...] + + +@dataclass(frozen=True) +class MiningCandidate(JsonDataclassMixin): + candidate_id: str + kind: Literal["factor_rank", "existing_strategy"] + factor_names: tuple[str, ...] = () + weights: tuple[float, ...] = () + directions: tuple[int, ...] = () + strategy_id: str | None = None + proxy_rank_ic: float = 0.0 + proxy_ir: float = 0.0 + observations: int = 0 + dates: int = 0 + + def definition(self) -> dict[str, Any]: + if self.kind == "existing_strategy": + return {"kind": self.kind, "strategy_id": self.strategy_id} + return { + "kind": self.kind, + "factor_names": list(self.factor_names), + "scoring": dict(zip(self.factor_names, self.weights, strict=True)), + "directions": { + factor_id: "high" if direction > 0 else "low" + for factor_id, direction in zip( + self.factor_names, + self.directions, + strict=True, + ) + }, + } + + +@dataclass(frozen=True) +class BeamSearchResult(JsonDataclassMixin): + candidates: tuple[MiningCandidate, ...] + trials_used: int + cancelled: bool + budget_exhausted: bool + elapsed_ms: float + + +@dataclass(frozen=True) +class ValidationFold(JsonDataclassMixin): + level: Literal["outer", "inner"] + outer_index: int + inner_index: int | None + train_labels: tuple[str, ...] + purge_labels: tuple[str, ...] + test_labels: tuple[str, ...] + embargo_labels: tuple[str, ...] + + @property + def train_start(self) -> str: + return self.train_labels[0] + + @property + def train_end(self) -> str: + return self.train_labels[-1] + + @property + def test_start(self) -> str: + return self.test_labels[0] + + @property + def test_end(self) -> str: + return self.test_labels[-1] + + +@dataclass(frozen=True) +class NestedFold(JsonDataclassMixin): + outer: ValidationFold + inner: tuple[ValidationFold, ...] + + +@dataclass(frozen=True) +class CandidateEvaluation(JsonDataclassMixin): + score: float | None + metrics: dict[str, Any] = field(default_factory=dict) + error: str | None = None + + +@dataclass(frozen=True) +class FoldMiningResult(JsonDataclassMixin): + outer_index: int + selected_factors: tuple[str, ...] + candidates: tuple[MiningCandidate, ...] + selected_candidate_id: str | None + inner_score: float | None + outer_evaluation: CandidateEvaluation | None + error: str | None = None + benchmark_evaluations: tuple[tuple[str, CandidateEvaluation], ...] = () + cross_evaluations: tuple[tuple[str, CandidateEvaluation], ...] = () + + +@dataclass(frozen=True) +class MiningResult(JsonDataclassMixin): + request: MiningRequest + folds: tuple[FoldMiningResult, ...] + proxy_trials_used: int + trials_used: int + cancelled: bool + elapsed_ms: float + + +class CandidateEvaluator(Protocol): + def evaluate_candidate( + self, + train: pl.DataFrame, + test: pl.DataFrame, + definition: Mapping[str, Any], + ) -> CandidateEvaluation | Mapping[str, Any] | float: ... + + +class FactorMetricProvider(Protocol): + def __call__( + self, + train: pl.DataFrame, + factor_names: Sequence[str], + ) -> Sequence[FactorMetric]: ... + + +CancelCheck = Callable[[], bool] | Any + + +def compute_rank_correlation( + panel: pl.DataFrame, + factor_names: Sequence[str], + start: Any | None = None, + end: Any | None = None, + *, + date_column: str = "date", +) -> CorrelationResult: + """Average pairwise daily cross-sectional rank correlations. + + One date partition is ranked and materialized at a time. Only the factor-by-factor + correlation sums and valid-day counts remain resident across dates. + """ + started = time.perf_counter() + names = _validate_factor_names(panel, factor_names) + if date_column not in panel.columns: + raise ValueError(f"panel is missing date column {date_column!r}") + + filter_started = time.perf_counter() + date_expr = pl.col(date_column).cast(pl.Utf8).str.slice(0, 10) + scoped = panel.select([ + pl.col(date_column), + *( + pl.when(pl.col(name).is_finite()) + .then(pl.col(name)) + .otherwise(None) + .alias(name) + for name in names + ), + ]) + if start is not None: + scoped = scoped.filter(date_expr >= str(start)[:10]) + if end is not None: + scoped = scoped.filter(date_expr <= str(end)[:10]) + if scoped.is_empty(): + raise ValueError("rank correlation date range contains no panel rows") + filter_ms = (time.perf_counter() - filter_started) * 1000.0 + + width = len(names) + pair_columns: dict[tuple[int, int], str] = {} + pair_expressions: list[pl.Expr] = [] + for left in range(width): + for right in range(left + 1, width): + column = f"_rho_{left}_{right}" + pair_columns[(left, right)] = column + pair_expressions.append( + pl.corr(names[left], names[right], method="spearman").alias(column) + ) + observation_columns = [f"_n_{index}" for index in range(width)] + + rank_started = time.perf_counter() + daily = scoped.group_by(date_column).agg([ + *pair_expressions, + *( + pl.col(name).count().alias(column) + for name, column in zip(names, observation_columns, strict=True) + ), + ]) + rank_ms = (time.perf_counter() - rank_started) * 1000.0 + + finish_started = time.perf_counter() + correlation = np.full((width, width), np.nan, dtype=np.float64) + counts = np.zeros((width, width), dtype=np.int32) + for (left, right), column in pair_columns.items(): + values = daily.get_column(column).to_numpy() + finite = np.isfinite(values) + count = int(np.count_nonzero(finite)) + if count: + value = float(np.mean(values[finite])) + correlation[left, right] = correlation[right, left] = value + counts[left, right] = counts[right, left] = count + for index, column in enumerate(observation_columns): + count = int(np.count_nonzero(daily.get_column(column).to_numpy() >= 2)) + if count: + correlation[index, index] = 1.0 + counts[index, index] = count + finish_ms = (time.perf_counter() - finish_started) * 1000.0 + n_dates = daily.height + elapsed_ms = (time.perf_counter() - started) * 1000.0 + timing = { + "filter": round(filter_ms, 3), + "rank_accumulate": round(rank_ms, 3), + "finalize": round(finish_ms, 3), + "total": round(elapsed_ms, 3), + } + logger.info( + "mining rank correlation factors=%d rows=%d dates=%d elapsed_ms=%.1f", + width, + scoped.height, + n_dates, + elapsed_ms, + ) + return CorrelationResult( + factor_names=names, + matrix=tuple(tuple(float(value) for value in row) for row in correlation), + pair_counts=tuple( + tuple(int(value) for value in row) + for row in counts + ), + elapsed_ms=round(elapsed_ms, 3), + n_dates=n_dates, + n_rows=scoped.height, + timing_ms=timing, + ) + + +def prune_correlated_factors( + metrics: Sequence[FactorMetric], + correlation: CorrelationResult, + threshold: float, +) -> PruneResult: + """Keep the strongest deterministic representative of correlated factors.""" + if not 0.0 < threshold <= 1.0: + raise ValueError("correlation threshold must be in (0, 1]") + by_name = {name: index for index, name in enumerate(correlation.factor_names)} + if len({metric.factor_id for metric in metrics}) != len(metrics): + raise ValueError("factor metrics must not contain duplicate factor_id values") + missing = sorted(metric.factor_id for metric in metrics if metric.factor_id not in by_name) + if missing: + raise ValueError(f"factor metrics missing from correlation result: {missing}") + + ordered = sorted( + metrics, + key=lambda metric: ( + -_finite_sort_value(metric.composite_score), + -_finite_sort_value(metric.ir), + -_finite_sort_value(metric.coverage), + _finite_sort_value(metric.turnover, worst=float("inf")), + metric.factor_id, + ), + ) + selected: list[str] = [] + excluded: list[FactorExclusion] = [] + for metric in ordered: + factor_index = by_name[metric.factor_id] + correlated = [ + (representative, correlation.matrix[factor_index][by_name[representative]]) + for representative in selected + if ( + correlation.pair_counts[factor_index][by_name[representative]] > 0 + and math.isfinite( + correlation.matrix[factor_index][by_name[representative]] + ) + and abs( + correlation.matrix[factor_index][by_name[representative]] + ) >= threshold + ) + ] + if not correlated: + selected.append(metric.factor_id) + continue + representative, rho = max( + correlated, + key=lambda item: (abs(item[1]), -selected.index(item[0])), + ) + excluded.append(FactorExclusion( + factor_id=metric.factor_id, + reason="correlation_threshold", + representative=representative, + rho=round(float(rho), 8), + )) + return PruneResult(selected=tuple(selected), excluded=tuple(excluded)) + + +def beam_search_factor_combinations( + panel: pl.DataFrame, + factor_names: Sequence[str], + *, + target_column: str = "_next_return", + date_column: str = "date", + train_start: Any | None = None, + train_end: Any | None = None, + max_combination_size: int = MAX_COMBINATION_SIZE, + beam_width: int = 16, + max_trials: int = 256, + cancel_check: CancelCheck | None = None, +) -> BeamSearchResult: + """Search equal and one-factor-double rank combinations on training data only.""" + started = time.perf_counter() + names = _validate_factor_names(panel, factor_names) + if target_column not in panel.columns: + raise ValueError(f"panel is missing target column {target_column!r}") + if date_column not in panel.columns: + raise ValueError(f"panel is missing date column {date_column!r}") + if not 1 <= max_combination_size <= MAX_COMBINATION_SIZE: + raise ValueError( + f"max_combination_size must be between 1 and {MAX_COMBINATION_SIZE}" + ) + if not 1 <= beam_width <= MAX_BEAM_WIDTH: + raise ValueError(f"beam_width must be between 1 and {MAX_BEAM_WIDTH}") + if max_trials <= 0: + raise ValueError("max_trials must be positive") + + date_expr = pl.col(date_column).cast(pl.Utf8).str.slice(0, 10) + scoped = panel.select([date_column, *names, target_column]) + if train_start is not None: + scoped = scoped.filter(date_expr >= str(train_start)[:10]) + if train_end is not None: + scoped = scoped.filter(date_expr <= str(train_end)[:10]) + if scoped.is_empty(): + raise ValueError("beam search training range contains no panel rows") + + ordered_names = tuple(sorted(names)) + name_to_column = {name: index for index, name in enumerate(ordered_names)} + if not scoped.get_column(date_column).is_sorted(): + scoped = scoped.sort(date_column) + blocks = tuple( + daily.select([*ordered_names, target_column]).to_numpy() + for daily in scoped.partition_by(date_column, maintain_order=True) + ) + trials_used = 0 + cancelled = False + + def evaluate( + factors: tuple[str, ...], + weights: tuple[float, ...], + directions: tuple[int, ...], + ) -> tuple[float, float, int, int] | None: + nonlocal trials_used, cancelled + if trials_used >= max_trials: + return None + if _cancelled(cancel_check): + cancelled = True + return None + trials_used += 1 + columns = tuple(name_to_column[name] for name in factors) + return _proxy_rank_ic(blocks, columns, weights, directions) + + directions_by_factor: dict[str, int] = {} + single_candidates: list[MiningCandidate] = [] + for factor_name in ordered_names: + proxy = evaluate((factor_name,), (1.0,), (1,)) + if proxy is None: + break + rank_ic, proxy_ir, observations, n_dates = proxy + if n_dates <= 0 or observations < 3: + continue + direction = 1 if rank_ic >= 0.0 else -1 + directions_by_factor[factor_name] = direction + single_candidates.append(_factor_candidate( + (factor_name,), + (1.0,), + (direction,), + abs(rank_ic), + abs(proxy_ir), + observations, + n_dates, + )) + + current_beam = _rank_candidates(single_candidates)[:beam_width] + retained = list(current_beam) + searchable_names = tuple(sorted(directions_by_factor)) + max_size = min(max_combination_size, len(searchable_names)) + for size in range(2, max_size + 1): + if cancelled or trials_used >= max_trials or not current_beam: + break + factor_sets: set[tuple[str, ...]] = set() + for candidate in current_beam: + for factor_name in searchable_names: + combined = tuple(sorted({*candidate.factor_names, factor_name})) + if len(combined) == size: + factor_sets.add(combined) + + expanded: list[MiningCandidate] = [] + stop = False + for factors in sorted(factor_sets): + directions = tuple(directions_by_factor[name] for name in factors) + patterns = [(1.0,) * size] + patterns.extend( + tuple(2.0 if index == doubled else 1.0 for index in range(size)) + for doubled in range(size) + ) + for weights in patterns: + proxy = evaluate(factors, weights, directions) + if proxy is None: + stop = True + break + rank_ic, proxy_ir, observations, n_dates = proxy + if n_dates <= 0 or observations < 3: + continue + expanded.append(_factor_candidate( + factors, + weights, + directions, + rank_ic, + proxy_ir, + observations, + n_dates, + )) + if stop: + break + current_beam = _rank_candidates(expanded)[:beam_width] + retained.extend(current_beam) + if stop: + break + + elapsed_ms = (time.perf_counter() - started) * 1000.0 + return BeamSearchResult( + candidates=tuple(_rank_candidates(retained)), + trials_used=trials_used, + cancelled=cancelled, + budget_exhausted=trials_used >= max_trials, + elapsed_ms=round(elapsed_ms, 3), + ) + + +def generate_nested_folds( + trading_labels: Sequence[Any], + config: NestedValidationConfig, +) -> tuple[NestedFold, ...]: + """Generate rolling nested folds from ordered trading labels, not calendar days.""" + labels = tuple(dict.fromkeys(str(label) for label in trading_labels)) + if not labels: + raise ValueError("trading labels are empty") + if tuple(sorted(labels)) != labels: + raise ValueError("trading labels must be sorted in ascending order") + outer_required = ( + config.outer_train_bars + config.purge_bars + config.outer_test_bars + ) + if len(labels) < outer_required: + raise ValueError( + "insufficient trading bars for outer validation: " + f"need at least {outer_required}, got {len(labels)}" + ) + inner_required = ( + config.inner_train_bars + config.purge_bars + config.inner_test_bars + ) + if config.outer_train_bars < inner_required: + raise ValueError( + "outer training window is too short for one inner fold: " + f"need at least {inner_required}, got {config.outer_train_bars}" + ) + + nested: list[NestedFold] = [] + outer_start = 0 + outer_index = 0 + while outer_start + outer_required <= len(labels): + outer = _make_validation_fold( + labels, + level="outer", + outer_index=outer_index, + inner_index=None, + train_start=outer_start, + train_bars=config.outer_train_bars, + test_bars=config.outer_test_bars, + purge_bars=config.purge_bars, + embargo_bars=config.embargo_bars, + ) + inner_folds: list[ValidationFold] = [] + inner_start = outer_start + inner_index = 0 + outer_train_stop = outer_start + config.outer_train_bars + while inner_start + inner_required <= outer_train_stop: + inner_folds.append(_make_validation_fold( + labels, + level="inner", + outer_index=outer_index, + inner_index=inner_index, + train_start=inner_start, + train_bars=config.inner_train_bars, + test_bars=config.inner_test_bars, + purge_bars=config.purge_bars, + embargo_bars=config.embargo_bars, + hard_stop=outer_train_stop, + )) + inner_index += 1 + inner_start += config.inner_step_bars + if not inner_folds: + raise ValueError(f"outer fold {outer_index} contains no valid inner fold") + nested.append(NestedFold(outer=outer, inner=tuple(inner_folds))) + outer_index += 1 + outer_start += config.outer_step_bars + return tuple(nested) + + +class MiningService: + """Run leakage-bounded mining and delegate real backtests to a callback.""" + + def run( + self, + panel: pl.DataFrame, + request: MiningRequest, + *, + factor_metrics: Sequence[FactorMetric] | None = None, + metric_provider: FactorMetricProvider | None = None, + evaluator: CandidateEvaluator | None = None, + cancel_check: CancelCheck | None = None, + ) -> MiningResult: + started = time.perf_counter() + _validate_factor_names(panel, request.factor_names) + required = {request.date_column, request.target_column} + missing = sorted(required - set(panel.columns)) + if missing: + raise ValueError(f"mining panel is missing required columns: {missing}") + if evaluator is not None: + if factor_metrics is not None: + raise ValueError("factor_metrics must not be supplied with evaluator") + if metric_provider is None: + raise ValueError("metric_provider is required when evaluator is supplied") + elif metric_provider is None and factor_metrics is None: + raise ValueError("factor_metrics or metric_provider is required") + + labels = panel.select( + pl.col(request.date_column).cast(pl.Utf8).str.slice(0, 10).unique().sort() + ).to_series().to_list() + nested_folds = generate_nested_folds(labels, request.validation) + if request.profile != "exploratory" and len(nested_folds) < 3: + raise ValueError( + f"{request.profile} mining requires at least 3 outer folds; " + f"got {len(nested_folds)}" + ) + fold_results: list[FoldMiningResult] = [] + proxy_trials_used = 0 + trials_used = 0 + cancelled = False + labels_evaluator = ( + getattr(evaluator, "evaluate_candidate_labels", None) + if evaluator is not None + else None + ) + + def run_benchmarks(nested_fold) -> tuple[tuple[str, CandidateEvaluation], ...]: + """Score every user-selected strategy on one outer test window. + + Benchmarks document how existing strategies would have done on the + same walk-forward windows, so they are evaluated even when the + factor track fails on this fold and never join the winner race. + """ + nonlocal trials_used, cancelled + if evaluator is None: + return () + benchmarks: list[tuple[str, CandidateEvaluation]] = [] + for strategy_id in request.existing_strategy_ids: + if _cancelled(cancel_check): + cancelled = True + break + if trials_used >= request.budget.max_trials: + benchmarks.append(( + _benchmark_signature(strategy_id), + CandidateEvaluation( + score=None, + error="real trial budget exhausted before benchmark evaluation", + ), + )) + continue + benchmark = benchmark_candidate(strategy_id) + if callable(labels_evaluator): + benchmark_evaluation = _evaluate_labels( + labels_evaluator, + nested_fold.outer.train_labels, + nested_fold.outer.test_labels, + benchmark, + ) + else: + benchmark_evaluation = _evaluate( + evaluator, + _train_panel_for_fold(panel, request.date_column, nested_fold.outer), + _panel_for_labels(panel, request.date_column, nested_fold.outer.test_labels), + benchmark, + ) + trials_used += 1 + benchmarks.append((benchmark.candidate_id, benchmark_evaluation)) + return tuple(benchmarks) + + if evaluator is None: + selection_phases_remaining = len(nested_folds) + evaluation_phases_remaining = 0 + else: + selection_phases_remaining = sum( + len(nested.inner) + 1 for nested in nested_folds + ) + evaluation_phases_remaining = selection_phases_remaining + + for nested in nested_folds: + if _cancelled(cancel_check): + cancelled = True + break + + if evaluator is None: + outer_train = _train_panel_for_fold( + panel, + request.date_column, + nested.outer, + ) + metrics = tuple( + metric_provider(outer_train, request.factor_names) + if metric_provider is not None + else factor_metrics or () + ) + correlation = compute_rank_correlation( + outer_train, + request.factor_names, + date_column=request.date_column, + ) + pruned = prune_correlated_factors( + metrics, + correlation, + request.correlation_threshold, + ) + proxy_remaining = request.budget.max_proxy_trials - proxy_trials_used + if proxy_remaining <= 0: + fold_results.append(FoldMiningResult( + outer_index=nested.outer.outer_index, + selected_factors=pruned.selected, + candidates=(), + selected_candidate_id=None, + inner_score=None, + outer_evaluation=None, + error="proxy trial budget exhausted", + )) + break + proxy_allowance = max( + 1, + proxy_remaining // selection_phases_remaining, + ) + beam = beam_search_factor_combinations( + outer_train, + _searchable_factors(pruned.selected, request.budget), + target_column=request.target_column, + date_column=request.date_column, + max_combination_size=request.budget.max_combination_size, + beam_width=request.budget.beam_width, + max_trials=proxy_allowance, + cancel_check=cancel_check, + ) + proxy_trials_used += beam.trials_used + selection_phases_remaining -= 1 + candidates = beam.candidates + fold_results.append(FoldMiningResult( + outer_index=nested.outer.outer_index, + selected_factors=pruned.selected, + candidates=tuple(candidates), + selected_candidate_id=None, + inner_score=None, + outer_evaluation=None, + )) + continue + + winners: list[tuple[float, MiningCandidate]] = [] + all_candidates: dict[str, MiningCandidate] = {} + selected_factor_union: set[str] = set() + for inner in nested.inner: + if _cancelled(cancel_check): + cancelled = True + break + inner_train = _train_panel_for_fold( + panel, + request.date_column, + inner, + ) + inner_test = _panel_for_labels( + panel, + request.date_column, + inner.test_labels, + ) + metrics = tuple( + metric_provider(inner_train, request.factor_names) + if metric_provider is not None + else factor_metrics or () + ) + correlation = compute_rank_correlation( + inner_train, + request.factor_names, + date_column=request.date_column, + ) + pruned = prune_correlated_factors( + metrics, + correlation, + request.correlation_threshold, + ) + selected_factor_union.update(pruned.selected) + + proxy_remaining = request.budget.max_proxy_trials - proxy_trials_used + if proxy_remaining <= 0: + break + proxy_allowance = max( + 1, + proxy_remaining // selection_phases_remaining, + ) + beam = beam_search_factor_combinations( + inner_train, + _searchable_factors(pruned.selected, request.budget), + target_column=request.target_column, + date_column=request.date_column, + max_combination_size=request.budget.max_combination_size, + beam_width=request.budget.beam_width, + max_trials=proxy_allowance, + cancel_check=cancel_check, + ) + proxy_trials_used += beam.trials_used + selection_phases_remaining -= 1 + candidates = beam.candidates + for candidate in candidates: + all_candidates[candidate.candidate_id] = candidate + + real_remaining = request.budget.max_trials - trials_used + if real_remaining <= 0: + break + real_allowance = min( + MAX_FINALISTS, + max(1, real_remaining // evaluation_phases_remaining), + ) + evaluated: list[tuple[float, MiningCandidate]] = [] + for candidate in candidates[:real_allowance]: + evaluation = ( + _evaluate_labels( + labels_evaluator, + inner.train_labels, + inner.test_labels, + candidate, + ) + if callable(labels_evaluator) + else _evaluate( + evaluator, + inner_train, + inner_test, + candidate, + ) + ) + trials_used += 1 + if evaluation.error is None and evaluation.score is not None: + evaluated.append((evaluation.score, candidate)) + evaluation_phases_remaining -= 1 + if evaluated: + winners.append(min( + evaluated, + key=lambda item: (-item[0], item[1].candidate_id), + )) + + if cancelled: + break + if not winners: + fold_results.append(FoldMiningResult( + outer_index=nested.outer.outer_index, + selected_factors=tuple(sorted(selected_factor_union)), + candidates=tuple(_rank_candidates(list(all_candidates.values()))), + selected_candidate_id=None, + inner_score=None, + outer_evaluation=None, + error="no candidate completed inner validation within budget", + benchmark_evaluations=run_benchmarks(nested), + )) + break + + by_candidate: dict[str, list[float]] = {} + for score, candidate in winners: + by_candidate.setdefault(candidate.candidate_id, []).append(score) + selected_id, selected_scores = min( + by_candidate.items(), + key=lambda item: (-len(item[1]), -float(np.mean(item[1])), item[0]), + ) + voted_candidate = all_candidates[selected_id] + inner_score = float(np.mean(selected_scores)) + if not callable(labels_evaluator): + del inner_train, inner_test + + outer_train = _train_panel_for_fold( + panel, + request.date_column, + nested.outer, + ) + outer_metrics = tuple( + metric_provider(outer_train, request.factor_names) + if metric_provider is not None + else factor_metrics or () + ) + outer_correlation = compute_rank_correlation( + outer_train, + request.factor_names, + date_column=request.date_column, + ) + outer_pruned = prune_correlated_factors( + outer_metrics, + outer_correlation, + request.correlation_threshold, + ) + proxy_remaining = request.budget.max_proxy_trials - proxy_trials_used + if proxy_remaining <= 0: + fold_results.append(FoldMiningResult( + outer_index=nested.outer.outer_index, + selected_factors=outer_pruned.selected, + candidates=tuple(_rank_candidates(list(all_candidates.values()))), + selected_candidate_id=None, + inner_score=round(inner_score, 8), + outer_evaluation=None, + error="proxy trial budget exhausted before outer retraining", + benchmark_evaluations=run_benchmarks(nested), + )) + break + proxy_allowance = max( + 1, + proxy_remaining // selection_phases_remaining, + ) + outer_beam = beam_search_factor_combinations( + outer_train, + _searchable_factors(outer_pruned.selected, request.budget), + target_column=request.target_column, + date_column=request.date_column, + max_combination_size=request.budget.max_combination_size, + beam_width=request.budget.beam_width, + max_trials=proxy_allowance, + cancel_check=cancel_check, + ) + proxy_trials_used += outer_beam.trials_used + selection_phases_remaining -= 1 + outer_candidates = outer_beam.candidates + by_refit_key = { + _candidate_refit_key(candidate): candidate + for candidate in outer_candidates + } + selected = by_refit_key.get(_candidate_refit_key(voted_candidate)) + if selected is None: + fold_results.append(FoldMiningResult( + outer_index=nested.outer.outer_index, + selected_factors=outer_pruned.selected, + candidates=outer_candidates, + selected_candidate_id=None, + inner_score=round(inner_score, 8), + outer_evaluation=None, + error=( + "outer retraining did not reproduce selected candidate structure" + if outer_candidates + else "outer training produced no candidate" + ), + benchmark_evaluations=run_benchmarks(nested), + )) + break + + if trials_used >= request.budget.max_trials: + fold_results.append(FoldMiningResult( + outer_index=nested.outer.outer_index, + selected_factors=outer_pruned.selected, + candidates=outer_candidates, + selected_candidate_id=selected.candidate_id, + inner_score=round(inner_score, 8), + outer_evaluation=None, + error="real trial budget exhausted before outer evaluation", + benchmark_evaluations=run_benchmarks(nested), + )) + break + outer_test = _panel_for_labels( + panel, + request.date_column, + nested.outer.test_labels, + ) + if callable(labels_evaluator): + del outer_train, outer_test + outer_evaluation = _evaluate_labels( + labels_evaluator, + nested.outer.train_labels, + nested.outer.test_labels, + selected, + ) + else: + outer_evaluation = _evaluate( + evaluator, + outer_train, + outer_test, + selected, + ) + trials_used += 1 + evaluation_phases_remaining -= 1 + fold_results.append(FoldMiningResult( + outer_index=nested.outer.outer_index, + selected_factors=outer_pruned.selected, + candidates=outer_candidates, + selected_candidate_id=selected.candidate_id, + inner_score=round(inner_score, 8), + outer_evaluation=outer_evaluation, + benchmark_evaluations=run_benchmarks(nested), + )) + if cancelled: + break + if trials_used >= request.budget.max_trials: + break + + if not cancelled and evaluator is not None: + winner_union: dict[str, MiningCandidate] = {} + for fold in fold_results: + if fold.selected_candidate_id is None: + continue + if fold.selected_candidate_id in winner_union: + continue + winner = next( + ( + candidate + for candidate in fold.candidates + if candidate.candidate_id == fold.selected_candidate_id + ), + None, + ) + if winner is not None: + winner_union[winner.candidate_id] = winner + if len(winner_union) > 1: + extended_folds: list[FoldMiningResult] = [] + for nested, fold in zip(nested_folds, fold_results, strict=False): + cross: list[tuple[str, CandidateEvaluation]] = [] + for candidate_id, candidate in winner_union.items(): + if candidate_id == fold.selected_candidate_id: + continue + if trials_used >= request.budget.max_trials: + cross.append(( + candidate_id, + CandidateEvaluation( + score=None, + error=( + "real trial budget exhausted before " + "cross-fold evaluation" + ), + ), + )) + continue + if callable(labels_evaluator): + evaluation = _evaluate_labels( + labels_evaluator, + nested.outer.train_labels, + nested.outer.test_labels, + candidate, + ) + else: + cross_train = _train_panel_for_fold( + panel, + request.date_column, + nested.outer, + ) + cross_test = _panel_for_labels( + panel, + request.date_column, + nested.outer.test_labels, + ) + evaluation = _evaluate( + evaluator, + cross_train, + cross_test, + candidate, + ) + trials_used += 1 + cross.append((candidate_id, evaluation)) + extended_folds.append(replace( + fold, + cross_evaluations=tuple(cross), + )) + fold_results = extended_folds + + return MiningResult( + request=request, + folds=tuple(fold_results), + proxy_trials_used=proxy_trials_used, + trials_used=trials_used, + cancelled=cancelled, + elapsed_ms=round((time.perf_counter() - started) * 1000.0, 3), + ) + + +def _validate_factor_names( + panel: pl.DataFrame, + factor_names: Sequence[str], +) -> tuple[str, ...]: + names = tuple(str(name) for name in factor_names) + if not names: + raise ValueError("factor_names must not be empty") + if len(names) > MAX_MINING_FACTORS: + raise ValueError(f"factor count exceeds hard limit {MAX_MINING_FACTORS}") + if len(set(names)) != len(names): + raise ValueError("factor_names must not contain duplicates") + missing = sorted(set(names) - set(panel.columns)) + if missing: + raise ValueError(f"panel is missing factor columns: {missing}") + return names + + +def _finite_sort_value(value: float, *, worst: float = float("-inf")) -> float: + try: + number = float(value) + except (TypeError, ValueError): + return worst + return number if math.isfinite(number) else worst + + +def _average_rank(values: np.ndarray) -> np.ndarray: + order = np.argsort(values, kind="stable") + ordered = values[order] + boundaries = np.concatenate(( + np.array([0]), + np.flatnonzero(ordered[1:] != ordered[:-1]) + 1, + np.array([len(ordered)]), + )) + counts = np.diff(boundaries) + average = (boundaries[:-1] + 1 + boundaries[1:]) / 2.0 + ranked = np.empty(len(values), dtype=np.float64) + ranked[order] = np.repeat(average, counts) + return ranked + + +def _proxy_rank_ic( + blocks: Sequence[np.ndarray], + columns: tuple[int, ...], + weights: tuple[float, ...], + directions: tuple[int, ...], +) -> tuple[float, float, int, int]: + daily_ics: list[float] = [] + observations = 0 + weight_values = np.asarray(weights, dtype=np.float64) + direction_values = np.asarray(directions, dtype=np.float64) + for block in blocks: + factors = block[:, columns].astype(np.float64, copy=False) + target = block[:, -1].astype(np.float64, copy=False) + valid = np.isfinite(target) & np.isfinite(factors).all(axis=1) + count = int(np.count_nonzero(valid)) + if count < 3: + continue + factor_values = factors[valid] + factor_ranks = np.column_stack([ + _average_rank(factor_values[:, index]) + for index in range(factor_values.shape[1]) + ]) + composite = (factor_ranks * direction_values) @ weight_values + composite_rank = _average_rank(composite) + target_rank = _average_rank(target[valid]) + if np.std(composite_rank) <= 0.0 or np.std(target_rank) <= 0.0: + continue + rho = float(np.corrcoef(composite_rank, target_rank)[0, 1]) + if math.isfinite(rho): + daily_ics.append(rho) + observations += count + if not daily_ics: + return 0.0, 0.0, observations, 0 + values = np.asarray(daily_ics, dtype=np.float64) + mean = float(np.mean(values)) + std = float(np.std(values)) + ir = mean / std if std > 1e-12 else 0.0 + return mean, ir, observations, len(daily_ics) + + +def compute_candidate_signature(definition: Mapping[str, Any]) -> str: + """Return the canonical artifact signature for a mining candidate definition.""" + kind = definition.get("kind") + if kind == "existing_strategy": + strategy_id = definition.get("strategy_id") + if not isinstance(strategy_id, str) or not strategy_id: + raise ValueError("existing strategy candidate requires strategy_id") + return f"strategy:{strategy_id}" + if kind != "factor_rank": + raise ValueError(f"unsupported mining candidate kind: {kind!r}") + + factor_names = definition.get("factor_names") + scoring = definition.get("scoring") + directions = definition.get("directions") + if not isinstance(factor_names, list) or not factor_names: + raise ValueError("factor candidate requires factor_names") + if not isinstance(scoring, Mapping) or set(scoring) != set(factor_names): + raise ValueError("factor candidate scoring keys must match factor_names") + if not isinstance(directions, Mapping) or set(directions) != set(factor_names): + raise ValueError("factor candidate direction keys must match factor_names") + + weights: list[str] = [] + direction_values: list[str] = [] + for factor_name in factor_names: + if not isinstance(factor_name, str) or not factor_name: + raise ValueError("factor names must be non-empty strings") + value = scoring[factor_name] + if isinstance(value, bool): + raise ValueError("factor weights must be numeric") + try: + weight = float(value) + except (TypeError, ValueError) as exc: + raise ValueError("factor weights must be numeric") from exc + if not math.isfinite(weight) or weight <= 0.0: + raise ValueError("factor weights must be finite and positive") + direction = directions[factor_name] + if direction not in {"high", "low"}: + raise ValueError("factor directions must be high or low") + weights.append(f"{weight:g}") + direction_values.append("1" if direction == "high" else "-1") + return ( + f"factor:{','.join(factor_names)}|w:{','.join(weights)}" + f"|d:{','.join(direction_values)}" + ) + + +def _factor_candidate( + factors: tuple[str, ...], + weights: tuple[float, ...], + directions: tuple[int, ...], + rank_ic: float, + proxy_ir: float, + observations: int, + dates: int, +) -> MiningCandidate: + candidate_id = compute_candidate_signature({ + "kind": "factor_rank", + "factor_names": list(factors), + "scoring": dict(zip(factors, weights, strict=True)), + "directions": { + factor_id: "high" if direction > 0 else "low" + for factor_id, direction in zip(factors, directions, strict=True) + }, + }) + return MiningCandidate( + candidate_id=candidate_id, + kind="factor_rank", + factor_names=factors, + weights=weights, + directions=directions, + proxy_rank_ic=round(float(rank_ic), 8), + proxy_ir=round(float(proxy_ir), 8), + observations=observations, + dates=dates, + ) + + +def benchmark_candidate(strategy_id: str) -> MiningCandidate: + """Build the fixed benchmark candidate for one user-selected strategy.""" + return MiningCandidate( + candidate_id=_benchmark_signature(strategy_id), + kind="existing_strategy", + strategy_id=strategy_id, + ) + + +def _benchmark_signature(strategy_id: str) -> str: + return compute_candidate_signature({ + "kind": "existing_strategy", + "strategy_id": strategy_id, + }) + + +def _searchable_factors( + selected: Sequence[str], + budget: MiningBudget, +) -> tuple[str, ...]: + """Cap beam-search inputs so singletons plus combinations fit the phase budget. + + ``selected`` arrives in training-fold metric order, so the cap keeps the + strongest factors and keeps the lexicographic singleton pass from consuming + the whole proxy allowance before any combination is scored. + """ + return tuple(selected[: max(1, budget.beam_width)]) + + +def _candidate_refit_key(candidate: MiningCandidate) -> tuple[Any, ...]: + if candidate.kind == "existing_strategy": + return (candidate.kind, candidate.strategy_id) + return (candidate.kind, candidate.factor_names, candidate.weights) + + +def _rank_candidates(candidates: Sequence[MiningCandidate]) -> list[MiningCandidate]: + return sorted( + candidates, + key=lambda candidate: ( + -candidate.proxy_rank_ic, + -candidate.proxy_ir, + candidate.candidate_id, + ), + ) + + +def _make_validation_fold( + labels: tuple[str, ...], + *, + level: Literal["outer", "inner"], + outer_index: int, + inner_index: int | None, + train_start: int, + train_bars: int, + test_bars: int, + purge_bars: int, + embargo_bars: int, + hard_stop: int | None = None, +) -> ValidationFold: + train_stop = train_start + train_bars + test_start = train_stop + purge_bars + test_stop = test_start + test_bars + embargo_stop = test_stop + embargo_bars + if hard_stop is not None: + embargo_stop = min(embargo_stop, hard_stop) + return ValidationFold( + level=level, + outer_index=outer_index, + inner_index=inner_index, + train_labels=labels[train_start:train_stop], + purge_labels=labels[train_stop:test_start], + test_labels=labels[test_start:test_stop], + embargo_labels=labels[test_stop:embargo_stop], + ) + + +def _train_panel_for_fold( + panel: pl.DataFrame, + date_column: str, + fold: ValidationFold, +) -> pl.DataFrame: + train = _panel_for_labels(panel, date_column, fold.train_labels) + if "_target_date" not in train.columns: + return train + target_date = pl.col("_target_date").cast(pl.Utf8).str.slice(0, 10) + daily = train.select(date_column, "_target_date").unique( + subset=[date_column], + maintain_order=True, + ) + allowed = daily.select( + (target_date.is_null() | (target_date <= fold.train_end)).alias("_allowed") + ).get_column("_allowed").to_list() + first_excluded = next( + (index for index, value in enumerate(allowed) if not value), + len(allowed), + ) + if not any(allowed[first_excluded:]): + return _panel_for_labels( + train, + date_column, + fold.train_labels[:first_excluded], + ) + return train.filter(target_date.is_null() | (target_date <= fold.train_end)) + + +def _panel_for_labels( + panel: pl.DataFrame, + date_column: str, + labels: Sequence[str], +) -> pl.DataFrame: + normalized = tuple(str(label)[:10] for label in labels) + if not normalized: + return panel.slice(0, 0) + values = panel.get_column(date_column) + if values.is_sorted(): + bounds: tuple[Any, Any] | None = None + if values.dtype == pl.Date: + bounds = ( + date.fromisoformat(normalized[0]), + date.fromisoformat(normalized[-1]), + ) + elif values.dtype == pl.Utf8: + bounds = (normalized[0], normalized[-1]) + if bounds is not None: + start = int(values.search_sorted(bounds[0], side="left")) + stop = int(values.search_sorted(bounds[1], side="right")) + sliced = panel.slice(start, stop - start) + actual = tuple( + str(value)[:10] + for value in sliced.get_column(date_column).unique( + maintain_order=True + ) + ) + if actual == normalized: + return sliced + date_expr = pl.col(date_column).cast(pl.Utf8).str.slice(0, 10) + return panel.filter(date_expr.is_in(normalized)) + + +def _cancelled(cancel_check: CancelCheck | None) -> bool: + if cancel_check is None: + return False + if callable(cancel_check): + return bool(cancel_check()) + is_set = getattr(cancel_check, "is_set", None) + return bool(is_set()) if callable(is_set) else False + + +def _evaluate_labels( + evaluator: Callable[ + [Sequence[str], Sequence[str], Mapping[str, Any]], + CandidateEvaluation | Mapping[str, Any] | float, + ], + train_labels: Sequence[str], + test_labels: Sequence[str], + candidate: MiningCandidate, +) -> CandidateEvaluation: + try: + raw = evaluator(train_labels, test_labels, candidate.definition()) + except Exception as exc: + return CandidateEvaluation(score=None, error=str(exc)) + return _normalize_evaluation(raw) + + +def _evaluate( + evaluator: CandidateEvaluator, + train: pl.DataFrame, + test: pl.DataFrame, + candidate: MiningCandidate, +) -> CandidateEvaluation: + try: + raw = evaluator.evaluate_candidate(train, test, candidate.definition()) + except Exception as exc: + return CandidateEvaluation(score=None, error=str(exc)) + return _normalize_evaluation(raw) + + +def _normalize_evaluation( + raw: CandidateEvaluation | Mapping[str, Any] | float, +) -> CandidateEvaluation: + if isinstance(raw, CandidateEvaluation): + return CandidateEvaluation( + score=_finite_score(raw.score), + metrics=raw.metrics, + error=raw.error, + ) + if isinstance(raw, Mapping): + score = raw.get("score") + error = raw.get("error") + metrics = raw.get("metrics", {}) + return CandidateEvaluation( + score=_finite_score(score), + metrics=dict(metrics) if isinstance(metrics, Mapping) else {}, + error=str(error) if error is not None else None, + ) + return CandidateEvaluation(score=_finite_score(raw)) + + +def _finite_score(value: Any) -> float | None: + try: + score = float(value) + except (TypeError, ValueError): + return None + return score if math.isfinite(score) else None diff --git a/backend/app/backtest/mining_runtime.py b/backend/app/backtest/mining_runtime.py new file mode 100644 index 0000000..0733d7c --- /dev/null +++ b/backend/app/backtest/mining_runtime.py @@ -0,0 +1,1489 @@ +"""Production mining runtime executed only inside a spawned worker.""" +from __future__ import annotations + +import json +import math +import os +import time +import uuid +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, replace +from datetime import date, timedelta +from itertools import pairwise +from pathlib import Path +from typing import Any, Literal + +import numpy as np +import polars as pl + +from app.backtest.factor import ( + FACTOR_COLUMNS, + FACTOR_METHODOLOGY_VERSION, + FACTOR_WARMUP_DAYS, + FactorBacktestService, + FactorBatchConfig, +) +from app.backtest.fundamentals import ( + FUNDAMENTAL_FACTOR_NAMES, + attach_fundamental_factors, + load_fundamental_snapshot, +) +from app.backtest.mining import ( + MAX_FINALISTS, + CandidateEvaluation, + FactorMetric, + MiningBudget, + MiningCandidate, + MiningRequest, + MiningResult, + MiningService, + benchmark_candidate, + compute_rank_correlation, + generate_nested_folds, + required_outer_folds, + required_trading_bars, + validation_config_for_profile, +) +from app.backtest.strategy import ( + BacktestResultPolicy, + ResolvedFeaturePlan, + StrategyBacktestConfig, + StrategyBacktestService, + StrategyDependencyResolver, + _merge_resolved_feature_plans, + build_matrix_cache_profile, +) +from app.services.mining_jobs import MiningRunStore +from app.services.mining_preflight import enriched_partition_dates +from app.services.mining_schedule import MINING_ALGORITHM_VERSION +from app.strategy import config as strategy_config +from app.strategy.engine import StrategyEngine + +ProgressCallback = Callable[[dict[str, Any]], None] +CancelCheck = Callable[[], bool] | Any +_PROFILE_NAMES = frozenset({"exploratory", "balanced", "strict"}) +_FACTOR_IDS = frozenset(str(item["id"]) for item in FACTOR_COLUMNS) +_MINING_MATRIX_CACHE_BYTES = 32 * 1024 * 1024 +_RESULT_POLICY = BacktestResultPolicy( + required_stats=frozenset({"total_return", "sharpe", "max_drawdown", "n_trades"}), + include_monte_carlo=False, + include_curves=False, + include_trades=False, + include_per_symbol_stats=False, + include_return_distribution=False, + include_benchmark=False, + include_strategy_info=False, +) +_REGIME_FILTERS: dict[str, dict[str, list[str]]] = { + "strong": {"states": ["strong", "lean_strong"]}, + "range": {"states": ["range"]}, + "weak": {"states": ["lean_weak", "weak"]}, +} + + +class MiningRuntimeCancelledError(RuntimeError): + pass + + +@dataclass(frozen=True) +class RuntimeRequest: + run_id: str + factor_names: tuple[str, ...] + strategy_ids: tuple[str, ...] + symbols: list[str] | None + asset_type: Literal["stock", "etf"] + start: date + end: date + profile: Literal["exploratory", "balanced", "strict"] + forward_horizon: int + commission_pct: float + stamp_tax_pct: float + slippage_bps: float + correlation_threshold: float + max_finalists: int + require_regime: bool + mining_request: MiningRequest + + +class TrainingMetricProvider: + """Compute fold-local metrics and retain only compact call telemetry.""" + + def __init__(self, target_column: str) -> None: + self.target_column = target_column + self.calls: list[dict[str, Any]] = [] + + def __call__( + self, + train: pl.DataFrame, + factor_names: Sequence[str], + ) -> tuple[FactorMetric, ...]: + metrics = tuple( + _factor_metric(train, factor_name, self.target_column) + for factor_name in factor_names + ) + labels = _date_labels(train) + self.calls.append({ + "start": labels[0] if labels else None, + "end": labels[-1] if labels else None, + "rows": train.height, + "metrics": metrics, + }) + return metrics + + +class MatcherCandidateEvaluator: + """Evaluate one fixed definition with the production matrix matcher.""" + + def __init__( + self, + service: StrategyBacktestService, + strategy_engine: StrategyEngine, + data_dir: Path, + request: RuntimeRequest, + base_market, + cancel_check: CancelCheck | None, + ) -> None: + self.service = service + self.strategy_engine = strategy_engine + self.data_dir = data_dir + self.request = request + self.base_market = base_market + self.cancel_check = cancel_check + self.backtest_count = 0 + self.peak_compute_cache_bytes = 0 + + def evaluate_candidate( + self, + train: pl.DataFrame, + test: pl.DataFrame, + definition: Mapping[str, Any], + ) -> CandidateEvaluation: + del train + return self.evaluate_test(test, definition) + + def evaluate_candidate_labels( + self, + train_labels: Sequence[str], + test_labels: Sequence[str], + definition: Mapping[str, Any], + ) -> CandidateEvaluation: + del train_labels + return self._evaluate_labels(test_labels, definition) + + def evaluate_test( + self, + test: pl.DataFrame, + definition: Mapping[str, Any], + *, + regime_state: str = "overall", + ) -> CandidateEvaluation: + return self._evaluate_labels( + _date_labels(test), + definition, + regime_state=regime_state, + ) + + def _evaluate_labels( + self, + labels: Sequence[str], + definition: Mapping[str, Any], + *, + regime_state: str = "overall", + ) -> CandidateEvaluation: + _raise_if_cancelled(self.cancel_check) + if not labels: + return CandidateEvaluation(score=None, error="test fold contains no dates") + try: + config = self._backtest_config( + definition, + date.fromisoformat(labels[0]), + date.fromisoformat(labels[-1]), + regime_state, + ) + prepared = self.service.prepare_matrix_optimization( + [config], + matrix_cache_max_bytes=_MINING_MATRIX_CACHE_BYTES, + market_data_override=self.base_market, + ) + try: + result = self.service.run( + config, + cancel_event=self.cancel_check, + prepared=prepared, + result_policy=_RESULT_POLICY, + ) + self.peak_compute_cache_bytes = max( + self.peak_compute_cache_bytes, + prepared.compute_cache.snapshot()["peak_bytes"], + ) + finally: + prepared.compute_cache.close() + self.backtest_count += 1 + if result.error: + return CandidateEvaluation(score=None, error=result.error) + metrics = { + key: _finite_or_none(result.stats.get(key)) + for key in ("total_return", "sharpe", "max_drawdown", "n_trades") + } + score = _finite_or_none(metrics.get("sharpe")) + if score is None: + return CandidateEvaluation( + score=None, + metrics=metrics, + error="backtest did not return a finite sharpe", + ) + return CandidateEvaluation(score=score, metrics=metrics) + except (OSError, ValueError, TypeError) as exc: + return CandidateEvaluation(score=None, error=str(exc)) + + def _backtest_config( + self, + definition: Mapping[str, Any], + start: date, + end: date, + regime_state: str, + ) -> StrategyBacktestConfig: + kind = str(definition.get("kind") or "") + if kind == "existing_strategy": + strategy_id = str(definition.get("strategy_id") or "") + strategy = self.strategy_engine.get(strategy_id) + overrides = strategy_config.load_override(self.data_dir, strategy_id) + params = self.strategy_engine.resolve_params(strategy, overrides=overrides) + elif kind == "factor_rank": + strategy_id = "factor_rank_research" + scoring = definition.get("scoring") + directions = definition.get("directions") + if not isinstance(scoring, Mapping) or not scoring: + raise ValueError("factor candidate has no scoring definition") + if not isinstance(directions, Mapping): + raise ValueError("factor candidate has no direction definition") + params = { + "scoring": {str(key): float(value) for key, value in scoring.items()}, + "directions": {str(key): str(value) for key, value in directions.items()}, + "entry_score": 70.0, + "exit_score": 40.0, + "top_rank": 20, + } + overrides = {} + else: + raise ValueError(f"unsupported mining candidate kind: {kind!r}") + + regime_filter = None + if regime_state != "overall": + regime_filter = _REGIME_FILTERS.get(regime_state) + if regime_filter is None: + raise ValueError(f"unsupported regime state: {regime_state}") + return StrategyBacktestConfig( + strategy_id=strategy_id, + symbols=self.request.symbols, + start=start, + end=end, + params=params, + overrides=overrides, + matching="open_t+1", + entry_fill="open_t+1", + exit_fill="open_t+1", + fees_pct=self.request.commission_pct, + commission_pct=self.request.commission_pct, + stamp_tax_pct=self.request.stamp_tax_pct, + slippage_bps=self.request.slippage_bps, + max_positions=10, + max_exposure_pct=1.0, + initial_capital=1_000_000.0, + position_sizing="equal", + mode="position", + asset_type=self.request.asset_type, + holding_days=self.request.forward_horizon, + minute_fill=False, + regime_filter=regime_filter, + ) + + +_SYMBOL_BATCH_SIZE = 512 + + +def _load_compact_factor_panel( + factor_service: FactorBacktestService, + config: FactorBatchConfig, + factor_names: Sequence[str], + *, + expected_generation: str, + cancel_check: CancelCheck | None, +) -> pl.DataFrame: + panel_columns = [ + "symbol", + "date", + "open", + "high", + "low", + "close", + "volume", + "amount", + "turnover_rate", + ] + if any( + name in ("limit_up_count_20d", "limit_up_count_60d") + for name in factor_names + ): + panel_columns.append("consecutive_limit_ups") + load_start = ( + config.start + if all(name == "turnover_rate" for name in factor_names) + else config.start - timedelta(days=FACTOR_WARMUP_DAYS) + ) + raw = factor_service.engine.load_panel( + config.symbols, + load_start, + config.end, + columns=panel_columns, + asset_type=config.asset_type, + expected_generation=expected_generation, + ) + if raw.is_empty(): + return raw + + fundamental_names = [ + str(name) + for name in factor_names + if str(name) in FUNDAMENTAL_FACTOR_NAMES + ] + if fundamental_names: + data_dir = getattr( + getattr(getattr(factor_service.engine, "repo", None), "store", None), + "data_dir", + None, + ) + raw = attach_fundamental_factors( + raw, + load_fundamental_snapshot(data_dir), + fundamental_names, + ) + + symbol = pl.col("symbol") + day = pl.col("date") + previous_symbol = symbol.shift(1) + previous_day = day.shift(1) + invalid_key = raw.select( + ( + symbol.is_null() + | day.is_null() + | (symbol < previous_symbol).fill_null(False) + | ( + (symbol == previous_symbol) + & (day <= previous_day) + ).fill_null(False) + ).any() + ).item() + if invalid_key: + raise ValueError( + "mining factor panel requires non-null, unique symbol/date keys " + "sorted by symbol and strictly increasing date" + ) + output_by_date: dict[date, list[pl.DataFrame]] = {} + group_sizes = ( + raw.group_by("symbol", maintain_order=True) + .len() + .get_column("len") + .to_list() + ) + row_offset = 0 + for offset in range(0, len(group_sizes), _SYMBOL_BATCH_SIZE): + _raise_if_cancelled(cancel_check) + row_count = sum(group_sizes[offset:offset + _SYMBOL_BATCH_SIZE]) + batch = raw.slice(row_offset, row_count) + row_offset += row_count + missing = set(factor_names) - set(batch.columns) + if missing: + batch = factor_service._compute_missing_factors( + batch, + missing, + assume_sorted=True, + ) + projected = batch.filter( + (pl.col("date") >= config.start) + & (pl.col("date") <= config.end) + & pl.col("close").is_not_null() + & (pl.col("close") > 0) + ).select([ + "symbol", + "date", + "close", + *( + pl.col(name).cast(pl.Float32, strict=False).alias(name) + for name in factor_names + ), + ]) + for daily in projected.partition_by("date", maintain_order=True): + output_by_date.setdefault(daily.item(0, "date"), []).append(daily) + del batch, projected + return pl.concat( + [ + pl.concat(output_by_date[label], how="vertical", rechunk=False) + for label in sorted(output_by_date) + ], + how="vertical", + rechunk=False, + ) + + +def run_mining_runtime( + payload: Mapping[str, Any], + *, + data_dir: Path, + service: StrategyBacktestService, + strategy_engine: StrategyEngine, + progress_cb: ProgressCallback | None = None, + cancel_check: CancelCheck | None = None, + rss_sampler: Any | None = None, +) -> dict[str, Any]: + """Run one persistent mining job and return only a compact IPC summary.""" + started = time.perf_counter() + emit = progress_cb or (lambda _message: None) + request = _decode_runtime_request(payload, data_dir, strategy_engine) + fingerprint = payload.get("data_fingerprint") + expected_generation = ( + fingerprint.get("generation") + if isinstance(fingerprint, Mapping) + else None + ) + if not isinstance(expected_generation, str) or not expected_generation: + raise ValueError("mining worker payload is missing its data generation") + store = MiningRunStore(data_dir) + phase_peak_rss_bytes: dict[str, int] = {} + + def start_phase() -> None: + if rss_sampler is not None: + rss_sampler.reset_phase() + + def finish_phase(name: str) -> None: + if rss_sampler is not None: + phase_peak_rss_bytes[name] = rss_sampler.phase_peak_rss_bytes() + + emit({"phase": "panel", "label": "加载因子面板", "done": 0, "total": 1}) + start_phase() + _raise_if_cancelled(cancel_check) + panel_started = time.perf_counter() + factor_service = FactorBacktestService(service.engine) + factor_config = FactorBatchConfig( + factor_names=list(request.factor_names), + symbols=request.symbols, + start=request.start, + end=request.end, + asset_type=request.asset_type, + commission_pct=request.commission_pct, + stamp_tax_pct=request.stamp_tax_pct, + slippage_bps=request.slippage_bps, + ) + generation = factor_service._data_generation(request.asset_type) + if generation != expected_generation: + raise ValueError( + "mining data generation changed after the run was queued" + ) + source_panel = _load_compact_factor_panel( + factor_service, + factor_config, + request.factor_names, + expected_generation=generation, + cancel_check=cancel_check, + ) + if source_panel.is_empty(): + raise ValueError("mining date range contains no enriched data") + service.engine.clear_panel_cache() + trading_dates = enriched_partition_dates( + data_dir, + request.asset_type, + request.start, + request.end, + ) + factor_service._assert_data_generation(request.asset_type, generation) + panel = attach_single_forward_return( + source_panel, + start=request.start, + end=request.end, + horizon=request.forward_horizon, + trading_dates=trading_dates, + factor_names=request.factor_names, + target_column=request.mining_request.target_column, + assume_unique_symbol_date=True, + ) + del source_panel + if panel.is_empty(): + raise ValueError("mining panel contains no valid price rows") + phase_ms: dict[str, float] = { + "panel": round((time.perf_counter() - panel_started) * 1000.0, 3) + } + finish_phase("panel") + emit({ + "phase": "panel", + "label": "因子面板已准备", + "done": 1, + "total": 1, + "rows": panel.height, + "factors": len(request.factor_names), + }) + + _raise_if_cancelled(cancel_check) + start_phase() + matrix_started = time.perf_counter() + emit({"phase": "matrix", "label": "准备共享撮合矩阵", "done": 0, "total": 1}) + base_market = _prepare_base_market( + service, + strategy_engine, + data_dir, + request, + expected_generation=generation, + cancel_check=cancel_check, + ) + factor_service._assert_data_generation(request.asset_type, generation) + phase_ms["matrix"] = round((time.perf_counter() - matrix_started) * 1000.0, 3) + finish_phase("matrix") + emit({ + "phase": "matrix", + "label": "共享撮合矩阵已准备", + "done": 1, + "total": 1, + "matrix_bytes": base_market.nbytes, + }) + + metric_provider = TrainingMetricProvider(request.mining_request.target_column) + evaluator = MatcherCandidateEvaluator( + service, + strategy_engine, + data_dir, + request, + base_market, + cancel_check, + ) + emit({"phase": "search", "label": "嵌套样本外搜索", "done": 0, "total": 1}) + start_phase() + search_started = time.perf_counter() + result = MiningService().run( + panel, + request.mining_request, + metric_provider=metric_provider, + evaluator=evaluator, + cancel_check=cancel_check, + ) + phase_ms["search"] = round((time.perf_counter() - search_started) * 1000.0, 3) + finish_phase("search") + if result.cancelled: + raise MiningRuntimeCancelledError("mining cancelled") + emit({ + "phase": "search", + "label": "候选搜索完成", + "done": 1, + "total": 1, + "proxy_trials": result.proxy_trials_used, + "real_trials": result.trials_used, + }) + + _raise_if_cancelled(cancel_check) + start_phase() + artifact_started = time.perf_counter() + emit({"phase": "artifacts", "label": "写入研究结果", "done": 0, "total": 4}) + artifact_frames = _build_artifacts( + panel, + request, + result, + metric_provider, + evaluator, + cancel_check, + ) + for done, (name, frame) in enumerate(artifact_frames.items(), start=1): + _raise_if_cancelled(cancel_check) + path = store.artifact_path(request.run_id, name) # type: ignore[arg-type] + _atomic_write_parquet(frame, path) + store.register_artifact(request.run_id, name) # type: ignore[arg-type] + emit({ + "phase": "artifacts", + "label": f"已写入 {name}", + "done": done, + "total": 4, + }) + phase_ms["artifacts"] = round((time.perf_counter() - artifact_started) * 1000.0, 3) + finish_phase("artifacts") + + folds = artifact_frames["folds"] + candidates = artifact_frames["candidates"] + factors = artifact_frames["factors"] + selected_overall = folds.filter( + (pl.col("regime_state") == "overall") + & ( + (pl.col("evaluation_kind") == "selected") + | pl.col("candidate_signature").is_null() + ) + ) + valid_folds = selected_overall.filter(~pl.col("skipped")).height + skipped_folds = selected_overall.filter(pl.col("skipped")).height + budget_exhausted = _budget_exhausted(result) + elapsed_ms = round((time.perf_counter() - started) * 1000.0, 3) + phase_ms["total"] = elapsed_ms + confidence = _confidence(request.profile) + return { + "status": ( + "succeeded_with_budget_exhausted" if budget_exhausted else "succeeded" + ), + "factor_count": len(request.factor_names), + "selected_factor_count": int(factors.filter(pl.col("selected")).height), + "candidate_count": candidates.height, + "valid_fold_count": valid_folds, + "skipped_fold_count": skipped_folds, + "confidence": confidence, + "budget_exhausted": budget_exhausted, + "elapsed_ms": elapsed_ms, + "data_as_of": request.end.isoformat(), + "algorithm_version": MINING_ALGORITHM_VERSION, + "methodology_version": FACTOR_METHODOLOGY_VERSION, + "proxy_trials_used": result.proxy_trials_used, + "trials_used": result.trials_used + max(0, evaluator.backtest_count - result.trials_used), + "panel_rows": panel.height, + "panel_scans": 1, + "matrix_bytes": base_market.nbytes, + "matrix_compute_cache_peak_bytes": evaluator.peak_compute_cache_bytes, + "phase_ms": phase_ms, + "phase_peak_rss_bytes": phase_peak_rss_bytes, + "artifacts": list(artifact_frames), + } + + +def attach_single_forward_return( + panel: pl.DataFrame, + *, + start: date, + end: date, + horizon: int, + trading_dates: Sequence[date], + factor_names: Sequence[str], + target_column: str = "_next_return", + assume_unique_symbol_date: bool = False, +) -> pl.DataFrame: + """Materialize exactly one global-axis forward label and its endpoint date.""" + if horizon <= 0: + raise ValueError("forward horizon must be positive") + names = tuple(str(name) for name in factor_names) + required = {"symbol", "date", "close", *names} + missing = sorted(required - set(panel.columns)) + if missing: + raise ValueError(f"factor panel is missing required columns: {missing}") + dates = tuple(sorted(dict.fromkeys( + value for value in trading_dates if start <= value <= end + ))) + if not dates: + raise ValueError("mining date range has no trading dates") + + valid_rows = ( + (pl.col("date") >= start) + & (pl.col("date") <= end) + & pl.col("close").is_not_null() + & (pl.col("close") > 0) + ) + if assume_unique_symbol_date: + invalid_count = panel.select((~valid_rows).sum()).item() + if invalid_count: + raise ValueError("prevalidated mining panel contains invalid price rows") + scoped = panel.select([ + "symbol", + "date", + "close", + *( + pl.col(name).cast(pl.Float32, strict=False).alias(name) + for name in names + ), + ]) + else: + scoped = ( + panel.filter(valid_rows) + .select([ + "symbol", + "date", + "close", + *( + pl.col(name).cast(pl.Float32, strict=False).alias(name) + for name in names + ), + ]) + .unique(subset=["symbol", "date"], keep="last") + ) + date_dtype = scoped.schema["date"] + target_date_column = "_target_date" + if len(dates) > horizon: + date_map = pl.DataFrame({ + "date": dates[:-horizon], + target_date_column: dates[horizon:], + }).with_columns( + pl.col("date").cast(date_dtype), + pl.col(target_date_column).cast(date_dtype), + ) + else: + date_map = pl.DataFrame( + schema={"date": date_dtype, target_date_column: date_dtype} + ) + prices = scoped.select("symbol", "date", "close") + lookup = prices.select( + "symbol", + pl.col("date").alias(target_date_column), + pl.col("close").alias("_target_close"), + ) + labels = ( + prices.join(date_map, on="date", how="left") + .join(lookup, on=["symbol", target_date_column], how="left") + .select( + "symbol", + "date", + pl.when(pl.col("_target_close").is_not_null()) + .then(pl.col("_target_close") / pl.col("close") - 1.0) + .otherwise(None) + .cast(pl.Float32) + .alias(target_column), + target_date_column, + ) + ) + if assume_unique_symbol_date: + labels = labels.sort(["date", "symbol"]) + if ( + not labels.get_column("symbol").equals(scoped.get_column("symbol")) + or not labels.get_column("date").equals(scoped.get_column("date")) + ): + raise ValueError("prevalidated mining panel is not sorted by date and symbol") + return scoped.select(["symbol", "date", *names]).hstack([ + labels.get_column(target_column), + labels.get_column(target_date_column), + ]) + return ( + scoped.select(["symbol", "date", *names]) + .join(labels, on=["symbol", "date"], how="left") + .sort(["date", "symbol"]) + ) + + +def _decode_runtime_request( + payload: Mapping[str, Any], + data_dir: Path, + strategy_engine: StrategyEngine, +) -> RuntimeRequest: + run_id = str(payload.get("run_id") or "") + request = payload.get("request") + if not run_id or not isinstance(request, Mapping): + raise ValueError("mining worker payload is missing run_id or request") + + factor_names = tuple(str(value) for value in request.get("factor_names") or ()) + if not factor_names or len(set(factor_names)) != len(factor_names): + raise ValueError("factor_names must be non-empty and unique") + unknown_factors = sorted(set(factor_names) - _FACTOR_IDS) + if unknown_factors: + raise ValueError(f"unknown mining factors: {unknown_factors}") + if len(factor_names) > 48: + raise ValueError("mining supports at most 48 factors") + + strategy_ids = tuple(str(value) for value in request.get("strategy_ids") or ()) + if len(set(strategy_ids)) != len(strategy_ids) or len(strategy_ids) > 8: + raise ValueError("strategy_ids must be unique and contain at most 8 strategies") + asset_type = str(request.get("asset_type") or "stock") + if asset_type not in {"stock", "etf"}: + raise ValueError("mining asset_type must be stock or etf") + for strategy_id in strategy_ids: + strategy = strategy_engine.get(strategy_id) + if strategy.meta.get("research_only"): + raise ValueError(f"research template cannot be selected as existing strategy: {strategy_id}") + if strategy.execution_backend != "matrix_native": + raise ValueError(f"mining strategy is not matrix-native: {strategy_id}") + if asset_type not in strategy.meta.get("asset_types", ["stock"]): + raise ValueError(f"mining strategy does not support {asset_type}: {strategy_id}") + + all_dates = enriched_partition_dates(data_dir, asset_type) + if not all_dates: + raise ValueError(f"no enriched {asset_type} trading dates are available") + requested_start = _optional_date(request.get("start")) + requested_end = _optional_date(request.get("end")) + if ( + requested_start is not None + and requested_end is not None + and requested_start > requested_end + ): + raise ValueError("mining start must not be after end") + start = max(requested_start or all_dates[0], all_dates[0]) + end = min(requested_end or all_dates[-1], all_dates[-1]) + if start > end: + raise ValueError("mining date range contains no enriched data") + + profile = str(request.get("budget_profile") or "balanced") + if profile not in _PROFILE_NAMES: + raise ValueError(f"unsupported mining profile: {profile}") + validation = validation_config_for_profile(profile) + forward_horizon = int(request.get("forward_horizon") or 5) + if forward_horizon not in {1, 3, 5}: + raise ValueError("forward_horizon must be 1, 3, or 5 trading days") + if validation.purge_bars < forward_horizon: + raise ValueError("validation purge must cover the forward horizon") + + budget = getattr(MiningBudget, profile)() + max_combination = int( + request.get("max_combination_factors") or budget.max_combination_size + ) + beam_width = int(request.get("beam_width") or budget.beam_width) + max_finalists = int(request.get("max_finalists") or MAX_FINALISTS) + if not 1 <= max_finalists <= MAX_FINALISTS: + raise ValueError(f"max_finalists must be between 1 and {MAX_FINALISTS}") + scoped_dates = [value for value in all_dates if start <= value <= end] + required_folds = required_outer_folds(profile) + required_bars = required_trading_bars(validation, required_folds) + if len(scoped_dates) < required_bars: + fold_label = "outer fold" if required_folds == 1 else "outer folds" + raise ValueError( + f"{profile} mining requires at least {required_bars} enriched trading " + f"bars for {required_folds} {fold_label}; effective range " + f"{start.isoformat()} to {end.isoformat()} has {len(scoped_dates)}" + ) + nested = generate_nested_folds( + [value.isoformat() for value in scoped_dates], + validation, + ) + reserved_regime_trials = 3 * len(nested) if request.get("require_regime", True) else 0 + real_trials = max(len(nested) + 1, budget.max_trials - reserved_regime_trials) + real_trials = min(real_trials, budget.max_trials) + budget = replace( + budget, + max_combination_size=max_combination, + beam_width=beam_width, + max_trials=real_trials, + ) + correlation_threshold = _bounded_float( + request.get("correlation_threshold", 0.75), + "correlation_threshold", + 0.0, + 1.0, + exclusive_min=True, + ) + commission_pct = _bounded_float( + request.get("commission_pct", 0.0002), "commission_pct", 0.0, 0.05 + ) + stamp_tax_pct = _bounded_float( + request.get("stamp_tax_pct", 0.0005), "stamp_tax_pct", 0.0, 0.05 + ) + slippage_bps = _bounded_float( + request.get("slippage_bps", 5.0), "slippage_bps", 0.0, 1000.0 + ) + symbols_value = request.get("symbols") + symbols = None + if symbols_value is not None: + if not isinstance(symbols_value, list): + raise ValueError("symbols must be a list or null") + symbols = list(dict.fromkeys(str(value) for value in symbols_value if value)) + if not symbols: + symbols = None + + mining_request = MiningRequest( + factor_names=factor_names, + existing_strategy_ids=strategy_ids, + correlation_threshold=correlation_threshold, + target_column="_next_return", + budget=budget, + validation=validation, + profile=profile, # type: ignore[arg-type] + ) + return RuntimeRequest( + run_id=run_id, + factor_names=factor_names, + strategy_ids=strategy_ids, + symbols=symbols, + asset_type=asset_type, # type: ignore[arg-type] + start=start, + end=end, + profile=profile, # type: ignore[arg-type] + forward_horizon=forward_horizon, + commission_pct=commission_pct, + stamp_tax_pct=stamp_tax_pct, + slippage_bps=slippage_bps, + correlation_threshold=correlation_threshold, + max_finalists=max_finalists, + require_regime=bool(request.get("require_regime", True)), + mining_request=mining_request, + ) + + +def _prepare_base_market( + service: StrategyBacktestService, + strategy_engine: StrategyEngine, + data_dir: Path, + request: RuntimeRequest, + *, + expected_generation: str | None = None, + cancel_check: CancelCheck | None = None, +): + resolver = StrategyDependencyResolver() + plans: list[ResolvedFeaturePlan] = [] + research = strategy_engine.get("factor_rank_research") + for offset in range(0, len(request.factor_names), 4): + factor_chunk = request.factor_names[offset:offset + 4] + research_params = { + "scoring": {factor_name: 1.0 for factor_name in factor_chunk}, + "directions": {factor_name: "high" for factor_name in factor_chunk}, + } + plans.append(resolver.resolve( + research, + params=research_params, + basic_filter=service._effective_basic_filter(research, {}), + entry_signals=research.entry_signals, + exit_signals=research.exit_signals, + overrides={}, + )) + for strategy_id in request.strategy_ids: + strategy = strategy_engine.get(strategy_id) + overrides = strategy_config.load_override(data_dir, strategy_id) + params = strategy_engine.resolve_params(strategy, overrides=overrides) + plans.append(resolver.resolve( + strategy, + params=params, + basic_filter=service._effective_basic_filter(strategy, overrides), + entry_signals=service._effective_signals( + overrides, "entry_signals", strategy.entry_signals + ), + exit_signals=service._effective_signals( + overrides, "exit_signals", strategy.exit_signals + ), + overrides=overrides, + )) + merged = _merge_resolved_feature_plans(plans) + profile = build_matrix_cache_profile( + strategy_engine, + request.asset_type, + requested_plan=merged, + requested_forward_bars=request.forward_horizon, + ) + warmup_days = max(120, int(max(merged.warmup_bars, 1) * 1.6)) + load_start = request.start - timedelta(days=warmup_days) + return service.engine.load_market_data_matrix_for_backtest( + request.symbols, + load_start, + request.end, + merged, + asset_type=request.asset_type, + cache_profile=profile, + coverage_start=load_start, + coverage_end=request.end, + expected_generation=expected_generation, + cancel_event=cancel_check, + ) + + +def _build_artifacts( + panel: pl.DataFrame, + request: RuntimeRequest, + result: MiningResult, + metric_provider: TrainingMetricProvider, + evaluator: MatcherCandidateEvaluator, + cancel_check: CancelCheck | None, +) -> dict[str, pl.DataFrame]: + nested = generate_nested_folds(_date_labels(panel), request.mining_request.validation) + last_train = _panel_for_dates(panel, nested[-1].outer.train_labels) + if "_target_date" in last_train.columns: + last_train = last_train.filter( + pl.col("_target_date").is_not_null() + & (pl.col("_target_date") <= date.fromisoformat(nested[-1].outer.train_end)) + ) + latest_metrics = { + metric.factor_id: metric + for metric in metric_provider(last_train, request.factor_names) + } + selected_factors = { + factor_name + for fold in result.folds + for factor_name in fold.selected_factors + } + direction_by_factor: dict[str, int] = {} + for fold in result.folds: + for candidate in fold.candidates: + for factor_name, direction in zip( + candidate.factor_names, candidate.directions, strict=True + ): + direction_by_factor.setdefault(factor_name, int(direction)) + metadata = {str(item["id"]): item for item in FACTOR_COLUMNS} + factor_rows = [] + for factor_name in request.factor_names: + metric = latest_metrics[factor_name] + factor_rows.append({ + "factor_name": factor_name, + "label": str(metadata.get(factor_name, {}).get("label", factor_name)), + "direction": direction_by_factor.get( + factor_name, 1 if metric.rank_ic >= 0 else -1 + ), + "score": _finite_or_none(metric.composite_score), + "ic_mean": _finite_or_none(metric.rank_ic), + "ir": _finite_or_none(metric.ir), + "coverage": _finite_or_none(metric.coverage), + "turnover": _finite_or_none(metric.turnover), + "spread_return": None, + "spread_sharpe": None, + "selected": factor_name in selected_factors, + "excluded_reason": None if factor_name in selected_factors else "not_selected", + }) + factors = pl.DataFrame(factor_rows) + + correlation = compute_rank_correlation(last_train, request.factor_names) + correlation_rows = [] + for row_id, left in enumerate(correlation.factor_names): + for column_id, right in enumerate(correlation.factor_names): + count = int(correlation.pair_counts[row_id][column_id]) + correlation_rows.append({ + "factor_x": left, + "factor_y": right, + "rho": ( + float(correlation.matrix[row_id][column_id]) if count > 0 else None + ), + "pair_count": count, + }) + correlation_frame = pl.DataFrame(correlation_rows) + + benchmark_by_id = { + benchmark_candidate(strategy_id).candidate_id: benchmark_candidate(strategy_id) + for strategy_id in request.strategy_ids + } + candidate_by_id: dict[str, MiningCandidate] = {} + for fold in result.folds: + for candidate in fold.candidates: + candidate_by_id.setdefault(candidate.candidate_id, candidate) + fold_rows: list[dict[str, Any]] = [] + for fold, nested_fold in zip(result.folds, nested, strict=False): + selected = ( + candidate_by_id.get(fold.selected_candidate_id) + if fold.selected_candidate_id is not None + else None + ) + evaluation = fold.outer_evaluation + fold_rows.append(_fold_row( + fold.outer_index, + selected, + nested_fold.outer, + evaluation, + regime_state="overall", + n_dates=len(nested_fold.outer.test_labels), + reason=fold.error, + )) + for candidate_id, cross_evaluation in fold.cross_evaluations: + fold_rows.append(_fold_row( + fold.outer_index, + candidate_by_id.get(candidate_id), + nested_fold.outer, + cross_evaluation, + regime_state="overall", + n_dates=len(nested_fold.outer.test_labels), + reason=None, + evaluation_kind="cross", + )) + for candidate_id, benchmark_evaluation in fold.benchmark_evaluations: + fold_rows.append(_fold_row( + fold.outer_index, + benchmark_by_id.get(candidate_id), + nested_fold.outer, + benchmark_evaluation, + regime_state="overall", + n_dates=len(nested_fold.outer.test_labels), + reason=None, + evaluation_kind="benchmark", + )) + if ( + selected is None + or evaluation is None + or evaluation.error is not None + or not request.require_regime + ): + continue + test = _panel_for_dates(panel, nested_fold.outer.test_labels) + for state in ("strong", "range", "weak"): + _raise_if_cancelled(cancel_check) + regime_evaluation = evaluator.evaluate_test( + test, + selected.definition(), + regime_state=state, + ) + n_dates = _regime_date_count( + panel, + nested_fold.outer, + state, + evaluator.data_dir, + ) + fold_rows.append(_fold_row( + fold.outer_index, + selected, + nested_fold.outer, + regime_evaluation, + regime_state=state, + n_dates=n_dates, + reason=None, + )) + folds = pl.DataFrame(fold_rows, schema_overrides={ + "total_return": pl.Float64, + "sharpe": pl.Float64, + "max_drawdown": pl.Float64, + "n_trades": pl.Int64, + }) + + candidate_rows = [] + overall_rows = [row for row in fold_rows if row["regime_state"] == "overall"] + winner_by_id = { + fold.selected_candidate_id: candidate_by_id[fold.selected_candidate_id] + for fold in result.folds + if fold.selected_candidate_id is not None + and fold.selected_candidate_id in candidate_by_id + } + ranked_candidates = _rank_artifact_candidates( + winner_by_id.values(), + overall_rows, + limit=request.max_finalists, + ) + [benchmark_by_id[cid] for cid in sorted(benchmark_by_id)] + for candidate in ranked_candidates: + rows = [row for row in overall_rows if row["candidate_signature"] == candidate.candidate_id] + successful = [row for row in rows if not row["skipped"]] + returns = [row["total_return"] for row in successful if row["total_return"] is not None] + sharpes = [row["sharpe"] for row in successful if row["sharpe"] is not None] + drawdowns = [row["max_drawdown"] for row in successful if row["max_drawdown"] is not None] + trades = [row["n_trades"] for row in successful if row["n_trades"] is not None] + definition = candidate.definition() + candidate_rows.append({ + "signature": candidate.candidate_id, + "name": _candidate_name(candidate), + "kind": ( + "existing_strategy" + if candidate.kind == "existing_strategy" + else "factor_combination" + ), + "factor_names_json": json.dumps( + list(candidate.factor_names), ensure_ascii=False, separators=(",", ":") + ), + "strategy_id": candidate.strategy_id, + "definition_json": json.dumps( + definition, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ), + "regime_state": "overall", + "score": _mean_or_none(sharpes), + "oos_return": _mean_or_none(returns), + "oos_sharpe": _mean_or_none(sharpes), + "oos_max_drawdown": min(drawdowns) if drawdowns else None, + "oos_positive_fold_ratio": ( + sum(value > 0 for value in returns) / len(returns) if returns else None + ), + "oos_n_trades": sum(int(value) for value in trades) if trades else None, + "confidence": _confidence(request.profile), + "valid_folds": len(successful), + "skipped_folds": len(rows) - len(successful), + "promoted_candidate_id": None, + "published_strategy_id": None, + }) + candidates = pl.DataFrame(candidate_rows, schema_overrides={ + "score": pl.Float64, + "oos_return": pl.Float64, + "oos_sharpe": pl.Float64, + "oos_max_drawdown": pl.Float64, + "oos_positive_fold_ratio": pl.Float64, + "oos_n_trades": pl.Int64, + "strategy_id": pl.Utf8, + "promoted_candidate_id": pl.Utf8, + "published_strategy_id": pl.Utf8, + }) if candidate_rows else _empty_candidates_frame() + return { + "factors": factors, + "correlation": correlation_frame, + "candidates": candidates, + "folds": folds, + } + + +def _factor_metric( + train: pl.DataFrame, + factor_name: str, + target_column: str, +) -> FactorMetric: + scoped = train.select("date", "symbol", factor_name, target_column) + finite_factor = pl.col(factor_name).is_not_null() & pl.col(factor_name).is_finite() + eligible = scoped.filter( + finite_factor + & pl.col(target_column).is_not_null() + & pl.col(target_column).is_finite() + ) + coverage = ( + scoped.select(finite_factor.mean()).item() if scoped.height else 0.0 + ) + daily = ( + eligible.group_by("date") + .agg( + pl.corr( + pl.col(factor_name).rank(method="average"), + pl.col(target_column).rank(method="average"), + ).alias("ic") + ) + .filter(pl.col("ic").is_not_null() & pl.col("ic").is_finite()) + .sort("date") + ) + values = daily["ic"].to_numpy() if not daily.is_empty() else np.array([]) + mean = float(np.mean(values)) if values.size else 0.0 + std = float(np.std(values)) if values.size else 0.0 + ir = mean / std if std > 1e-12 else 0.0 + turnover = _top_quintile_turnover(eligible, factor_name, direction=1 if mean >= 0 else -1) + score = abs(ir) * float(coverage or 0.0) / (1.0 + turnover) + return FactorMetric( + factor_id=factor_name, + composite_score=round(score, 8), + ir=round(abs(ir), 8), + coverage=round(float(coverage or 0.0), 8), + turnover=round(turnover, 8), + rank_ic=round(mean, 8), + ) + + +def _top_quintile_turnover( + panel: pl.DataFrame, + factor_name: str, + *, + direction: int, +) -> float: + if panel.is_empty(): + return 1.0 + ranked = ( + panel.select("date", "symbol", factor_name) + .sort(["date", factor_name, "symbol"], descending=[False, direction < 0, False]) + .with_columns( + pl.col(factor_name).rank(method="average").over("date").alias("_rank"), + pl.len().over("date").alias("_count"), + ) + .filter( + pl.col("_rank") + > pl.col("_count") * (0.8 if direction > 0 else 0.0) + ) + ) + if direction < 0: + ranked = ranked.filter(pl.col("_rank") <= pl.col("_count") * 0.2) + holdings = [ + set(str(value) for value in daily["symbol"].to_list()) + for daily in ranked.partition_by("date", maintain_order=True) + ] + if len(holdings) < 2: + return 0.0 + values = [] + for previous, current in pairwise(holdings): + denominator = max(len(previous), len(current), 1) + values.append(1.0 - len(previous & current) / denominator) + return float(np.mean(values)) if values else 0.0 + + +def _fold_row( + fold_index: int, + candidate: MiningCandidate | None, + validation_fold, + evaluation: CandidateEvaluation | None, + *, + regime_state: str, + n_dates: int, + reason: str | None, + evaluation_kind: str = "selected", +) -> dict[str, Any]: + error = reason or (evaluation.error if evaluation is not None else None) + metrics = evaluation.metrics if evaluation is not None else {} + return { + "candidate_signature": candidate.candidate_id if candidate is not None else None, + "evaluation_kind": evaluation_kind, + "fold": fold_index, + "label": f"OOS {fold_index + 1}", + "regime_state": regime_state, + "n_dates": n_dates, + "train_start": validation_fold.train_start, + "train_end": validation_fold.train_end, + "test_start": validation_fold.test_start, + "test_end": validation_fold.test_end, + "selected_factors_json": json.dumps( + list(candidate.factor_names) if candidate is not None else [], + ensure_ascii=False, + separators=(",", ":"), + ), + "total_return": _finite_or_none(metrics.get("total_return")), + "sharpe": _finite_or_none(metrics.get("sharpe")), + "max_drawdown": _finite_or_none(metrics.get("max_drawdown")), + "n_trades": _int_or_none(metrics.get("n_trades")), + "skipped": evaluation is None or error is not None, + "reason": error, + } + + +def _regime_date_count( + panel: pl.DataFrame, + validation_fold, + regime_state: str, + data_dir: Path, +) -> int: + labels = tuple( + label + for label in _date_labels(panel) + if label <= validation_fold.test_end + ) + mask = StrategyBacktestService._build_regime_mask( + labels, + _REGIME_FILTERS[regime_state], + data_dir, + required_start=date.fromisoformat(validation_fold.test_start), + required_end=date.fromisoformat(validation_fold.test_end), + ) + if mask is None: + return 0 + return sum( + bool(allowed) + for label, allowed in zip(labels, mask, strict=True) + if validation_fold.test_start <= label <= validation_fold.test_end + ) + + +def _rank_artifact_candidates( + candidates: Sequence[MiningCandidate], + overall_rows: Sequence[Mapping[str, Any]], + *, + limit: int, +) -> list[MiningCandidate]: + return sorted( + candidates, + key=lambda candidate: _candidate_artifact_rank(candidate, overall_rows), + )[:limit] + + +def _candidate_artifact_rank( + candidate: MiningCandidate, + overall_rows: Sequence[Mapping[str, Any]], +) -> tuple[float, str]: + sharpes = [ + row.get("sharpe") + for row in overall_rows + if row.get("candidate_signature") == candidate.candidate_id + and not row.get("skipped") + and row.get("sharpe") is not None + ] + mean_sharpe = _mean_or_none(sharpes) + return ( + -(mean_sharpe if mean_sharpe is not None else float("-inf")), + candidate.candidate_id, + ) + + +def _candidate_name(candidate: MiningCandidate) -> str: + if candidate.kind == "existing_strategy": + return f"已有策略 · {candidate.strategy_id}" + return "因子组合 · " + " + ".join(candidate.factor_names) + + +def _empty_candidates_frame() -> pl.DataFrame: + return pl.DataFrame(schema={ + "signature": pl.Utf8, + "name": pl.Utf8, + "kind": pl.Utf8, + "factor_names_json": pl.Utf8, + "strategy_id": pl.Utf8, + "definition_json": pl.Utf8, + "regime_state": pl.Utf8, + "score": pl.Float64, + "oos_return": pl.Float64, + "oos_sharpe": pl.Float64, + "oos_max_drawdown": pl.Float64, + "oos_positive_fold_ratio": pl.Float64, + "oos_n_trades": pl.Int64, + "confidence": pl.Utf8, + "valid_folds": pl.Int64, + "skipped_folds": pl.Int64, + "promoted_candidate_id": pl.Utf8, + "published_strategy_id": pl.Utf8, + }) + + +def _atomic_write_parquet(frame: pl.DataFrame, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + frame.write_parquet(temporary) + with temporary.open("r+b") as stream: + os.fsync(stream.fileno()) + os.replace(temporary, path) + except OSError: + temporary.unlink(missing_ok=True) + raise + + +def _panel_for_dates(panel: pl.DataFrame, labels: Sequence[str]) -> pl.DataFrame: + return panel.filter( + pl.col("date").cast(pl.Utf8).str.slice(0, 10).is_in(list(labels)) + ) + + +def _date_labels(panel: pl.DataFrame) -> tuple[str, ...]: + if panel.is_empty(): + return () + return tuple( + panel.select(pl.col("date").cast(pl.Utf8).str.slice(0, 10).unique().sort()) + .to_series() + .to_list() + ) + + +def _raise_if_cancelled(cancel_check: CancelCheck | None) -> None: + if cancel_check is None: + return + cancelled = cancel_check() if callable(cancel_check) else cancel_check.is_set() + if cancelled: + raise MiningRuntimeCancelledError("mining cancelled") + + +def _optional_date(value: Any) -> date | None: + if value in (None, ""): + return None + if isinstance(value, date): + return value + try: + return date.fromisoformat(str(value)) + except ValueError as exc: + raise ValueError(f"invalid ISO date: {value!r}") from exc + + +def _bounded_float( + value: Any, + name: str, + minimum: float, + maximum: float, + *, + exclusive_min: bool = False, +) -> float: + try: + number = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be numeric") from exc + valid_min = number > minimum if exclusive_min else number >= minimum + if not math.isfinite(number) or not valid_min or number > maximum: + left = "(" if exclusive_min else "[" + raise ValueError(f"{name} must be in {left}{minimum}, {maximum}]") + return number + + +def _finite_or_none(value: Any) -> float | None: + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None + + +def _int_or_none(value: Any) -> int | None: + number = _finite_or_none(value) + return int(number) if number is not None else None + + +def _mean_or_none(values: Sequence[float]) -> float | None: + return float(np.mean(values)) if values else None + + +def _confidence(profile: str) -> str: + return {"exploratory": "low", "balanced": "standard", "strict": "high"}[profile] + + +def _budget_exhausted(result: MiningResult) -> bool: + if result.proxy_trials_used >= result.request.budget.max_proxy_trials: + return True + if result.trials_used >= result.request.budget.max_trials: + return True + return any("budget exhausted" in (fold.error or "") for fold in result.folds) diff --git a/backend/app/backtest/regime_alignment.py b/backend/app/backtest/regime_alignment.py new file mode 100644 index 0000000..3f0fe5f --- /dev/null +++ b/backend/app/backtest/regime_alignment.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import date +from typing import Any + +import numpy as np + +RegimePoint = tuple[str, float] + +REGIME_THREE_LEVEL_MAP = { + "strong": "strong", + "lean_strong": "strong", + "range": "range", + "lean_weak": "weak", + "weak": "weak", +} + + +def three_level_regime(state: str) -> str: + return REGIME_THREE_LEVEL_MAP.get(state, state) + + +def _date_text(value: object) -> str: + return str(value)[:10] + + +def _normalize_regime_point(value: Any) -> RegimePoint: + if isinstance(value, Mapping): + state = str(value.get("state", "")) + score = float(value.get("score", 0) or 0) + return state, score + if isinstance(value, (tuple, list)) and len(value) >= 2: + return str(value[0]), float(value[1] or 0) + raise ValueError("市场环境数据格式无效") + + +def align_regime_t_minus_one( + labels: Sequence[str], + regime_by_date: Mapping[object, Any], + required_start: date | None, + required_end: date | None, +) -> list[RegimePoint | None]: + """Align each label with the preceding label's regime without any I/O.""" + regime_map = { + _date_text(key): _normalize_regime_point(value) + for key, value in regime_by_date.items() + } + if not regime_map: + raise ValueError("市场环境数据为空, 请先在数据页完成市场环境计算后再回测") + + aligned: list[RegimePoint | None] = [None] * len(labels) + required_start_text = str(required_start) if required_start is not None else None + required_end_text = str(required_end) if required_end is not None else None + missing_dates: list[str] = [] + if labels and required_start_text is not None: + first_label = _date_text(labels[0]) + if first_label >= required_start_text and ( + required_end_text is None or first_label <= required_end_text + ): + raise ValueError( + f"市场环境数据覆盖不完整: 正式首日 {first_label} 缺少前一交易日环境, " + "请把前一交易日行情包含在预热区间" + ) + for index in range(1, len(labels)): + current_label = _date_text(labels[index]) + previous_label = _date_text(labels[index - 1]) + point = regime_map.get(previous_label) + if point is not None: + aligned[index] = point + continue + required = ( + (required_start_text is None or current_label >= required_start_text) + and (required_end_text is None or current_label <= required_end_text) + ) + if required: + missing_dates.append(previous_label) + + if missing_dates: + first_missing = missing_dates[0] + suffix = f" 等 {len(missing_dates)} 天" if len(missing_dates) > 1 else "" + raise ValueError( + f"市场环境数据覆盖不完整: 缺少前一交易日环境 {first_missing}{suffix}, " + "请先补算对应区间" + ) + return aligned + + +def build_regime_filter_mask( + labels: Sequence[str], + regime_filter: Mapping[str, Any] | None, + regime_by_date: Mapping[object, Any], + *, + required_start: date | None = None, + required_end: date | None = None, +) -> np.ndarray | None: + """Build a T-1 regime filter mask from caller-supplied regime data. + + States are matched against the raw five-level labels, so each regime + level can be filtered on its own. Callers that want the aggregated + three-level view must list the raw states explicitly, e.g. + ``["strong", "lean_strong"]`` for the strong bucket. + """ + if not regime_filter: + return None + allowed_states = { + str(state) + for state in (regime_filter.get("states") or []) + } + min_score = regime_filter.get("min_score") + if not allowed_states and min_score is None: + return None + + aligned = align_regime_t_minus_one( + labels, + regime_by_date, + required_start, + required_end, + ) + mask = np.ones(len(labels), dtype=bool) + for index, point in enumerate(aligned): + if point is None: + continue + state, score = point + mask[index] = ( + (not allowed_states or state in allowed_states) + and (min_score is None or score >= float(min_score)) + ) + return mask diff --git a/backend/app/backtest/strategy.py b/backend/app/backtest/strategy.py index 811b70c..2981a5f 100644 --- a/backend/app/backtest/strategy.py +++ b/backend/app/backtest/strategy.py @@ -20,11 +20,13 @@ import numpy as np import polars as pl from app.backtest.engine import BacktestEngine, MatcherConfig, SimResult, SimulationOptions +from app.backtest.fundamentals import FUNDAMENTAL_FACTOR_NAMES from app.backtest.matrix import ( MarketDataMatrix, MatrixCacheProfile, MatrixComputeCache, MatrixPipelineConfig, + MatrixPrewarmCancelledError, MatrixStrategyPipeline, apply_time_masks, build_market_matrix, @@ -59,7 +61,7 @@ _EXECUTION_COLUMNS = frozenset({ "symbol", "date", "open", "high", "low", "close", "volume", "name", "score", "signal_limit_up", "signal_limit_down", }) -_LIMIT_BASE_COLUMNS = frozenset({"raw_close", "raw_high"}) +_LIMIT_BASE_COLUMNS = frozenset({"raw_close", "raw_high", "raw_low"}) _INSTRUMENT_COLUMNS = frozenset({"name", "total_shares", "float_shares"}) @@ -81,6 +83,8 @@ class ResolvedFeaturePlan: warmup_bars: int full_feature_fallback: bool = False execution_backend: str = "polars_expr" + # 财务因子列不落 enriched 存储, 由 engine 在加载口按公告日门控附加。 + fundamental_columns: frozenset[str] = frozenset() def _merge_resolved_feature_plans( @@ -108,6 +112,7 @@ def _merge_resolved_feature_plans( warmup_bars=max(plan.warmup_bars for plan in plans), full_feature_fallback=any(plan.full_feature_fallback for plan in plans), execution_backend="matrix_native", + fundamental_columns=_union("fundamental_columns"), ) @@ -206,6 +211,9 @@ class StrategyDependencyResolver: warmup_bars=plan.warmup_bars, full_feature_fallback=full_fallback, execution_backend=strategy.execution_backend, + fundamental_columns=frozenset( + required_features & FUNDAMENTAL_FACTOR_NAMES + ), ) @staticmethod @@ -224,6 +232,18 @@ class StrategyDependencyResolver: required_features = set(strategy.required_features) required_features.update(strategy.matrix_strategy.required_fields()) + parameter_fields = getattr( + strategy.matrix_strategy, + "required_fields_for_params", + None, + ) + parameter_scoring: dict[str, float] = {} + if callable(parameter_fields): + parameter_scoring = { + str(name): 1.0 + for name in parameter_fields(params) + } + required_features.update(scoring_dependencies(parameter_scoring)) required_features.update(_basic_filter_dependencies(basic_filter)) scoring = effective_scoring(strategy.meta.get("scoring"), overrides) required_features.update(scoring_dependencies(scoring)) @@ -239,6 +259,7 @@ class StrategyDependencyResolver: 60, int(strategy.matrix_strategy.required_warmup_bars(params)), scoring_warmup_bars(scoring), + scoring_warmup_bars(parameter_scoring), ) matrix_columns = set(base_columns) | set(instrument_columns) | { "signal_limit_up", @@ -254,6 +275,9 @@ class StrategyDependencyResolver: warmup_bars=warmup_bars, full_feature_fallback=False, execution_backend="matrix_native", + fundamental_columns=frozenset( + required_features & FUNDAMENTAL_FACTOR_NAMES + ), ) @@ -346,10 +370,13 @@ def prewarm_matrix_cache( asset_type: str, latest_date: date, years: int = 5, + cancel_event: threading.Event | None = None, ) -> dict[str, object]: """Build the shared full-universe mmap outside a user backtest request.""" if years <= 0: raise ValueError("matrix cache prewarm years must be positive") + if cancel_event is not None and cancel_event.is_set(): + raise MatrixPrewarmCancelledError("matrix cache prewarm cancelled") profile = build_matrix_cache_profile( strategy_engine, asset_type, @@ -390,6 +417,7 @@ def prewarm_matrix_cache( cache_profile=profile, coverage_start=coverage_start, coverage_end=latest_date, + cancel_event=cancel_event, ) result = { "asset_type": asset_type, @@ -1696,14 +1724,7 @@ class StrategyBacktestService: required_start: date | None = None, required_end: date | None = None, ) -> np.ndarray | None: - """构造逐日 regime mask。强制 T-1 防未来函数: regime[T-1] 决定 entry[T]。 - - timestamp_labels[i] 的入场资格 = 它的"前一交易日"的 regime 是否满足条件。 - "前一交易日"用 timestamp_labels 自身的顺序确定(回测时间轴上的前一天)。 - 边界: 首日无前一日环境 → 默认允许(不阻断)。 - regime_filter 为 None 时返回 None(不过滤)。启用过滤后缺少正式区间所需的 - 环境数据则 fail-closed, 避免界面显示已过滤但实际静默放行。 - """ + """构造逐日 T-1 regime mask, 保留历史静态入口兼容调用方。""" if not regime_filter: return None allowed_states = set(regime_filter.get("states") or []) @@ -1713,52 +1734,25 @@ class StrategyBacktestService: if data_dir is None: raise ValueError("市场环境过滤不可用: 未找到环境数据目录") + from app.backtest.regime_alignment import build_regime_filter_mask from app.services import regime_builder + regime_df = regime_builder.load_regime_history(data_dir) - if regime_df.is_empty(): - raise ValueError("市场环境数据为空, 请先在数据页完成市场环境计算后再回测") - - # 构建 date(ISO) → (state, score) 映射 - regime_map: dict[str, tuple[str, int]] = {} - for r in regime_df.iter_rows(named=True): - d = r.get("date") - ds = str(d)[:10] if d is not None else None - if ds: - regime_map[ds] = (str(r.get("state", "")), int(r.get("score", 0) or 0)) - - # 对每个 label, 找它的前一交易日的 regime(timestamp_labels 顺序里的前一天) - n = len(timestamp_labels) - mask = np.ones(n, dtype=bool) # 默认允许 - required_start_text = str(required_start) if required_start is not None else None - required_end_text = str(required_end) if required_end is not None else None - missing_dates: list[str] = [] - for i in range(1, n): - current_label = timestamp_labels[i][:10] - prev_label = timestamp_labels[i - 1][:10] - entry = regime_map.get(prev_label) - if entry is None: - required = ( - (required_start_text is None or current_label >= required_start_text) - and (required_end_text is None or current_label <= required_end_text) - ) - if required: - missing_dates.append(prev_label) - continue - state, score = entry - ok = True - if allowed_states and state not in allowed_states: - ok = False - if min_score is not None and score < min_score: - ok = False - mask[i] = ok - if missing_dates: - first_missing = missing_dates[0] - suffix = f" 等 {len(missing_dates)} 天" if len(missing_dates) > 1 else "" - raise ValueError( - f"市场环境数据覆盖不完整: 缺少前一交易日环境 {first_missing}{suffix}, " - "请先补算对应区间" - ) - return mask + regime_by_date = { + row["date"]: { + "state": row.get("state", ""), + "score": row.get("score", 0), + } + for row in regime_df.iter_rows(named=True) + if row.get("date") is not None + } + return build_regime_filter_mask( + timestamp_labels, + regime_filter, + regime_by_date, + required_start=required_start, + required_end=required_end, + ) def _build_candidate_filter_mask( self, diff --git a/backend/app/backtest/worker.py b/backend/app/backtest/worker.py index 1da69c6..f4fb102 100644 --- a/backend/app/backtest/worker.py +++ b/backend/app/backtest/worker.py @@ -22,6 +22,9 @@ class BacktestWorkerError(RuntimeError): """Raised when a spawned worker fails before returning a task result.""" +_CANCEL_GRACE_SECONDS = 5.0 + + class _PeakRssSampler: """Track whole-task and resettable phase RSS peaks with one sampling thread.""" @@ -136,6 +139,10 @@ def make_worker_task(kind: str, data_dir: Path, config) -> dict[str, Any]: encoded = asdict(config) encoded["start"] = config.start.isoformat() encoded["end"] = config.end.isoformat() + elif kind == "mining": + if not isinstance(config, dict): + raise TypeError("mining worker config must be a dict") + encoded = dict(config) else: raise ValueError(f"unsupported worker task kind: {kind}") return { @@ -165,8 +172,8 @@ def _worker_entry(task: dict[str, Any], event_queue, cancel_event) -> None: from app.backtest.engine import BacktestEngine from app.backtest.optimizer import StrategyOptimizer from app.backtest.strategy import StrategyBacktestService - from app.strategy.engine import StrategyEngine from app.strategy import config as strategy_config + from app.strategy.engine import StrategyEngine from app.tickflow.repository import DataStore, KlineRepository data_dir = Path(task["data_dir"]) @@ -201,6 +208,18 @@ def _worker_entry(task: dict[str, Any], event_queue, cancel_event) -> None: optimizer = StrategyOptimizer(service, strategy_engine) walkforward = WalkForwardService(optimizer, service, strategy_engine) result = walkforward.run(config, _progress, cancel_event) + elif kind == "mining": + from app.backtest.mining_runtime import run_mining_runtime + + result = run_mining_runtime( + task["config"], + data_dir=data_dir, + service=service, + strategy_engine=strategy_engine, + progress_cb=_progress, + cancel_check=cancel_event, + rss_sampler=sampler, + ) else: raise ValueError(f"unsupported worker task kind: {kind}") @@ -261,11 +280,20 @@ def run_worker_task( result: dict[str, Any] | None = None failure: dict[str, Any] | None = None ipc_started = time.perf_counter() + cancel_started: float | None = None try: while result is None and failure is None: if cancel_event is not None and cancel_event.is_set(): process_cancel.set() + if cancel_started is None: + cancel_started = time.monotonic() + elif time.monotonic() - cancel_started >= _CANCEL_GRACE_SECONDS: + process.terminate() + process.join(timeout=5.0) + raise BacktestWorkerError( + "backtest worker did not stop within 5 seconds after cancellation" + ) try: message = events.get(timeout=0.1) except queue.Empty: diff --git a/backend/app/enriched_generation.py b/backend/app/enriched_generation.py new file mode 100644 index 0000000..87fa81c --- /dev/null +++ b/backend/app/enriched_generation.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +import json +import os +import threading +import time +import uuid +import weakref +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any, BinaryIO + +import polars as pl + + +class EnrichedGenerationUnavailableError(RuntimeError): + """The enriched dataset has no stable generation available for readers.""" + + +_WRITER_LOCKS_GUARD = threading.Lock() +_WRITER_LOCKS: dict[tuple[str, str], threading.RLock] = {} +_ACTIVE_PUBLICATIONS: weakref.WeakValueDictionary[str, EnrichedPublication] = ( + weakref.WeakValueDictionary() +) + + +def _marker_path(data_dir: Path, asset_type: str) -> Path: + return Path(data_dir) / f".matrix_generation_{asset_type}.json" + + +def _writer_lock(data_dir: Path, asset_type: str) -> threading.RLock: + key = (str(Path(data_dir).resolve()), asset_type) + with _WRITER_LOCKS_GUARD: + return _WRITER_LOCKS.setdefault(key, threading.RLock()) + + +def _read_marker(path: Path) -> dict[str, Any] | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return None + except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise EnrichedGenerationUnavailableError( + "enriched data generation marker is invalid" + ) from exc + if not isinstance(payload, dict): + raise EnrichedGenerationUnavailableError( + "enriched data generation marker is invalid" + ) + return payload + + +def _fsync_directory(path: Path) -> None: + if os.name == "nt": + return + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _write_marker(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + with temporary.open("x", encoding="utf-8", newline="\n") as stream: + json.dump(payload, stream, separators=(",", ":")) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + _fsync_directory(path.parent) + finally: + temporary.unlink(missing_ok=True) + + +def _unlock_file(stream: BinaryIO) -> None: + if os.name == "nt": + import msvcrt + + stream.seek(0) + msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) + return + import fcntl + + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + + +def _try_lock_file(stream: BinaryIO) -> None: + if os.name == "nt": + import msvcrt + + stream.seek(0) + try: + msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) + except OSError as exc: + raise EnrichedGenerationUnavailableError( + "another enriched publication is active" + ) from exc + return + import fcntl + + try: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + raise EnrichedGenerationUnavailableError( + "another enriched publication is active" + ) from exc + + +@contextmanager +def _exclusive_generation_lock(data_dir: Path, asset_type: str) -> Iterator[None]: + lock_path = Path(data_dir) / f".matrix_generation_{asset_type}.lock" + lock_path.parent.mkdir(parents=True, exist_ok=True) + with ( + _writer_lock(data_dir, asset_type), + lock_path.open("a+b") as stream, + ): + stream.seek(0, os.SEEK_END) + if stream.tell() == 0: + stream.write(b"0") + stream.flush() + _try_lock_file(stream) + try: + yield + finally: + _unlock_file(stream) + + +def _process_is_alive(pid: Any) -> bool: + if not isinstance(pid, int) or pid <= 0: + return False + if pid == os.getpid(): + return True + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except (OSError, PermissionError): + return True + return True + + +def _ready_payload(generation: str) -> dict[str, Any]: + return { + "state": "ready", + "generation": generation, + "updated_at_ns": time.time_ns(), + } + + +def get_enriched_generation( + data_dir: Path, + asset_type: str = "stock", + *, + initialize: bool = True, +) -> str: + path = _marker_path(data_dir, asset_type) + payload = _read_marker(path) + if payload is None: + if not initialize: + raise EnrichedGenerationUnavailableError( + "enriched data generation marker is unavailable" + ) + with _exclusive_generation_lock(data_dir, asset_type): + payload = _read_marker(path) + if payload is None: + generation = uuid.uuid4().hex + _write_marker(path, _ready_payload(generation)) + return generation + state = payload.get("state", "ready") + generation = payload.get("generation") + if state != "ready" or not isinstance(generation, str) or not generation: + raise EnrichedGenerationUnavailableError( + "enriched data is being published; retry after the update finishes" + ) + return generation + + +def enriched_publication_incomplete( + data_dir: Path, + asset_type: str = "stock", +) -> bool: + try: + payload = _read_marker(_marker_path(data_dir, asset_type)) + except EnrichedGenerationUnavailableError: + return True + if payload is None: + return False + return ( + payload.get("state", "ready") != "ready" + or not isinstance(payload.get("generation"), str) + or not payload["generation"] + ) + + +def bump_enriched_generation(data_dir: Path, asset_type: str = "stock") -> str: + path = _marker_path(data_dir, asset_type) + with _exclusive_generation_lock(data_dir, asset_type): + current = _read_marker(path) + if current is not None and current.get("state", "ready") != "ready": + raise EnrichedGenerationUnavailableError( + "cannot bump an incomplete enriched publication" + ) + generation = uuid.uuid4().hex + _write_marker(path, _ready_payload(generation)) + return generation + + +class EnrichedPublication: + """Publish one logical enriched write batch under a stable generation token.""" + + def __init__( + self, + data_dir: Path, + asset_type: str = "stock", + *, + recover: bool = False, + ) -> None: + self.data_dir = Path(data_dir) + self.asset_type = asset_type + self.recover = recover + self._publishing = False + self._changed = False + self._base_generation: str | None = None + self._publication_id = uuid.uuid4().hex + + def begin(self) -> None: + with _exclusive_generation_lock(self.data_dir, self.asset_type): + self._claim_or_verify() + + def mark_changed(self) -> None: + if not self._publishing: + raise RuntimeError("enriched publication has not started") + self._changed = True + + def abandon(self) -> None: + if not self._publishing or self._changed: + return + path = _marker_path(self.data_dir, self.asset_type) + with _exclusive_generation_lock(self.data_dir, self.asset_type): + current = _read_marker(path) + if current is not None and current.get("publication_id") == self._publication_id: + _write_marker(path, _ready_payload(str(self._base_generation))) + self._publishing = False + + def write_parquet(self, df: pl.DataFrame, out: Path) -> None: + out.parent.mkdir(parents=True, exist_ok=True) + temporary = out.with_name(f".{out.name}.{uuid.uuid4().hex}.tmp") + try: + df.write_parquet(temporary) + with temporary.open("r+b") as stream: + stream.flush() + os.fsync(stream.fileno()) + with _exclusive_generation_lock(self.data_dir, self.asset_type): + self._claim_or_verify() + os.replace(temporary, out) + _fsync_directory(out.parent) + self._changed = True + finally: + temporary.unlink(missing_ok=True) + + def commit(self) -> str | None: + if not self._changed: + return None + path = _marker_path(self.data_dir, self.asset_type) + with _exclusive_generation_lock(self.data_dir, self.asset_type): + current = _read_marker(path) + if current is None or current.get("publication_id") != self._publication_id: + raise EnrichedGenerationUnavailableError( + "enriched publication ownership was lost" + ) + generation = uuid.uuid4().hex + _write_marker(path, _ready_payload(generation)) + self._publishing = False + return generation + + def _claim_or_verify(self) -> None: + path = _marker_path(self.data_dir, self.asset_type) + try: + current = _read_marker(path) + except EnrichedGenerationUnavailableError: + if not self.recover: + raise + current = None + if self._publishing: + if current is None or current.get("publication_id") != self._publication_id: + raise EnrichedGenerationUnavailableError( + "enriched publication ownership was lost" + ) + return + _ACTIVE_PUBLICATIONS[self._publication_id] = self + if current is not None and current.get("state", "ready") != "ready": + current_id = current.get("publication_id") + current_owner = _ACTIVE_PUBLICATIONS.get(str(current_id)) + owner_pid = current.get("owner_pid") + if current_owner is not None or ( + owner_pid != os.getpid() and _process_is_alive(owner_pid) + ): + raise EnrichedGenerationUnavailableError( + "another enriched publication is active" + ) + if not self.recover: + raise EnrichedGenerationUnavailableError( + "another enriched publication is incomplete" + ) + generation = None if current is None else current.get("generation") + if not isinstance(generation, str) or not generation: + generation = uuid.uuid4().hex + self._base_generation = generation + _write_marker(path, { + "state": "publishing", + "generation": generation, + "publication_id": self._publication_id, + "owner_pid": os.getpid(), + "updated_at_ns": time.time_ns(), + }) + self._publishing = True diff --git a/backend/app/indicators/pipeline.py b/backend/app/indicators/pipeline.py index 24ade27..99ea33f 100644 --- a/backend/app/indicators/pipeline.py +++ b/backend/app/indicators/pipeline.py @@ -22,6 +22,10 @@ from pathlib import Path import polars as pl from app.config import settings +from app.enriched_generation import ( + EnrichedPublication, + enriched_publication_incomplete, +) from app.market_time import cn_today from app.parquet import scan_daily_parquet, scan_enriched_parquet, scan_parquet_compat from app.price_limits import ( @@ -350,7 +354,12 @@ def _resolve_needed(needed: set[str] | None) -> set[str]: return want -def compute_indicators(df: pl.DataFrame, needed: set[str] | None = None) -> pl.DataFrame: +def compute_indicators( + df: pl.DataFrame, + needed: set[str] | None = None, + *, + assume_sorted: bool = False, +) -> pl.DataFrame: """从 OHLCV 数据计算全套技术指标。 输入必须包含: symbol, date, open, high, low, close, volume @@ -370,7 +379,7 @@ def compute_indicators(df: pl.DataFrame, needed: set[str] | None = None) -> pl.D want = _resolve_needed(needed) - df = df.sort(["symbol", "date"]) + df = df if assume_sorted else df.sort(["symbol", "date"]) # Pass 1: 均线 + EMA + MACD 基础 + BOLL 基础 + KDJ 基础 + ATR 基础 + 量价 + 极值 prev_close = pl.col("close").shift(1).over("symbol") @@ -670,7 +679,7 @@ def compute_limit_signals( signal_limit_down_recovery (跌停翘板) signal_broken_limit_up (炸板: 最高价触及涨停价但收盘未封住) - 输入必须包含: symbol, date, raw_close, raw_high, open, high, low, close, + 输入必须包含: symbol, date, raw_close, raw_high, raw_low, open, high, low, close, change_pct, vol_ratio_5d。 """ if df.is_empty(): @@ -879,9 +888,10 @@ def compute_limit_signals( pl.when( pl.col("_prev_raw_close").is_not_null() & (pl.col("_prev_raw_close") > 0) + & (pl.col("raw_low") > 0) ).then( (~pl.col("signal_limit_down").fill_null(False)) # 最终没跌停 - & (pl.col("low") <= pl.col("_effective_limit_down") + 0.005) # 曾触及跌停 + & (pl.col("raw_low") <= pl.col("_effective_limit_down") + 0.005) # 曾触及跌停(原始价口径, 跌停价为原始价基准) & (pl.col("close") > pl.col("open")) # 收阳 ).otherwise(None).cast(pl.Boolean) .alias("signal_limit_down_recovery") @@ -1045,6 +1055,11 @@ def run_pipeline(data_dir: Path | None = None, t0 = _t.perf_counter() d = Path(data_dir or settings.data_dir) + if enriched_publication_incomplete(d, "stock"): + logger.warning("检测到未完成的 enriched 发布,改为全量重建") + symbols = None + new_dates_only = False + publication = EnrichedPublication(d, "stock", recover=True) daily_dir = d / "kline_daily" enriched_base = d / "kline_daily_enriched" factor_path = d / "adj_factor" / "all.parquet" @@ -1134,7 +1149,7 @@ def run_pipeline(data_dir: Path | None = None, out = enriched_base / f"date={ds}" / "part.parquet" out.parent.mkdir(parents=True, exist_ok=True) date_df = _select_storage_cols(date_df).sort(["symbol"]) - date_df.write_parquet(out) + publication.write_parquet(date_df, out) written += date_df.height t_write_new = _t.perf_counter() logger.info("增量写入: %.2fs, %d 行", t_write_new - t_new, written) @@ -1166,10 +1181,11 @@ def run_pipeline(data_dir: Path | None = None, existing = existing.filter(~pl.col("symbol").is_in(list(sym_set))) date_df_storage = pl.concat([existing, date_df_storage], how="diagonal_relaxed") date_df_storage = date_df_storage.sort(["symbol"]) - date_df_storage.write_parquet(out) + publication.write_parquet(date_df_storage, out) written += date_df.height logger.info("除权重算: %d 只, 共写入 %d 行", len(sym_set), written) + publication.commit() t_done = _t.perf_counter() logger.info("增量管道完成: %.2fs, %d 行", t_done - t0, written) return written @@ -1267,7 +1283,7 @@ def run_pipeline(data_dir: Path | None = None, existing = existing.filter(~pl.col("symbol").is_in(batch_syms)) date_df_storage = pl.concat([existing, date_df_storage], how="diagonal_relaxed") date_df_storage = date_df_storage.sort(["symbol"]) - date_df_storage.write_parquet(out) + publication.write_parquet(date_df_storage, out) written += date_df_storage.height else: # 全量模式: 缓冲到 date_buffers, 最后一次性写入 @@ -1308,11 +1324,12 @@ def run_pipeline(data_dir: Path | None = None, out = base / f"date={ds}" / "part.parquet" out.parent.mkdir(parents=True, exist_ok=True) merged = pl.concat(dfs, how="diagonal_relaxed").sort(["symbol"]) - merged.write_parquet(out) + publication.write_parquet(merged, out) date_buffers.clear() gc.collect() + publication.commit() t_done = _t.perf_counter() adj_label = "含复权" if not factors.is_empty() else "无复权" logger.info("enriched 完成 [%s]: %.2fs, 共 %d 行, %s", @@ -1339,9 +1356,13 @@ def _load_recent_history(enriched_base: Path, symbols: list[str], days: int) -> from datetime import date, timedelta cutoff = date.today() - timedelta(days=days + 30) # 多读 30 天余量 + cast_options = pl.ScanCastOptions(integer_cast="allow-float") try: lf = ( - scan_enriched_parquet(str(enriched_base / "**" / "*.parquet"), cast_options=_cast) + scan_enriched_parquet( + str(enriched_base / "**" / "*.parquet"), + cast_options=cast_options, + ) .filter( (pl.col("symbol").is_in(symbols)) & (pl.col("date") >= cutoff) @@ -1814,10 +1835,10 @@ def _compute_limit_signals_today(df: pl.DataFrame, instruments: pl.DataFrame) -> # 跌停翘板 pl.when(no_price_limit) .then(False) - .when(valid_prev_raw | has_authoritative_down) + .when((valid_prev_raw | has_authoritative_down) & (pl.col("raw_low") > 0)) .then( (~is_limit_down.fill_null(True)) - & (pl.col("low") <= effective_limit_down + 0.005) + & (pl.col("raw_low") <= effective_limit_down + 0.005) & (pl.col("close") > pl.col("open")) ).otherwise(None).cast(pl.Boolean) .alias("signal_limit_down_recovery"), diff --git a/backend/app/jobs/daily_pipeline.py b/backend/app/jobs/daily_pipeline.py index 6a70541..34f2698 100644 --- a/backend/app/jobs/daily_pipeline.py +++ b/backend/app/jobs/daily_pipeline.py @@ -542,6 +542,28 @@ def run_now( stage_errors.append(f"compute_regime: {e}") skipped.append("regime") + # Step 2.7: 市场主线(概念/行业涨停梯队聚合) 增量计算 — regime 同开关。 + # 只窄扫连板 >=1 的行, 增量通常 1 天, 开销可忽略。软失败: 不阻断主管道。 + mainline_rows = 0 + if not _prefs_regime.get_pipeline_regime_enabled(): + skipped.append("mainline") + else: + try: + emit("compute_mainline", 93, "计算市场主线…") + from app.services import market_mainline + for _kind in ("concept", "industry"): + rows = market_mainline.compute_mainline_incremental( + repo, repo.store.data_dir, kind=_kind + ) + mainline_rows += rows.height if not rows.is_empty() else 0 + if mainline_rows: + logger.info("compute_mainline: %d rows", mainline_rows) + emit("compute_mainline", 94, f"市场主线 {mainline_rows} 行") + except Exception as e: + logger.warning("compute_mainline failed (soft): %s", e) + stage_errors.append(f"compute_mainline: {e}") + skipped.append("mainline") + # Step 3: 刷新视图 emit("refresh_views", 95, "刷新 DuckDB 视图…") _refresh_views(repo) @@ -561,6 +583,7 @@ def run_now( "etf_adj_factor_symbols": etf_adj_symbols, "minute_rows": written_minute, "regime_days": regime_days, + "mainline_rows": mainline_rows, "lagging_symbols": len(lagging_symbols), "skipped_stages": skipped, "stage_errors": stage_errors, @@ -626,37 +649,54 @@ def _refresh_instruments_view(repo: KlineRepository) -> None: logger.warning("refresh instruments view failed: %s", e) -def _run_tracked(fn, job_label: str) -> None: +def _run_tracked(fn, job_label: str) -> bool: """调度触发时包装 JobStore 跟踪，确保同步历史有记录。 单飞: 若已有活跃(pending∨running)任务(手动同步中), 本次调度直接跳过, 不并发。 重任务执行槽: 再挡一层僵尸并发(reap 后线程仍活时不得并行写 parquet)。 + 返回 True 仅表示任务已成功并且执行槽已释放。 """ from app.services.pipeline_jobs import job_store, release_run_slot, try_acquire_run_slot job_id, is_new = job_store.create() if not is_new: logger.info("scheduled %s 跳过: 已有活跃任务在运行 (job_id=%s)", job_label, job_id) - return + return False if not try_acquire_run_slot(): logger.warning("scheduled %s 跳过: 重任务执行槽被占用(疑似上次任务卡死)", job_label) job_store.fail(job_id, f"scheduled {job_label} skipped: 已有数据任务在运行") - return + return False def progress(stage: str, pct: int, msg: str, stage_pct: int | None = None, skip_log: bool = False) -> None: job_store.progress(job_id, stage, pct, msg, stage_pct=stage_pct, skip_log=skip_log) + succeeded = False try: job_store.start(job_id) result = fn(on_progress=progress) job_store.succeed(job_id, result) + succeeded = True logger.info("scheduled %s completed: job_id=%s", job_label, job_id) except Exception: logger.exception("scheduled %s failed: job_id=%s", job_label, job_id) job_store.fail(job_id, f"scheduled {job_label} failed") finally: release_run_slot() + return succeeded + + +def _scheduled_pipeline_task(pipeline_fn) -> None: + """Run weekly mining only after the tracked daily pipeline has fully succeeded.""" + if not _run_tracked(pipeline_fn, "daily_pipeline"): + return + try: + from app.services.mining_schedule import run_weekly_mining + + result = run_weekly_mining(_get_app_state()) + logger.info("scheduled mining result: %s", result) + except Exception: + logger.exception("scheduled mining enqueue failed; daily pipeline remains succeeded") # ================================================================ @@ -916,7 +956,7 @@ def start_scheduler(repo: KlineRepository, capset: CapabilitySet) -> AsyncIOSche return result scheduler.add_job( - lambda: _run_tracked(_pipeline_then_refresh, "daily_pipeline"), + lambda: _scheduled_pipeline_task(_pipeline_then_refresh), trigger=CronTrigger(day_of_week="mon-fri", hour=sched["hour"], minute=sched["minute"], timezone="Asia/Shanghai"), diff --git a/backend/app/main.py b/backend/app/main.py index 09ea0f2..2beb09f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging import sys -import threading from contextlib import asynccontextmanager from pathlib import Path @@ -13,15 +12,42 @@ from fastapi.responses import FileResponse, JSONResponse from fastapi.staticfiles import StaticFiles from app import __version__ -from app.api import analysis, auth as auth_api, backtest, data, ext_data, financials, indices, intraday, kline, market_recap, monitor_rules, alerts, overview, pipeline, regime, rps, screener, settings as settings_api, signals, stock_analysis, strategy, watchlist +from app.api import ( + alerts, + analysis, + backtest, + data, + ext_data, + financials, + indices, + intraday, + kline, + market_recap, + mining, + monitor_rules, + overview, + pipeline, + regime, + rps, + screener, + signals, + stock_analysis, + strategy, + watchlist, +) +from app.api import auth as auth_api +from app.api import settings as settings_api from app.api.routes import router as core_router from app.config import settings +from app.enriched_generation import EnrichedGenerationUnavailableError from app.extensions.loader import ( configure_backend_extensions, current_extension_context, start_backend_extensions, ) from app.jobs import daily_pipeline +from app.services.matrix_prewarm_owner import MatrixCachePrewarmOwner +from app.services.mining_process_lock import MiningProcessLock from app.services.quote_service import QuoteService from app.tickflow import client as tf_client from app.tickflow.policy import detect_capabilities @@ -57,7 +83,7 @@ if not getattr(sys, "frozen", False): @asynccontextmanager -async def lifespan(app: FastAPI): +async def _application_lifespan(app: FastAPI): logger.info( "Tick Stock Panel v%s starting (mode=%s)", __version__, tf_client.current_mode(), @@ -76,9 +102,19 @@ async def lifespan(app: FastAPI): repo = KlineRepository(store) app.state.datastore = store app.state.repo = repo + from app.services.mining_manager import MiningJobManager + + mining_manager = MiningJobManager(store.data_dir) + recovered_mining_runs = mining_manager.recover_interrupted() + app.state.mining_manager = mining_manager + if recovered_mining_runs: + logger.warning("recovered %d interrupted mining runs", recovered_mining_runs) # 在接受回测请求前固定 managed generation，避免首批并发 worker 各自创建版本。 if settings.backtest_matrix_disk_cache_enabled: - repo.get_matrix_data_generation("stock") + try: + repo.get_matrix_data_generation("stock") + except EnrichedGenerationUnavailableError as exc: + logger.warning("enriched generation requires a full rebuild: %s", exc) # 指标异步预热标志: enriched 缓存在后台线程构建, 完成后置 True app.state.indicators_ready = False repo._on_warmup_done = lambda: setattr(app.state, "indicators_ready", True) # noqa: SLF001 @@ -188,51 +224,50 @@ async def lifespan(app: FastAPI): app.state.strategy_engine = strategy_engine logger.info("strategy engine loaded: %d strategies", len(strategy_engine.list_strategies())) - matrix_prewarm_lock = threading.Lock() - matrix_prewarm_running = False + matrix_prewarm_owner = MatrixCachePrewarmOwner() def _schedule_matrix_cache_prewarm() -> None: - nonlocal matrix_prewarm_running if ( not settings.backtest_matrix_disk_cache_enabled or not settings.backtest_matrix_cache_prewarm ): return - with matrix_prewarm_lock: - if matrix_prewarm_running: - logger.info("matrix cache prewarm already in progress, skip") - return - matrix_prewarm_running = True def _prewarm() -> None: - nonlocal matrix_prewarm_running + from app.backtest.engine import BacktestEngine + from app.backtest.matrix import MatrixPrewarmCancelledError + from app.backtest.strategy import prewarm_matrix_cache + from app.services.heavy_job_limiter import ( + HeavyJobCancelledError, + shared_heavy_job_limiter, + ) + try: latest = repo.latest_enriched_date("stock") if latest is None: logger.info("matrix cache prewarm skipped: no stock enriched data") return - from app.backtest.engine import BacktestEngine - from app.backtest.strategy import prewarm_matrix_cache - result = prewarm_matrix_cache( - BacktestEngine(repo), - strategy_engine, - asset_type="stock", - latest_date=latest, - years=settings.backtest_matrix_cache_prewarm_years, - ) + with shared_heavy_job_limiter.slot( + "normal", + cancel_event=matrix_prewarm_owner.cancel_event, + ): + result = prewarm_matrix_cache( + BacktestEngine(repo), + strategy_engine, + asset_type="stock", + latest_date=latest, + years=settings.backtest_matrix_cache_prewarm_years, + cancel_event=matrix_prewarm_owner.cancel_event, + ) logger.info("matrix cache prewarm done: %s", result) + except (HeavyJobCancelledError, MatrixPrewarmCancelledError): + logger.info("matrix cache prewarm cancelled") except Exception: # noqa: BLE001 logger.exception("matrix cache prewarm failed") - finally: - with matrix_prewarm_lock: - matrix_prewarm_running = False - threading.Thread( - target=_prewarm, - name="matrix-cache-prewarm", - daemon=True, - ).start() + if not matrix_prewarm_owner.schedule(_prewarm): + logger.info("matrix cache prewarm already running or shutting down, skip") repo._on_refresh_done = _schedule_matrix_cache_prewarm # noqa: SLF001 if repo.enriched_ready: @@ -281,26 +316,44 @@ async def lifespan(app: FastAPI): extension_registry, ) - yield + try: + yield + finally: + repo._on_refresh_done = None # noqa: SLF001 + if not matrix_prewarm_owner.shutdown(timeout=5.0): + logger.warning("matrix cache prewarm did not stop within 5 seconds") + mmanager = getattr(app.state, "mining_manager", None) + if mmanager: + mmanager.shutdown() + if app.state.scheduler: + app.state.scheduler.shutdown(wait=False) + ps = getattr(app.state, "pull_scheduler", None) + if ps: + ps.stop() + fsc = getattr(app.state, "financial_scheduler", None) + if fsc: + fsc.stop() + qs = getattr(app.state, "quote_service", None) + if qs: + qs.stop() + dsvc = getattr(app.state, "depth_service", None) + if dsvc: + dsvc.stop_polling() + wbot = getattr(app.state, "wecom_bot_service", None) + if wbot: + wbot.stop() + logger.info("shutdown") - if app.state.scheduler: - app.state.scheduler.shutdown(wait=False) - ps = getattr(app.state, "pull_scheduler", None) - if ps: - ps.stop() - fsc = getattr(app.state, "financial_scheduler", None) - if fsc: - fsc.stop() - qs = getattr(app.state, "quote_service", None) - if qs: - qs.stop() - dsvc = getattr(app.state, "depth_service", None) - if dsvc: - dsvc.stop_polling() - wbot = getattr(app.state, "wecom_bot_service", None) - if wbot: - wbot.stop() - logger.info("shutdown") + +@asynccontextmanager +async def lifespan(app: FastAPI): + mining_process_lock = MiningProcessLock(settings.data_dir) + mining_process_lock.acquire() + try: + async with _application_lifespan(app): + yield + finally: + mining_process_lock.release() app = FastAPI( @@ -374,6 +427,7 @@ app.include_router(kline.router) app.include_router(watchlist.router) app.include_router(screener.router) app.include_router(backtest.router) +app.include_router(mining.router) app.include_router(intraday.router) app.include_router(indices.router) app.include_router(overview.router) diff --git a/backend/app/services/ext_data.py b/backend/app/services/ext_data.py index 9129541..8a6ff27 100644 --- a/backend/app/services/ext_data.py +++ b/backend/app/services/ext_data.py @@ -1,6 +1,7 @@ """扩展数据服务 — 配置管理 + 文件解析 + Parquet 存储。""" from __future__ import annotations +import copy import json import logging import re @@ -187,17 +188,48 @@ class ExtConfig: # 配置持久化 # --------------------------------------------------------------------------- +# load_all 进程内缓存: kline/screener/watchlist 等热路径每请求调用, 每次都 +# iterdir + 逐 config.json read_text+parse 纯重复; 以配置目录的 +# (目录名, mtime_ns, size) 签名失效 (新增/编辑/删除配置都会改变签名)。 +_load_all_cache: dict[str, tuple[tuple, list[ExtConfig]]] = {} + + +def _ext_config_dir_signature(base: Path) -> tuple | None: + """配置目录下所有 config.json 的 (目录名, mtime_ns, size) 签名; 出错返回 None (禁用缓存)。""" + try: + sig = [] + for d in sorted(base.iterdir()): + cp = d / "config.json" + if d.is_dir() and cp.exists(): + st = cp.stat() + sig.append((d.name, st.st_mtime_ns, st.st_size)) + return tuple(sig) + except Exception: # noqa: BLE001 + return None + + class ExtConfigStore: """扩展数据配置文件读写 — 每个表独立目录 data/ext/{config_id}/config.json。""" + # 与创建端点 CreateExtReq.id 的 pattern 一致; load_all 之外的 config_id + # 来自 URL path 参数, 必须先过白名单再拼路径, 防止 ../ 穿越删除。 + _VALID_ID = re.compile(r"^[a-zA-Z0-9_]+$") + def __init__(self, data_dir: Path) -> None: self._base = data_dir / "ext_data" def _config_path(self, config_id: str) -> Path: + if not self._VALID_ID.match(config_id): + raise ValueError(f"非法 config_id: {config_id!r}") return self._base / config_id / "config.json" def load_all(self) -> list[ExtConfig]: # 兼容旧版: 如果目录为空且旧配置文件存在则迁移 + sig = _ext_config_dir_signature(self._base) + if sig is not None: + cached = _load_all_cache.get(str(self._base)) + if cached is not None and cached[0] == sig: + return copy.deepcopy(cached[1]) if not self._base.exists() or not any(self._base.iterdir()): old = self._base.parent / "ext_configs.json" if not old.exists(): @@ -215,10 +247,16 @@ class ExtConfigStore: configs.append(ExtConfig.from_dict(raw)) except Exception as e: logger.warning("扩展表配置解析失败 %s: %s", cp, e) + if sig is not None and configs: + # 缓存存私有副本, 命中时返回深拷贝, 调用方改配置对象不会污染缓存。 + _load_all_cache[str(self._base)] = (sig, copy.deepcopy(configs)) return configs def get(self, config_id: str) -> ExtConfig | None: - cp = self._config_path(config_id) + try: + cp = self._config_path(config_id) + except ValueError: + return None if not cp.exists(): return None try: @@ -238,7 +276,10 @@ class ExtConfigStore: def delete(self, config_id: str) -> bool: import shutil - cp = self._config_path(config_id) + try: + cp = self._config_path(config_id) + except ValueError: + return False if not cp.exists(): return False shutil.rmtree(cp.parent, ignore_errors=True) diff --git a/backend/app/services/financial_sync.py b/backend/app/services/financial_sync.py index fe0b5a3..7b857eb 100644 --- a/backend/app/services/financial_sync.py +++ b/backend/app/services/financial_sync.py @@ -154,7 +154,7 @@ def _sync_table( ) -def _merge_share_history(*frames: pl.DataFrame) -> pl.DataFrame: +def _merge_report_history(*frames: pl.DataFrame) -> pl.DataFrame: valid = [ frame for frame in frames @@ -162,65 +162,74 @@ def _merge_share_history(*frames: pl.DataFrame) -> pl.DataFrame: ] if not valid: return pl.DataFrame() - return ( + merged = ( pl.concat(valid, how="diagonal_relaxed") .filter(pl.col("symbol").is_not_null() & pl.col("period_end").is_not_null()) - .unique(subset=["symbol", "period_end"], keep="last") - .sort(["symbol", "period_end"]) + ) + # 同一 (symbol, period_end) 多条时保留 announce_date 最新一条 (业绩修正以最新公告为准)。 + if "announce_date" in merged.columns: + merged = merged.sort(["symbol", "period_end", "announce_date"], nulls_last=True) + return merged.unique(subset=["symbol", "period_end"], keep="last").sort( + ["symbol", "period_end"] ) -def _sync_shares_for_symbols( +def _sync_history_table_for_symbols( + table: str, symbols: list[str], data_dir: Path, capset: CapabilitySet, ) -> int: - """首次拉全量股本历史，后续更新最新记录并补齐新增标的历史。""" - existing = get_financial_df(data_dir, "shares") + """历史累积同步: 保留已有各期记录, 仅拉最新期 + 为新标的补全量历史。 + + 与 shares 同一模式。若改为 latest_only 全量覆盖, 历史各期会在每次同步时 + 被冲掉, 财务因子将永远只有单期快照, 任何回测都是未来函数。 + """ + existing = get_financial_df(data_dir, table) if existing.is_empty() or not {"symbol", "period_end"} <= set(existing.columns): - return _sync_table("shares", symbols, data_dir, capset, latest_only=False) + return _sync_table(table, symbols, data_dir, capset, latest_only=False) existing_symbols = set(existing["symbol"].drop_nulls().to_list()) missing_symbols = [symbol for symbol in symbols if symbol not in existing_symbols] missing_history = ( - _fetch_table("shares", missing_symbols, capset, latest_only=False) + _fetch_table(table, missing_symbols, capset, latest_only=False) if missing_symbols else pl.DataFrame() ) current_symbols = [symbol for symbol in symbols if symbol in existing_symbols] - latest = _fetch_table("shares", current_symbols, capset, latest_only=True) - merged = _merge_share_history(existing, missing_history, latest) - return _write_table("shares", merged, data_dir) + latest = _fetch_table(table, current_symbols, capset, latest_only=True) + merged = _merge_report_history(existing, missing_history, latest) + return _write_table(table, merged, data_dir) def sync_metrics(data_dir: Path, capset: CapabilitySet) -> int: - """同步核心财务指标 (metrics)。""" + """同步核心财务指标 (metrics), 历史各期累积保留。""" symbols = _get_symbols(data_dir) - return _sync_table("metrics", symbols, data_dir, capset, latest_only=True) + return _sync_history_table_for_symbols("metrics", symbols, data_dir, capset) def sync_income(data_dir: Path, capset: CapabilitySet) -> int: - """同步利润表。""" + """同步利润表, 历史各期累积保留。""" symbols = _get_symbols(data_dir) - return _sync_table("income", symbols, data_dir, capset, latest_only=True) + return _sync_history_table_for_symbols("income", symbols, data_dir, capset) def sync_balance_sheet(data_dir: Path, capset: CapabilitySet) -> int: - """同步资产负债表。""" + """同步资产负债表, 历史各期累积保留。""" symbols = _get_symbols(data_dir) - return _sync_table("balance_sheet", symbols, data_dir, capset, latest_only=True) + return _sync_history_table_for_symbols("balance_sheet", symbols, data_dir, capset) def sync_cash_flow(data_dir: Path, capset: CapabilitySet) -> int: - """同步现金流量表。""" + """同步现金流量表, 历史各期累积保留。""" symbols = _get_symbols(data_dir) - return _sync_table("cash_flow", symbols, data_dir, capset, latest_only=True) + return _sync_history_table_for_symbols("cash_flow", symbols, data_dir, capset) def sync_shares(data_dir: Path, capset: CapabilitySet) -> int: """同步历史股本表。""" symbols = _get_symbols(data_dir) - return _sync_shares_for_symbols(symbols, data_dir, capset) + return _sync_history_table_for_symbols("shares", symbols, data_dir, capset) def sync_all(data_dir: Path, capset: CapabilitySet) -> dict[str, int]: @@ -232,10 +241,8 @@ def sync_all(data_dir: Path, capset: CapabilitySet) -> dict[str, int]: symbols = _get_symbols(data_dir) results: dict[str, int] = {} for table in FINANCIAL_TABLES: - results[table] = ( - _sync_shares_for_symbols(symbols, data_dir, capset) - if table == "shares" - else _sync_table(table, symbols, data_dir, capset, latest_only=True) + results[table] = _sync_history_table_for_symbols( + table, symbols, data_dir, capset ) # 同步完成后注册 DuckDB 视图 @@ -428,10 +435,8 @@ class FinancialScheduler: symbols = _get_symbols(self._data_dir) result: dict[str, int] = {} for t in FINANCIAL_TABLES: - result[t] = ( - _sync_shares_for_symbols(symbols, self._data_dir, self._capset) - if t == "shares" - else _sync_table(t, symbols, self._data_dir, self._capset, latest_only=True) + result[t] = _sync_history_table_for_symbols( + t, symbols, self._data_dir, self._capset ) self._record_sync(t) _refresh_financials_views(self._data_dir) diff --git a/backend/app/services/heavy_job_limiter.py b/backend/app/services/heavy_job_limiter.py new file mode 100644 index 0000000..65c7e88 --- /dev/null +++ b/backend/app/services/heavy_job_limiter.py @@ -0,0 +1,121 @@ +"""Weighted process-local limiter for memory-heavy jobs.""" + +from __future__ import annotations + +import threading +import time +from collections.abc import Iterator +from contextlib import contextmanager +from typing import ClassVar, Literal + +HeavyJobKind = Literal["normal", "mining"] + + +class HeavyJobLimitTimeoutError(TimeoutError): + """Raised when a heavy-job slot cannot be acquired before its deadline.""" + + +class HeavyJobCancelledError(RuntimeError): + """Raised when slot acquisition is cancelled while waiting.""" + + +class HeavyJobLimiter: + """A weighted limiter where normal jobs cost one slot and mining costs two.""" + + _WEIGHTS: ClassVar[dict[HeavyJobKind, int]] = {"normal": 1, "mining": 2} + + def __init__(self, capacity: int = 2, *, cancel_poll_interval: float = 0.05) -> None: + if capacity <= 0: + raise ValueError("capacity must be positive") + if cancel_poll_interval <= 0: + raise ValueError("cancel_poll_interval must be positive") + self.capacity = capacity + self._cancel_poll_interval = cancel_poll_interval + self._used = 0 + self._acquired = {"normal": 0, "mining": 0} + self._condition = threading.Condition() + + @property + def in_use(self) -> int: + with self._condition: + return self._used + + @property + def available(self) -> int: + with self._condition: + return self.capacity - self._used + + def acquire( + self, + kind: HeavyJobKind = "normal", + *, + timeout: float | None = None, + cancel_event: threading.Event | None = None, + ) -> bool: + """Wait for capacity and return ``False`` on cancellation or timeout.""" + weight = self._weight(kind) + if weight > self.capacity: + raise ValueError(f"{kind} requires {weight} slots, capacity is {self.capacity}") + if timeout is not None and timeout < 0: + raise ValueError("timeout must be non-negative") + + deadline = None if timeout is None else time.monotonic() + timeout + with self._condition: + while True: + if cancel_event is not None and cancel_event.is_set(): + return False + if self._used + weight <= self.capacity: + self._used += weight + self._acquired[kind] += 1 + return True + + remaining = None if deadline is None else deadline - time.monotonic() + if remaining is not None and remaining <= 0: + return False + wait_for = remaining + if cancel_event is not None: + wait_for = self._cancel_poll_interval + if remaining is not None: + wait_for = min(wait_for, remaining) + self._condition.wait(wait_for) + + def release(self, kind: HeavyJobKind = "normal") -> None: + """Return capacity previously acquired for ``kind``.""" + weight = self._weight(kind) + with self._condition: + if self._acquired[kind] == 0: + raise RuntimeError(f"cannot release unacquired {kind} capacity") + self._acquired[kind] -= 1 + self._used -= weight + self._condition.notify_all() + + @contextmanager + def slot( + self, + kind: HeavyJobKind = "normal", + *, + timeout: float | None = None, + cancel_event: threading.Event | None = None, + ) -> Iterator[HeavyJobLimiter]: + """Acquire weighted capacity for the duration of a ``with`` block.""" + acquired = self.acquire(kind, timeout=timeout, cancel_event=cancel_event) + if not acquired: + if cancel_event is not None and cancel_event.is_set(): + raise HeavyJobCancelledError(f"{kind} job was cancelled while waiting") + raise HeavyJobLimitTimeoutError(f"timed out waiting for {kind} job capacity") + try: + yield self + finally: + self.release(kind) + + @classmethod + def _weight(cls, kind: HeavyJobKind) -> int: + try: + return cls._WEIGHTS[kind] + except KeyError as exc: + raise ValueError(f"unsupported heavy job kind: {kind!r}") from exc + + +shared_heavy_job_limiter = HeavyJobLimiter(capacity=2) +# Short alias for entry points that prefer the existing module-singleton naming style. +heavy_job_limiter = shared_heavy_job_limiter diff --git a/backend/app/services/kline_sync.py b/backend/app/services/kline_sync.py index 3aa00ca..46789c1 100644 --- a/backend/app/services/kline_sync.py +++ b/backend/app/services/kline_sync.py @@ -15,7 +15,7 @@ import polars as pl from app.data_providers.base import AssetType from app.indicators.pipeline import filter_halt_days -from app.market_time import cn_now +from app.market_time import CN_TZ, cn_now, cn_today from app.services import preferences from app.tickflow.capabilities import Cap, CapabilitySet from app.tickflow.client import get_client @@ -266,7 +266,9 @@ def sync_daily_by_quotes(repo: KlineRepository) -> int: if df.is_empty(): return 0 - today = _date.today() + # 分区日期用北京交易日 (与 quote_service._build_daily 的 cn_today 一致), + # 避免 UTC 服务器在盘中把日分区写成服务器本地日期。 + today = cn_today() daily_df = df.with_columns(pl.lit(today).cast(pl.Date).alias("date")) # 过滤停牌 (open/high 为 0; close 可能被填充为前收盘价, 不能用全零判断) @@ -833,8 +835,10 @@ def fetch_minute_single( ) -> pl.DataFrame: """实时拉取单股单日分钟 K(不写入本地)。优先自定义分钟源, 回退 TickFlow。""" from datetime import datetime - start_time = datetime(trade_date.year, trade_date.month, trade_date.day, 9, 25, 0) - end_time = datetime(trade_date.year, trade_date.month, trade_date.day, 15, 5, 0) + # 北京时间窗口必须带时区: naive datetime 会被 .timestamp() 按服务器本地时区解释, + # UTC 容器上窗口整体偏移 8 小时, 分时补拉必然为空。 + start_time = datetime(trade_date.year, trade_date.month, trade_date.day, 9, 25, 0, tzinfo=CN_TZ) + end_time = datetime(trade_date.year, trade_date.month, trade_date.day, 15, 5, 0, tzinfo=CN_TZ) # 自定义数据源分流: 与 sync_minute_batch 一致, 配了自定义分钟源时走 custom provider, # 避免无 TickFlow Pro+ 权限的用户分时图首次打开(本地无数据)时补拉失败返回空。 diff --git a/backend/app/services/market_mainline.py b/backend/app/services/market_mainline.py new file mode 100644 index 0000000..2344957 --- /dev/null +++ b/backend/app/services/market_mainline.py @@ -0,0 +1,248 @@ +"""市场主线(板块/概念)识别 — 基于涨停梯队的历史聚合。 + +用户判据的量化: 主升阶段的主线 = 同一概念内涨停家数多、最高连板高、 +梯队档位填得满(2 板到最高板之间不断层)。对每个交易日按概念聚合涨停梯队, +截面 rank 归一后加权成主线分, 持久化为日频时序, 供市场环境页展示 +"什么阶段走什么主升"。 + +口径限制(重要): 概念成分来自 ext_gn_ths 快照(本地自 2026-07 起留存, 无历史 +版本)。历史主线是把"今天的成分"回看历史 — 早年存在归属漂移(新概念不会 +出现在旧时段、成分调整会错归属)。MEMBERSHIP_NOTE 随 API 返回给前端展示。 + +性能: 全量回填只窄扫 enriched 的 4 列并先过滤连板 >=1(全历史 ~10 万行), +join 概念映射后 group_by, 峰值内存 <100MB。 +""" +from __future__ import annotations + +import logging +from datetime import date +from pathlib import Path + +import polars as pl + +from app.services.rps_rotation import _load_concept_map_df + +logger = logging.getLogger(__name__) + +MEMBERSHIP_NOTE = ( + "概念成分为当前快照回看历史(本地自 2026-07 起留存, 无历史版本), " + "早年主线存在归属漂移, 越近越准" +) + +MAINLINE_DIR = "mainline_history" +_TOP_PER_DAY = 30 # 每日持久化的主线数(按分数截断) +_INDUSTRY_LEVEL = 2 # 行业主线取前两级(如 计算机-软件开发) +_MIN_LIMIT_UP = 3 # 单概念当日最少涨停家数(低于此不参与排名) + +# 主线分权重: 概念内涨停家数 / 最高连板 / 梯队档位数 / 二板以上家数 +_SCORE_WEIGHTS = { + "limit_up_count": 0.35, + "max_boards": 0.25, + "rungs_filled": 0.25, + "ge2_count": 0.15, +} + + +def _resolve_filter_config(filter_cfg: dict | None) -> dict: + """解析过滤配置; None 时读用户偏好(宽基/风格标签过滤, 见 preferences 文档)。""" + if filter_cfg is not None: + return { + "min_members": int(filter_cfg.get("min_members", 4)), + "max_members": int(filter_cfg.get("max_members", 600)), + "blacklist": {str(x) for x in filter_cfg.get("blacklist") or []}, + } + try: + from app.services import preferences + + cfg = preferences.get_mainline_filter_config() + return { + "min_members": int(cfg["min_members"]), + "max_members": int(cfg["max_members"]), + "blacklist": set(cfg["blacklist"]), + } + except Exception: + return {"min_members": 4, "max_members": 600, "blacklist": set()} + + +def mainline_path(data_dir: Path) -> Path: + return data_dir / MAINLINE_DIR / "part.parquet" + + +def load_mainline_history(data_dir: Path, kind: str = "concept") -> pl.DataFrame: + """读取主线时序(全部 kind), 不存在返回空 DataFrame。""" + p = mainline_path(data_dir) + if not p.exists(): + return pl.DataFrame() + try: + df = pl.read_parquet(p) + except Exception as e: + logger.warning("load_mainline_history failed: %s", e) + return pl.DataFrame() + if df.is_empty() or "kind" not in df.columns: + return df + return df.filter(pl.col("kind") == kind) + + +def _industry_member(member: str, kind: str) -> str: + """行业维度取前 _INDUSTRY_LEVEL 级; 概念原样返回。""" + if kind != "industry": + return member + return "-".join(member.split("-")[:_INDUSTRY_LEVEL]) + + +def compute_mainline_range(repo, data_dir: Path, start: date, end: date, + kind: str = "concept", + filter_cfg: dict | None = None) -> pl.DataFrame: + """计算 [start, end] 每日主线排行(按 _SCORE_WEIGHTS 加权截面分)。 + + filter_cfg: {"min_members", "max_members", "blacklist"}; None 时读用户偏好。 + 宽基/风格标签(融资融券/沪深股通等数千成分)按成员数上限过滤, + 用户黑名单按名称过滤(不论大小)。修改配置后重算主线生效。 + + 返回列: date, kind, member, limit_up_count, ge2_count, max_boards, + boards_sum, rungs_filled, leader_symbol, score, rank。空数据返回空表。 + """ + if start > end: + return pl.DataFrame() + enriched_dir = repo.store.data_dir / "kline_daily_enriched" + if not enriched_dir.exists(): + return pl.DataFrame() + + map_df, _ = _load_concept_map_df(repo, kind) + if map_df.is_empty(): + return pl.DataFrame() + + cfg = _resolve_filter_config(filter_cfg) + if cfg["min_members"] > 1 or cfg["max_members"] < 5000 or cfg["blacklist"]: + member_counts = map_df.group_by(kind).len().rename({"len": "_members"}) + member_counts = member_counts.filter( + pl.col("_members").ge(cfg["min_members"]) + & pl.col("_members").le(cfg["max_members"]) + & ~pl.col(kind).is_in(sorted(cfg["blacklist"])) + ) + allowed = member_counts.select(kind) + map_df = map_df.join(allowed, on=kind, how="semi") + if map_df.is_empty(): + return pl.DataFrame() + + limit_rows = ( + pl.scan_parquet(enriched_dir / "**" / "*.parquet") + .select(["date", "symbol", "consecutive_limit_ups", "amount"]) + .filter( + (pl.col("date") >= start) & (pl.col("date") <= end) + & (pl.col("consecutive_limit_ups") >= 1) + ) + .collect() + ) + if limit_rows.is_empty(): + return pl.DataFrame() + + limit_rows = limit_rows.with_columns(pl.col("symbol").str.to_uppercase().alias("_sym_up")) + joined = limit_rows.join(map_df, on="_sym_up", how="inner") + if joined.is_empty(): + return pl.DataFrame() + joined = joined.with_columns( + pl.col(kind).map_elements( + lambda m: _industry_member(str(m), kind), + return_dtype=pl.Utf8, + ).alias("member") + ) + + agg = ( + joined.group_by(["date", "member"]) + .agg( + pl.len().alias("limit_up_count"), + (pl.col("consecutive_limit_ups") >= 2).sum().alias("ge2_count"), + pl.col("consecutive_limit_ups").max().alias("max_boards"), + pl.col("consecutive_limit_ups").sum().alias("boards_sum"), + pl.col("consecutive_limit_ups") + .filter(pl.col("consecutive_limit_ups") >= 2) + .n_unique() + .alias("rungs_filled"), + pl.col("symbol") + .sort_by( + pl.col("consecutive_limit_ups"), pl.col("amount"), + descending=[True, True], + ) + .first() + .alias("leader_symbol"), + ) + ) + + # 截面 rank 归一(0-1) → 加权主线分(0-100)。分母 max(n-1,1) 保证单概念日不除零。 + agg = agg.filter(pl.col("limit_up_count") >= _MIN_LIMIT_UP) + norm_exprs = [] + for col in _SCORE_WEIGHTS: + norm_exprs.append( + ((pl.col(col).rank(method="average") - 1.0) + / pl.max_horizontal(pl.len().over("date") - 1, 1)).over("date").alias(f"_{col}_r") + ) + agg = agg.with_columns(norm_exprs) + agg = agg.with_columns( + ( + 100.0 * sum( + _SCORE_WEIGHTS[col] * pl.col(f"_{col}_r") for col in _SCORE_WEIGHTS + ) + ).alias("score") + ) + agg = agg.with_columns( + pl.col("score").rank(method="ordinal", descending=True).over("date").alias("rank") + ) + result = ( + agg.filter(pl.col("rank") <= _TOP_PER_DAY) + .drop([f"_{col}_r" for col in _SCORE_WEIGHTS]) + .with_columns(pl.lit(kind).alias("kind")) + .select([ + "date", "kind", "member", "limit_up_count", "ge2_count", + "max_boards", "boards_sum", "rungs_filled", "leader_symbol", + "score", "rank", + ]) + .sort(["date", "rank"]) + ) + return result + + +def upsert_mainline_history(data_dir: Path, new_rows: pl.DataFrame) -> None: + """按 (date, kind) 整日覆盖 upsert; schema 以 new_rows 为权威(同 regime 模式)。""" + if new_rows.is_empty() or "date" not in new_rows.columns: + return + p = mainline_path(data_dir) + p.parent.mkdir(parents=True, exist_ok=True) + old = pl.read_parquet(p) if p.exists() else pl.DataFrame() + if old.is_empty(): + combined = new_rows + else: + # 按 (date, kind) 整日覆盖: anti-join 掉本次重算的 (日, 维度) 组合 + kept = old.join( + new_rows.select(["date", "kind"]).unique(), + on=["date", "kind"], + how="anti", + ) + target_cols = new_rows.columns + keep_exprs = [ + pl.col(c) if c in kept.columns else pl.lit(None).alias(c) + for c in target_cols + ] + kept = kept.select(keep_exprs) + combined = pl.concat([kept, new_rows.select(target_cols)], how="vertical_relaxed") + combined = combined.sort(["date", "kind", "rank"]) + combined.write_parquet(p) + + +def compute_mainline_incremental(repo, data_dir: Path, *, today: date | None = None, + kind: str = "concept") -> pl.DataFrame: + """增量补算主线(供 daily_pipeline / 手动触发): 补 enriched 已有而主线缺失的日。""" + today = today or date.today() + from app.services.regime_builder import enriched_date_set + + enriched_dates = enriched_date_set(repo) + existing = load_mainline_history(data_dir, kind) + existing_dates = set(existing["date"].to_list()) if not existing.is_empty() else set() + missing = sorted(d for d in enriched_dates if d not in existing_dates and d <= today) + if not missing: + return pl.DataFrame() + logger.info("mainline incremental(%s): compute %d days", kind, len(missing)) + new_rows = compute_mainline_range(repo, data_dir, missing[0], missing[-1], kind=kind) + if not new_rows.is_empty(): + upsert_mainline_history(data_dir, new_rows) + return new_rows diff --git a/backend/app/services/market_phase.py b/backend/app/services/market_phase.py new file mode 100644 index 0000000..3f10703 --- /dev/null +++ b/backend/app/services/market_phase.py @@ -0,0 +1,255 @@ +"""市场情绪周期阶段(冰点/启动/主升/高潮/退潮/修复) — 纯函数模块。 + +与 regime_builder 的 5 档 state(强势/偏强/震荡/偏弱/弱势)并存: +- state: 综合情绪分(赚钱/投机/抗跌/趋势 4 维加权), 回测环境过滤与挖掘在用, 不动。 +- phase: 基于"连板梯队"的阶段(用户判据: 高度、宽度、晋级率、梯队完整度), + 刻画情绪周期位置(冰点→启动→主升→高潮→退潮), 供市场环境页分析与主线识别。 + +驱动量(全部可从已存储的 consecutive_limit_ups 派生, 2020-08 起全历史可回算): +- height 高度: 当日最高连板数 +- first_board 首板宽度: 首板(1 连板)家数 +- ge2/ge3/ge5 宽度: N 板以上家数 +- promo 晋级率: 昨日连板池今日继续封板的比例 (池 <10 家记 null) +- seal_rate 封板率: regime 已有列 +- ladder_completeness 梯队完整度: 2..height 档位中非空占比 + +阈值标定: 2020-08~2026-08 全市场 1454 个交易日的 p10/p60/p90 分位数 +(标定脚本一次性运行, 不提交); 关键异常段抽查(2024-09/10 rally→climax→ebb, +2024-01/02 微盘退潮)人工核过归属。阈值集中在下方, 调整只需改这里。 +""" +from __future__ import annotations + +import logging + +import polars as pl + +logger = logging.getLogger(__name__) + +# ───────────────────────── 阶段词汇 ───────────────────────── +PHASE_ICE = "ice" +PHASE_IGNITE = "ignite" +PHASE_RALLY = "rally" +PHASE_CLIMAX = "climax" +PHASE_EBB = "ebb" +PHASE_REPAIR = "repair" + +PHASE_LABELS = { + PHASE_ICE: "冰点", + PHASE_IGNITE: "启动", + PHASE_RALLY: "主升", + PHASE_CLIMAX: "高潮", + PHASE_EBB: "退潮", + PHASE_REPAIR: "修复", +} + +# 规则判定优先级: 高潮 > 主升 > 退潮 > 启动 > 冰点 > 修复(兜底) +_PHASE_PRIORITY = (PHASE_CLIMAX, PHASE_RALLY, PHASE_EBB, PHASE_IGNITE, PHASE_ICE) + +# ───────────────────────── 阈值(标定自 2020-08~2026-08 分位数) ───────────────────────── +# 高潮: 情绪极端宣泄 — 二板以上宽度或首板数达到 p90 的 ~2 倍以上(历史 <2% 天数) +CLIMAX_GE2 = 50 # p90(25) 的 2 倍 +CLIMAX_FIRST_BOARD = 220 # p90(88) 的 2.5 倍 +# 主升: 高度/宽度/晋级率同时高于中位 (p60), 或晋级率极强 (p85+) +RALLY_HEIGHT = 7 # p60 +RALLY_GE2 = 15 # p60 +RALLY_PROMO = 0.23 # p60 +RALLY_PROMO_ALT = 0.30 # p85+ +RALLY_GE2_ALT = 12 +RALLY_HEIGHT_ALT = 5 +# 退潮: 晋级率崩至 p20 以下且宽度自近期高位回落; 或晋级率/封板率双弱 +EBB_PROMO = 0.15 # p20 +EBB_PROMO_STRICT = 0.13 +EBB_SEAL = 0.57 # ~p10-p15 +EBB_RECENT_GE2 = 12 # 5 日前 ge2 高于此才认定"自高位退潮" +EBB_RECENT_HEIGHT = 6 +# 启动: 宽度/高度自低位扩张且晋级率恢复 +IGNITE_GE2_DELTA = 3 # ge2 较 5 日前增加量 +IGNITE_GE2 = 8 # p20-p40 +IGNITE_PROMO = 0.20 # ~p55 +IGNITE_HEIGHT_DELTA = 1 # height 较 5 日前抬升 +IGNITE_HEIGHT = 5 # p40 +IGNITE_PROMO_SOFT = 0.19 +# 冰点: 高度/宽度/首板同时贴地 (p10) +ICE_HEIGHT = 4 # p10-p20 +ICE_GE2 = 6 # p10 +ICE_FIRST_BOARD = 24 # p10 + +# 晋级率最小池(家数), 低于此记 null(小样本噪声) +PROMO_MIN_POOL = 10 +# 平滑与持续性: EMA alpha≈1/3 (约 5 日), 阶段切换需连续 CONFIRM_DAYS 日同标签 +_EMA_ALPHA = 1.0 / 3.0 +_CONFIRM_DAYS = 2 +# 大盘弱档否决: 正面阶段(主升/高潮/启动)不允许出现在 5 档 state 为弱势/偏弱的日子。 +# 涨停梯队可能与大盘背离(如 2024-01 微盘崩期间中字头涨停生态走强), 该否决 +# 保证"主升"标签在大盘层面也成立; state 列缺失时(单元测试)不启用否决。 +_POSITIVE_PHASES = frozenset({PHASE_CLIMAX, PHASE_RALLY, PHASE_IGNITE}) +_VETO_STATES = frozenset({"weak", "lean_weak"}) + + +def with_prev_consecutive(df: pl.DataFrame) -> pl.DataFrame: + """按 symbol 追加昨日连板数列 _prev_consec (供晋级率)。 + + df 需含 symbol/date/consecutive_limit_ups; 输入应覆盖前一交易日 + (调用方保证 warmup 或直接传全量), 每个符号首行 _prev_consec 为 null。 + """ + if "_prev_consec" in df.columns: + return df + return ( + df.sort(["symbol", "date"]) + .with_columns( + pl.col("consecutive_limit_ups").shift(1).over("symbol").alias("_prev_consec") + ) + ) + + +def ladder_daily_aggs() -> list[pl.Expr]: + """group_by("date").agg(...) 可直接拼接的梯队聚合表达式。 + + 要求 df 含 consecutive_limit_ups; 含 _prev_consec 时附带晋级率分子/分母。 + """ + consec = pl.col("consecutive_limit_ups") + exprs = [ + consec.eq(1).sum().alias("first_board"), + consec.ge(2).sum().alias("ge2_count"), + consec.ge(3).sum().alias("ge3_count"), + consec.ge(5).sum().alias("ge5_count"), + consec.filter(consec.ge(2)).n_unique().alias("rungs_filled"), + ] + return exprs + + +def ladder_promo_aggs() -> list[pl.Expr]: + """晋级率聚合(分子/分母); 要求 df 已含 _prev_consec 列。""" + prev = pl.col("_prev_consec") + consec = pl.col("consecutive_limit_ups") + return [ + prev.ge(1).sum().alias("promo_pool"), + (prev.ge(1) & consec.eq(prev + 1)).sum().alias("promo_ok"), + ] + + +def finalize_ladder_row(r: dict) -> dict: + """把聚合行的梯队原始值整理为持久化字段(晋级率/ladder_completeness)。""" + height = int(r.get("max_consecutive") or 0) + rungs = int(r.get("rungs_filled") or 0) + completeness = (rungs / (height - 1)) if height >= 3 else 0.0 + pool = int(r.get("promo_pool") or 0) + ok = int(r.get("promo_ok") or 0) + promo = (ok / pool) if pool >= PROMO_MIN_POOL else None + return { + "first_board": int(r.get("first_board") or 0), + "ge2_count": int(r.get("ge2_count") or 0), + "ge3_count": int(r.get("ge3_count") or 0), + "ge5_count": int(r.get("ge5_count") or 0), + "ladder_completeness": round(completeness, 4), + "promo_pool": pool, + "promo_rate": round(promo, 4) if promo is not None else None, + } + + +def _ema(values: list[float], alpha: float = _EMA_ALPHA) -> list[float]: + out: list[float] = [] + cur = None + for v in values: + if v is None or v != v: # None 或 NaN + if cur is None: + out.append(None) + continue + out.append(cur) # ffill: 缺失沿用上一平滑值 + continue + cur = v if cur is None else cur + alpha * (v - cur) + out.append(cur) + # 前向回填: 序列开头缺失用首个有效值 + first_valid = next((i for i, x in enumerate(out) if x is not None), None) + if first_valid is not None: + for i in range(first_valid): + out[i] = out[first_valid] + else: + out = [0.0] * len(values) + return out + + +def classify_phase_series(daily: pl.DataFrame) -> pl.DataFrame: + """对完整日序打阶段标签, 追加 phase 列。 + + 输入列: date, max_consecutive, first_board, ge2_count, promo_rate, seal_rate + (promo_rate 允许 null)。处理: promo 前向填充 → 各驱动 EMA 平滑 → + 逐日规则判定(按优先级) → 连续 _CONFIRM_DAYS 日同标签才切换(持续性)。 + """ + required = {"date", "max_consecutive", "first_board", "ge2_count", "promo_rate", "seal_rate"} + missing = required - set(daily.columns) + if missing: + raise ValueError(f"classify_phase_series 缺少列: {sorted(missing)}") + + rows = daily.sort("date") + n = rows.height + states = rows["state"].to_list() if "state" in rows.columns else None + height_s = _ema([float(v) if v is not None else None for v in rows["max_consecutive"].to_list()]) + first_s = _ema([float(v) if v is not None else None for v in rows["first_board"].to_list()]) + ge2_s = _ema([float(v) if v is not None else None for v in rows["ge2_count"].to_list()]) + promo_s = _ema([float(v) if v is not None else None for v in rows["promo_rate"].to_list()]) + seal_s = _ema([float(v) if v is not None else None for v in rows["seal_rate"].to_list()]) + + def raw_label(i: int) -> str: + h, fb, g2, pr, sr = height_s[i], first_s[i], ge2_s[i], promo_s[i], seal_s[i] + g2_prev = ge2_s[max(0, i - 5)] + h_prev = height_s[max(0, i - 5)] + # 高潮 + if g2 >= CLIMAX_GE2 or fb >= CLIMAX_FIRST_BOARD: + return PHASE_CLIMAX + # 主升 + if h >= RALLY_HEIGHT and g2 >= RALLY_GE2 and pr >= RALLY_PROMO: + return PHASE_RALLY + if pr >= RALLY_PROMO_ALT and g2 >= RALLY_GE2_ALT and h >= RALLY_HEIGHT_ALT: + return PHASE_RALLY + # 冰点: 高度/宽度/首板同时贴地 — 优先于退潮(持续死寂的市场是"冰点" + # 而非"自高位退潮"; 退潮的规则 B 不带 from_high 条件, 顺序反了会把 + # 长期冰点误标成退潮) + if h <= ICE_HEIGHT and g2 <= ICE_GE2 and fb <= ICE_FIRST_BOARD: + return PHASE_ICE + # 退潮: 自高位回落 + 晋级率坍塌, 或晋级/封板双弱 + from_high = g2_prev >= EBB_RECENT_GE2 or h_prev >= EBB_RECENT_HEIGHT + if from_high and (pr <= EBB_PROMO and g2 < g2_prev): + return PHASE_EBB + if pr <= EBB_PROMO_STRICT and sr <= EBB_SEAL: + return PHASE_EBB + # 启动: 自低位扩张 + if g2 - g2_prev >= IGNITE_GE2_DELTA and g2 >= IGNITE_GE2 and pr >= IGNITE_PROMO: + return PHASE_IGNITE + if h - h_prev >= IGNITE_HEIGHT_DELTA and h >= IGNITE_HEIGHT and pr >= IGNITE_PROMO_SOFT: + return PHASE_IGNITE + return PHASE_REPAIR + + labels: list[str] = [] + current = None + pending: str | None = None + pending_run = 0 + for i in range(n): + raw = raw_label(i) + if ( + states is not None + and raw in _POSITIVE_PHASES + and states[i] in _VETO_STATES + ): + raw = PHASE_REPAIR + if current is None: + current = raw + labels.append(raw) + continue + if raw == current: + labels.append(current) + pending, pending_run = None, 0 + continue + if raw == pending: + pending_run += 1 + else: + pending, pending_run = raw, 1 + if pending_run >= _CONFIRM_DAYS: + current = raw + labels.append(current) + pending, pending_run = None, 0 + else: + labels.append(current) + return daily.with_columns( + pl.Series("phase", labels, dtype=pl.Utf8).alias("phase") + ).sort("date") diff --git a/backend/app/services/matrix_prewarm_owner.py b/backend/app/services/matrix_prewarm_owner.py new file mode 100644 index 0000000..9c01cb2 --- /dev/null +++ b/backend/app/services/matrix_prewarm_owner.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import threading +from collections.abc import Callable + + +class MatrixCachePrewarmOwner: + def __init__(self) -> None: + self._lock = threading.Lock() + self._cancel_event = threading.Event() + self._thread: threading.Thread | None = None + + @property + def cancel_event(self) -> threading.Event: + return self._cancel_event + + def schedule(self, target: Callable[[], None]) -> bool: + with self._lock: + if self._cancel_event.is_set(): + return False + if self._thread is not None and self._thread.is_alive(): + return False + thread = threading.Thread( + target=self._run, + args=(target,), + name="matrix-cache-prewarm", + daemon=True, + ) + self._thread = thread + thread.start() + return True + + def shutdown(self, timeout: float = 5.0) -> bool: + self._cancel_event.set() + with self._lock: + thread = self._thread + if thread is None: + return True + thread.join(timeout=max(0.0, timeout)) + return not thread.is_alive() + + def _run(self, target: Callable[[], None]) -> None: + try: + target() + finally: + with self._lock: + if self._thread is threading.current_thread(): + self._thread = None diff --git a/backend/app/services/mining_candidates.py b/backend/app/services/mining_candidates.py new file mode 100644 index 0000000..7625862 --- /dev/null +++ b/backend/app/services/mining_candidates.py @@ -0,0 +1,793 @@ +"""Trusted promotion and explicit publication for persisted mining candidates. + +Concurrency protection is process-local; V1 remains a single-process service. +""" +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import threading +import uuid +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Any + +import polars as pl +import pyarrow.parquet as pq + +from app.backtest.candidates import CandidateStore +from app.backtest.factor import FACTOR_COLUMNS +from app.backtest.mining import compute_candidate_signature, evaluate_candidate_gate +from app.services.mining_jobs import SUCCESS_RUN_STATUSES, MiningRunStore +from app.strategy.ai_generator import AIStrategyGenerator +from app.strategy.engine import StrategyEngine + +_MAX_ARTIFACT_BYTES = 8 * 1024 * 1024 +_MAX_ARTIFACT_ROWS = 32 +_MAX_UNCOMPRESSED_BYTES = 32 * 1024 * 1024 +_MAX_DEFINITION_BYTES = 16 * 1024 +_FACTOR_IDS = frozenset(str(item["id"]) for item in FACTOR_COLUMNS) +_FACTOR_DEFINITION_FIELDS = frozenset({"kind", "factor_names", "scoring", "directions"}) +_EXISTING_DEFINITION_FIELDS = frozenset({"kind", "strategy_id"}) +_BACKLINK_FIELDS = frozenset({"promoted_candidate_id", "published_strategy_id"}) +_PUBLISHED_PREFIX = "mined_factor_" +_STRATEGY_ID_PATTERN = re.compile(r"^mined_factor_[A-Za-z0-9_-]{1,49}$") +_ARTIFACT_SCHEMA = { + "signature": pl.String, + "name": pl.String, + "kind": pl.String, + "factor_names_json": pl.String, + "strategy_id": pl.String, + "definition_json": pl.String, + "regime_state": pl.String, + "score": pl.Float64, + "oos_return": pl.Float64, + "oos_sharpe": pl.Float64, + "oos_max_drawdown": pl.Float64, + "oos_positive_fold_ratio": pl.Float64, + "oos_n_trades": pl.Int64, + "confidence": pl.String, + "valid_folds": pl.Int64, + "skipped_folds": pl.Int64, + "promoted_candidate_id": pl.String, + "published_strategy_id": pl.String, +} +_LOCK = threading.RLock() + + +class MiningCandidateService: + def __init__( + self, + data_dir: Path | str, + run_store: MiningRunStore, + candidate_store: CandidateStore, + strategy_engine: StrategyEngine, + *, + strategy_cache_invalidator: Callable[[Path], None] | None = None, + monitor_state_invalidator: Callable[[], None] | None = None, + ) -> None: + self.data_dir = Path(data_dir).resolve() + self.run_store = run_store + self.candidate_store = candidate_store + self.strategy_engine = strategy_engine + if strategy_cache_invalidator is None: + from app.services.strategy_cache import clear_cache + + strategy_cache_invalidator = clear_cache + self._strategy_cache_invalidator = strategy_cache_invalidator + self._monitor_state_invalidator = monitor_state_invalidator + + def promote(self, run_id: str, signature: str) -> dict[str, Any]: + with _LOCK: + manifest, summary, path, frame, row, definition = self._load_candidate( + run_id, signature + ) + kind, name, source_id, config = self._promotion_config( + manifest, summary, row, definition + ) + item = self.candidate_store.create_or_get_by_provenance( + origin_run_id=run_id, + candidate_signature=signature, + kind=kind, + name=name, + source_id=source_id, + config=config, + metrics=self._candidate_metrics(row), + data_as_of=_optional_string(summary.get("data_as_of")), + status="pending", + ) + if row.get("promoted_candidate_id") != item["id"]: + self._write_backlink( + path, + frame, + signature, + "promoted_candidate_id", + item["id"], + ) + return item + + def publish(self, run_id: str, signature: str) -> dict[str, Any]: + with _LOCK: + manifest, summary, path, frame, row, definition = self._load_candidate( + run_id, signature + ) + gate = evaluate_candidate_gate( + confidence=row.get("confidence"), + valid_folds=row.get("valid_folds"), + positive_fold_ratio=row.get("oos_positive_fold_ratio"), + sharpe=row.get("oos_sharpe"), + max_drawdown=row.get("oos_max_drawdown"), + n_trades=row.get("oos_n_trades"), + ) + if not gate.qualified: + raise ValueError( + "candidate does not meet the promotion gate: " + + "; ".join(gate.reasons) + ) + asset_type = self._asset_type(manifest) + if definition["kind"] == "existing_strategy": + published_id = str(definition["strategy_id"]) + self._validate_publication_backlink(row, published_id) + self._verify_public_strategy(published_id, asset_type) + if row.get("published_strategy_id") != published_id: + self._write_backlink( + path, + frame, + signature, + "published_strategy_id", + published_id, + ) + return {"ok": True, "strategy_id": published_id} + + published_id = self._validate_published_id( + _published_strategy_id(run_id, signature) + ) + self._validate_publication_backlink(row, published_id) + source = self._render_factor_strategy( + manifest, summary, row, definition, published_id + ) + + target = self._custom_strategy_path(published_id) + created = self._publish_or_verify_source( + target, + source, + published_id, + run_id, + signature, + asset_type, + ) + try: + self._strategy_cache_invalidator(self.data_dir) + if self._monitor_state_invalidator is not None: + self._monitor_state_invalidator() + except Exception as exc: + rollback_error = ( + self._rollback_created_source(target, source) if created else None + ) + message = f"strategy runtime invalidation failed: {exc}" + if rollback_error is not None: + message += f"; strategy rollback failed: {rollback_error}" + raise RuntimeError(message) from exc + if row.get("published_strategy_id") != published_id: + self._write_backlink( + path, + frame, + signature, + "published_strategy_id", + published_id, + ) + return {"ok": True, "strategy_id": published_id} + + def _load_candidate( + self, + run_id: str, + signature: str, + ) -> tuple[ + dict[str, Any], + dict[str, Any], + Path, + pl.DataFrame, + dict[str, Any], + dict[str, Any], + ]: + if not isinstance(signature, str) or not signature: + raise ValueError("candidate signature must not be empty") + manifest = self.run_store.get(run_id) + if manifest is None: + raise KeyError(run_id) + if manifest.get("status") not in SUCCESS_RUN_STATUSES: + raise ValueError("mining candidates require a successful run") + path = self._registered_candidates_path(manifest) + frame = self._read_artifact(path) + matches = frame.filter(pl.col("signature") == signature) + if matches.height == 0: + raise KeyError(signature) + if matches.height != 1: + raise ValueError("mining candidates artifact contains a duplicate signature") + row = matches.row(0, named=True) + self._asset_type(manifest) + definition = self._validate_definition(manifest, row, signature) + summary = self._validated_summary(self.run_store.read_summary(run_id)) + return manifest, summary, path, frame, row, definition + + def _registered_candidates_path(self, manifest: Mapping[str, Any]) -> Path: + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, Mapping): + raise ValueError("mining candidates artifact is not registered") + raw_path = artifacts.get("candidates") + if raw_path != "candidates.parquet": + raise ValueError( + "only the registered candidates.parquet artifact can be used" + ) + path = self.run_store.artifact_path( + str(manifest["run_id"]), "candidates" + ) + if path.is_symlink() or not path.is_file(): + raise ValueError("mining candidates artifact is unavailable") + return path + + @staticmethod + def _read_artifact(path: Path) -> pl.DataFrame: + try: + size = path.stat().st_size + except OSError as exc: + raise RuntimeError("failed to read mining candidates artifact") from exc + if size <= 0 or size > _MAX_ARTIFACT_BYTES: + raise ValueError("mining candidates artifact exceeds its size limit") + try: + parquet = pq.ParquetFile(path) + metadata = parquet.metadata + schema = pl.read_parquet_schema(path) + except Exception as exc: + raise RuntimeError("failed to read mining candidates artifact") from exc + if metadata is None or metadata.num_rows > _MAX_ARTIFACT_ROWS: + raise ValueError("mining candidates artifact exceeds its row limit") + uncompressed = sum( + metadata.row_group(index).total_byte_size + for index in range(metadata.num_row_groups) + ) + if uncompressed > _MAX_UNCOMPRESSED_BYTES: + raise ValueError( + "mining candidates artifact exceeds its uncompressed size limit" + ) + missing = sorted(set(_ARTIFACT_SCHEMA) - set(schema)) + if missing: + raise ValueError(f"mining candidates artifact schema is invalid: {missing}") + invalid_types = [ + name for name, dtype in _ARTIFACT_SCHEMA.items() + if schema[name] != dtype + ] + if invalid_types: + raise ValueError( + "mining candidates artifact column types are invalid: " + f"{invalid_types}" + ) + try: + return pl.read_parquet(path, columns=list(_ARTIFACT_SCHEMA)) + except Exception as exc: + raise RuntimeError("failed to read mining candidates artifact") from exc + + def _validate_definition( + self, + manifest: Mapping[str, Any], + row: Mapping[str, Any], + signature: str, + ) -> dict[str, Any]: + raw_definition = row.get("definition_json") + if ( + not isinstance(raw_definition, str) + or len(raw_definition.encode("utf-8")) > _MAX_DEFINITION_BYTES + ): + raise ValueError("mining candidate definition is unavailable") + try: + definition = json.loads(raw_definition) + except json.JSONDecodeError as exc: + raise ValueError("mining candidate definition is invalid") from exc + if not isinstance(definition, dict): + raise ValueError("mining candidate definition must be an object") + kind = definition.get("kind") + if kind == "factor_rank": + if set(definition) != _FACTOR_DEFINITION_FIELDS: + raise ValueError("factor candidate definition contains unsupported fields") + expected_kind = "factor_combination" + elif kind == "existing_strategy": + if set(definition) != _EXISTING_DEFINITION_FIELDS: + raise ValueError("existing candidate definition contains unsupported fields") + expected_kind = "existing_strategy" + else: + raise ValueError(f"unsupported mining candidate kind: {kind!r}") + if row.get("kind") != expected_kind: + raise ValueError("mining candidate kind does not match its definition") + if ( + not isinstance(row.get("name"), str) + or not row["name"] + or len(row["name"]) > 80 + or row.get("regime_state") != "overall" + ): + raise ValueError("mining candidate row metadata is invalid") + if kind == "factor_rank": + self._validate_factor_definition(manifest, row, definition) + else: + self._validate_existing_definition(manifest, row, definition) + computed = compute_candidate_signature(definition) + if row.get("signature") != signature or computed != signature: + raise ValueError("mining candidate signature does not match its definition") + score = row.get("score") + if ( + score is not None + and ( + isinstance(score, bool) + or not isinstance(score, (int, float)) + or not math.isfinite(float(score)) + ) + ): + raise ValueError("candidate score must be finite") + return definition + + @staticmethod + def _request(manifest: Mapping[str, Any]) -> Mapping[str, Any]: + request = manifest.get("request") + if not isinstance(request, Mapping): + raise ValueError("mining origin request is unavailable") + return request + + def _validate_factor_definition( + self, + manifest: Mapping[str, Any], + row: Mapping[str, Any], + definition: Mapping[str, Any], + ) -> None: + factor_names = definition.get("factor_names") + if ( + not isinstance(factor_names, list) + or not 1 <= len(factor_names) <= 4 + or len(set(factor_names)) != len(factor_names) + or any(not isinstance(name, str) or not name for name in factor_names) + ): + raise ValueError("factor candidate must contain 1 to 4 unique factors") + scoring = definition.get("scoring") + directions = definition.get("directions") + if not isinstance(scoring, Mapping) or set(scoring) != set(factor_names): + raise ValueError("factor scoring keys must exactly match factor names") + if not isinstance(directions, Mapping) or set(directions) != set(factor_names): + raise ValueError("factor direction keys must exactly match factor names") + for factor_name in factor_names: + weight = scoring[factor_name] + if ( + isinstance(weight, bool) + or not isinstance(weight, (int, float)) + or not math.isfinite(float(weight)) + or float(weight) <= 0.0 + ): + raise ValueError("factor weights must be finite and positive") + if directions[factor_name] not in {"high", "low"}: + raise ValueError("factor directions must be high or low") + unknown = sorted(set(factor_names) - _FACTOR_IDS) + if unknown: + raise ValueError(f"factor candidate contains unknown factors: {unknown}") + selected = self._request(manifest).get("factor_names") + if not isinstance(selected, list) or not set(factor_names) <= set(selected): + raise ValueError("factor candidate contains factors absent from its origin request") + try: + persisted_names = json.loads(str(row.get("factor_names_json"))) + except json.JSONDecodeError as exc: + raise ValueError("factor candidate factor list is invalid") from exc + if persisted_names != factor_names or row.get("strategy_id") is not None: + raise ValueError("factor candidate columns do not match its definition") + + def _validate_existing_definition( + self, + manifest: Mapping[str, Any], + row: Mapping[str, Any], + definition: Mapping[str, Any], + ) -> None: + strategy_id = definition.get("strategy_id") + if not isinstance(strategy_id, str) or not strategy_id: + raise ValueError("existing candidate strategy ID is invalid") + selected = self._request(manifest).get("strategy_ids") + if not isinstance(selected, list) or strategy_id not in selected: + raise ValueError("existing candidate was absent from its origin request") + if row.get("strategy_id") != strategy_id: + raise ValueError("existing candidate strategy ID does not match its definition") + asset_type = str(self._request(manifest).get("asset_type") or "stock") + self._verify_public_strategy(strategy_id, asset_type) + + def _promotion_config( + self, + manifest: Mapping[str, Any], + summary: Mapping[str, Any], + row: Mapping[str, Any], + definition: Mapping[str, Any], + ) -> tuple[str, str, str, dict[str, Any]]: + request = self._request(manifest) + signature = str(row["signature"]) + provenance = { + "origin_run_id": str(manifest["run_id"]), + "candidate_signature": signature, + "regime_state": str(row.get("regime_state") or "overall"), + "algorithm_version": str(summary.get("algorithm_version") or "mining-v1"), + "methodology_version": str(summary.get("methodology_version") or "factor_v2"), + } + common = { + "start": request.get("start"), + "end": request.get("end"), + "asset_type": request.get("asset_type") or "stock", + "matching": "open_t+1", + "entry_fill": "open_t+1", + "exit_fill": "open_t+1", + "commission_pct": request.get("commission_pct", 0.0002), + "stamp_tax_pct": request.get("stamp_tax_pct", 0.0005), + "slippage_bps": request.get("slippage_bps", 5.0), + "mode": "position", + "minute_fill": False, + **provenance, + } + name = str(row.get("name") or signature)[:80] + if definition["kind"] == "existing_strategy": + strategy_id = str(definition["strategy_id"]) + return "strategy", name, strategy_id, { + **common, + "strategy_id": strategy_id, + } + factor_names = list(definition["factor_names"]) + source_id = _published_strategy_id(str(manifest["run_id"]), signature) + return "strategy", name, source_id, { + **common, + "strategy_id": source_id, + "factor_names": factor_names, + "directions": [definition["directions"][name] for name in factor_names], + "weights": [float(definition["scoring"][name]) for name in factor_names], + } + + @staticmethod + def _candidate_metrics(row: Mapping[str, Any]) -> dict[str, Any]: + fields = ( + "oos_sharpe", + "oos_return", + "oos_max_drawdown", + "oos_positive_fold_ratio", + "oos_n_trades", + "valid_folds", + "skipped_folds", + "confidence", + ) + result: dict[str, Any] = {} + for field in fields: + value = row.get(field) + if value is None: + continue + if field == "confidence": + if not isinstance(value, str) or not value or len(value) > 32: + raise ValueError("candidate confidence is invalid") + elif ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + ): + raise ValueError(f"candidate metric {field} must be finite") + result[field] = value + return result + + @staticmethod + def _validated_summary(summary: Mapping[str, Any]) -> dict[str, str]: + result: dict[str, str] = {} + for field in ("data_as_of", "algorithm_version", "methodology_version"): + value = summary.get(field) + if not isinstance(value, str) or not value or len(value) > 120: + raise ValueError(f"mining summary {field} is invalid") + result[field] = value + return result + + @staticmethod + def _asset_type(manifest: Mapping[str, Any]) -> str: + asset_type = MiningCandidateService._request(manifest).get("asset_type") + if asset_type not in {"stock", "etf"}: + raise ValueError("mining origin asset_type is invalid") + return str(asset_type) + + def _verify_public_strategy( + self, + strategy_id: str, + asset_type: str, + *, + path: Path | None = None, + run_id: str | None = None, + signature: str | None = None, + ) -> None: + strategy = self.strategy_engine.get(strategy_id) + if strategy.meta.get("research_only"): + raise ValueError("research-only strategy cannot be published") + if strategy.execution_backend != "matrix_native": + raise ValueError("published strategy must be matrix-native") + if "1d" not in strategy.meta.get("timeframes", []): + raise ValueError("published strategy must support 1d") + if asset_type not in strategy.meta.get("asset_types", []): + raise ValueError("published strategy does not support the run asset type") + public_ids = { + str(meta.get("id")) for meta in self.strategy_engine.list_strategies() + } + if strategy_id not in public_ids: + raise ValueError("published strategy is not publicly discoverable") + if path is not None and ( + strategy.source != "custom" + or strategy.file_path is None + or strategy.file_path.resolve() != path.resolve() + or strategy.meta.get("origin_run_id") != run_id + or strategy.meta.get("candidate_signature") != signature + ): + raise ValueError("published strategy provenance is invalid") + + @staticmethod + def _validate_published_id(strategy_id: str) -> str: + if not _STRATEGY_ID_PATTERN.fullmatch(strategy_id): + raise ValueError( + f"strategy_id must use {_PUBLISHED_PREFIX!r} and safe characters" + ) + return strategy_id + + @staticmethod + def _validate_publication_backlink( + row: Mapping[str, Any], + strategy_id: str, + ) -> None: + existing = _optional_string(row.get("published_strategy_id")) + if existing is not None and existing != strategy_id: + raise ValueError("candidate publication backlink is inconsistent") + + def _custom_strategy_path(self, strategy_id: str) -> Path: + unresolved_root = self.data_dir / "strategies" / "custom" + unresolved_root.mkdir(parents=True, exist_ok=True) + if unresolved_root.is_symlink(): + raise ValueError("custom strategy directory must not be a symlink") + root = unresolved_root.resolve() + if not root.is_relative_to(self.data_dir): + raise ValueError("custom strategy directory escapes data_dir") + path = (root / f"{strategy_id}.py").resolve(strict=False) + if path.parent != root: + raise ValueError("strategy publication path escapes custom directory") + return path + + def _render_factor_strategy( + self, + manifest: Mapping[str, Any], + summary: Mapping[str, Any], + row: Mapping[str, Any], + definition: Mapping[str, Any], + strategy_id: str, + ) -> str: + asset_type = self._asset_type(manifest) + factor_names = list(definition["factor_names"]) + scoring = { + name: float(definition["scoring"][name]) for name in factor_names + } + directions = {name: definition["directions"][name] for name in factor_names} + meta = { + "id": strategy_id, + "name": row["name"], + "description": "Published mining factor-rank candidate", + "tags": ["mining", "factor-rank"], + "asset_types": [asset_type], + "timeframes": ["1d"], + "research_only": False, + "origin_run_id": manifest["run_id"], + "candidate_signature": row["signature"], + "mining_algorithm_version": summary["algorithm_version"], + "factor_methodology_version": summary["methodology_version"], + "params": [ + { + "id": "entry_score", + "label": "Entry minimum score", + "type": "float", + "default": 70.0, + "min": 0.0, + "max": 100.0, + "step": 5.0, + }, + { + "id": "exit_score", + "label": "Exit maximum score", + "type": "float", + "default": 40.0, + "min": 0.0, + "max": 100.0, + "step": 5.0, + }, + { + "id": "top_rank", + "label": "Daily selection limit", + "type": "int", + "default": 20, + "min": 1, + "max": 100, + "step": 1, + }, + ], + "scoring": {}, + "order_by": "score", + "descending": True, + "limit": 100, + } + return ( + '"""Trusted factor-rank strategy published from a mining run."""\n' + "from app.strategy.builtin.factor_rank_research import " + "FactorRankResearchMatrixStrategy\n\n" + f"META = {meta!r}\n\n" + 'EXECUTION_BACKEND = "matrix_native"\n' + 'ENTRY_SIGNALS = ["signal_factor_rank_entry"]\n' + 'EXIT_SIGNALS = ["signal_factor_rank_exit"]\n' + "STOP_LOSS = -0.08\n" + "MAX_HOLD_DAYS = 30\n\n" + f"SCORING = {scoring!r}\n" + f"DIRECTIONS = {directions!r}\n" + "MATRIX_STRATEGY = FactorRankResearchMatrixStrategy(SCORING, DIRECTIONS)\n" + ) + + def _publish_or_verify_source( + self, + path: Path, + source: str, + strategy_id: str, + run_id: str, + signature: str, + asset_type: str, + ) -> bool: + validation = AIStrategyGenerator().validate_code(source) + if not validation.get("valid"): + raise ValueError( + f"rendered strategy failed validation: {validation.get('error')}" + ) + if validation.get("meta", {}).get("id") != strategy_id: + raise ValueError("rendered strategy META id is invalid") + if path.exists() or path.is_symlink(): + self._verify_existing_source( + path, source, strategy_id, run_id, signature, asset_type + ) + return False + if self.strategy_engine.has(strategy_id): + raise ValueError(f"strategy ID already exists: {strategy_id}") + + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + linked = False + try: + with temporary.open("x", encoding="utf-8", newline="\n") as stream: + stream.write(source) + stream.flush() + os.fsync(stream.fileno()) + try: + os.link(temporary, path) + except FileExistsError as exc: + raise ValueError(f"strategy path already exists: {strategy_id}") from exc + except OSError as exc: + raise RuntimeError("failed to create strategy source") from exc + linked = True + self._fsync_directory(path.parent) + try: + self.strategy_engine.reload() + self._verify_public_strategy( + strategy_id, + asset_type, + path=path, + run_id=run_id, + signature=signature, + ) + except Exception as exc: + rollback_error = self._rollback_publication(path, temporary) + message = f"strategy publication failed: {exc}" + if rollback_error is not None: + message += f"; registry rollback failed: {rollback_error}" + raise RuntimeError(message) from exc + finally: + temporary.unlink(missing_ok=True) + if linked: + self._fsync_directory(path.parent) + return True + + def _verify_existing_source( + self, + path: Path, + source: str, + strategy_id: str, + run_id: str, + signature: str, + asset_type: str, + ) -> None: + if path.is_symlink() or not path.is_file(): + raise ValueError("strategy publication target is not a regular file") + try: + existing_source = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise ValueError("existing strategy source is unreadable") from exc + if existing_source != source: + raise ValueError(f"strategy ID collision: {strategy_id}") + try: + self._verify_public_strategy( + strategy_id, + asset_type, + path=path, + run_id=run_id, + signature=signature, + ) + except ValueError: + self.strategy_engine.reload() + self._verify_public_strategy( + strategy_id, + asset_type, + path=path, + run_id=run_id, + signature=signature, + ) + + def _rollback_publication(self, path: Path, temporary: Path) -> Exception | None: + try: + if path.exists() and temporary.exists() and os.path.samefile(path, temporary): + path.unlink() + self._fsync_directory(path.parent) + self.strategy_engine.reload() + except Exception as exc: + return exc + return None + + def _rollback_created_source(self, path: Path, source: str) -> Exception | None: + try: + if path.is_file() and not path.is_symlink(): + if path.read_text(encoding="utf-8") != source: + return RuntimeError("published strategy source changed before rollback") + path.unlink() + self._fsync_directory(path.parent) + self.strategy_engine.reload() + except Exception as exc: + return exc + return None + + @staticmethod + def _fsync_directory(path: Path) -> None: + if os.name == "nt": + return + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + @staticmethod + def _write_backlink( + path: Path, + frame: pl.DataFrame, + signature: str, + field: str, + value: str, + ) -> None: + if field not in _BACKLINK_FIELDS: + raise ValueError("unsupported mining candidate backlink") + matches = frame.filter(pl.col("signature") == signature) + if matches.height != 1: + raise ValueError("mining candidate backlink target is no longer unique") + updated = frame.with_columns( + pl.when(pl.col("signature") == signature) + .then(pl.lit(value)) + .otherwise(pl.col(field)) + .alias(field) + ) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + updated.write_parquet(temporary) + with temporary.open("r+b") as stream: + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + except Exception as exc: + temporary.unlink(missing_ok=True) + raise RuntimeError("failed to update mining candidate artifact") from exc + + +def _published_strategy_id(run_id: str, signature: str) -> str: + payload = f"{run_id}\0{signature}".encode() + digest = hashlib.blake2b(payload, digest_size=10).hexdigest() + return f"{_PUBLISHED_PREFIX}{digest}" + + +def _optional_string(value: Any) -> str | None: + return value if isinstance(value, str) and value else None diff --git a/backend/app/services/mining_jobs.py b/backend/app/services/mining_jobs.py new file mode 100644 index 0000000..343ab8a --- /dev/null +++ b/backend/app/services/mining_jobs.py @@ -0,0 +1,620 @@ +"""Persistent metadata and bounded event storage for mining runs.""" + +from __future__ import annotations + +import json +import math +import os +import re +import threading +import uuid +from collections.abc import Collection, Mapping +from datetime import UTC, date, datetime +from enum import Enum +from pathlib import Path +from typing import Any, Literal, cast + +MiningRunStatus = Literal[ + "queued", + "running", + "cancelling", + "succeeded", + "succeeded_with_budget_exhausted", + "failed", + "cancelled", + "interrupted", + "skipped_prerequisite", +] +ArtifactName = Literal["factors", "correlation", "candidates", "folds"] + +RUN_STATUSES: frozenset[str] = frozenset( + { + "queued", + "running", + "cancelling", + "succeeded", + "succeeded_with_budget_exhausted", + "failed", + "cancelled", + "interrupted", + "skipped_prerequisite", + } +) +ACTIVE_RUN_STATUSES: frozenset[str] = frozenset({"queued", "running", "cancelling"}) +SUCCESS_RUN_STATUSES: frozenset[str] = frozenset({"succeeded", "succeeded_with_budget_exhausted"}) +TERMINAL_RUN_STATUSES: frozenset[str] = RUN_STATUSES - ACTIVE_RUN_STATUSES +ARTIFACT_NAMES: frozenset[str] = frozenset({"factors", "correlation", "candidates", "folds"}) +MAX_EVENTS = 256 +MAX_EVENT_PAYLOAD_BYTES = 16 * 1024 +_SCHEMA_VERSION = 1 +_RUN_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$") +_STORE_LOCK = threading.RLock() + +_ALLOWED_TRANSITIONS: dict[str, frozenset[str]] = { + "queued": frozenset( + {"running", "cancelling", "cancelled", "failed", "interrupted", "skipped_prerequisite"} + ), + "running": frozenset( + { + "cancelling", + "succeeded", + "succeeded_with_budget_exhausted", + "failed", + "cancelled", + "interrupted", + "skipped_prerequisite", + } + ), + "cancelling": frozenset( + { + "succeeded", + "succeeded_with_budget_exhausted", + "failed", + "cancelled", + "interrupted", + } + ), + "succeeded": frozenset(), + "succeeded_with_budget_exhausted": frozenset(), + "failed": frozenset(), + "cancelled": frozenset(), + "interrupted": frozenset(), + "skipped_prerequisite": frozenset(), +} + + +class MiningRunStoreError(RuntimeError): + pass + + +class MiningRunValidationError(MiningRunStoreError, ValueError): + pass + + +class InvalidMiningStatusTransitionError(MiningRunStoreError): + pass + + +def canonicalize_request(request: Mapping[str, Any]) -> dict[str, Any]: + """Return a JSON-safe request whose mapping order cannot affect its signature.""" + if not isinstance(request, Mapping): + raise MiningRunValidationError("request must be a mapping") + return cast(dict[str, Any], _canonicalize_json_value(request)) + + +def compute_run_signature(request: Mapping[str, Any], data_fingerprint: Any) -> str: + """Hash every request dimension and the data fingerprint using BLAKE2b.""" + import hashlib + + signature_input = { + "request": canonicalize_request(request), + "data_fingerprint": _canonicalize_json_value(data_fingerprint), + } + payload = json.dumps( + signature_input, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.blake2b(payload, digest_size=32).hexdigest() + + +class MiningRunStore: + """Store one manifest, summary, artifact registry, and bounded event log per run.""" + + def __init__(self, data_dir: Path | str | None = None) -> None: + if data_dir is None: + from app.config import settings + + data_dir = settings.data_dir + self.runs_root = (Path(data_dir).resolve() / "research" / "mining" / "runs").resolve() + self.runs_root.mkdir(parents=True, exist_ok=True) + + def create( + self, + request: Mapping[str, Any], + data_fingerprint: Any, + *, + run_id: str | None = None, + ) -> dict[str, Any]: + """Create a queued run and its initial on-disk files.""" + safe_run_id = self._validate_run_id(uuid.uuid4().hex if run_id is None else run_id) + canonical_request = canonicalize_request(request) + canonical_fingerprint = _canonicalize_json_value(data_fingerprint) + now = _now_iso() + manifest = { + "schema_version": _SCHEMA_VERSION, + "run_id": safe_run_id, + "status": "queued", + "request": canonical_request, + "data_fingerprint": canonical_fingerprint, + "run_signature": compute_run_signature(canonical_request, canonical_fingerprint), + "artifacts": {}, + "created_at": now, + "updated_at": now, + "started_at": None, + "finished_at": None, + "cancellation_requested_at": None, + "error": None, + } + run_dir = self._run_dir(safe_run_id) + with _STORE_LOCK: + if run_dir.exists(): + raise MiningRunValidationError(f"run already exists: {safe_run_id}") + run_dir.mkdir(parents=False) + _atomic_write_json(run_dir / "summary.json", {}) + _atomic_write_text(run_dir / "events.jsonl", "") + _atomic_write_json(run_dir / "manifest.json", manifest) + return manifest + + def get(self, run_id: str) -> dict[str, Any] | None: + """Read a manifest, filling defaults for manifests written by older versions.""" + safe_run_id = self._validate_run_id(run_id) + with _STORE_LOCK: + return self._read_manifest_path( + self._run_dir(safe_run_id) / "manifest.json", safe_run_id + ) + + def transition_status( + self, + run_id: str, + status: MiningRunStatus, + *, + error: str | None = None, + ) -> dict[str, Any]: + """Apply a validated state transition and atomically replace the manifest.""" + if status not in RUN_STATUSES: + raise MiningRunValidationError(f"unsupported mining run status: {status!r}") + safe_run_id = self._validate_run_id(run_id) + with _STORE_LOCK: + manifest = self._required_manifest(safe_run_id) + return self._transition_locked(manifest, status, error=error) + + def write_summary(self, run_id: str, summary: Mapping[str, Any]) -> dict[str, Any]: + """Atomically replace a run's scalar or compact aggregate summary.""" + if not isinstance(summary, Mapping): + raise MiningRunValidationError("summary must be a mapping") + safe_run_id = self._validate_run_id(run_id) + clean_summary = cast(dict[str, Any], _canonicalize_json_value(summary)) + with _STORE_LOCK: + self._required_manifest(safe_run_id) + _atomic_write_json(self._run_dir(safe_run_id) / "summary.json", clean_summary) + return clean_summary + + def read_summary(self, run_id: str) -> dict[str, Any]: + safe_run_id = self._validate_run_id(run_id) + with _STORE_LOCK: + self._required_manifest(safe_run_id) + path = self._run_dir(safe_run_id) / "summary.json" + if not path.exists(): + return {} + value = _read_json(path) + if not isinstance(value, dict): + raise MiningRunStoreError(f"invalid summary for run {safe_run_id}") + return value + + def artifact_path(self, run_id: str, name: ArtifactName) -> Path: + """Return the safe default Parquet path for an artifact.""" + safe_run_id = self._validate_run_id(run_id) + self._validate_artifact_name(name) + return self._safe_artifact_path(safe_run_id, Path(f"{name}.parquet")) + + def register_artifact( + self, + run_id: str, + name: ArtifactName, + path: Path | str | None = None, + ) -> dict[str, Any]: + """Record a Parquet artifact path relative to its owning run directory.""" + safe_run_id = self._validate_run_id(run_id) + self._validate_artifact_name(name) + artifact_path = self._safe_artifact_path( + safe_run_id, + Path(path) if path is not None else Path(f"{name}.parquet"), + ) + if artifact_path.suffix.lower() != ".parquet": + raise MiningRunValidationError("mining artifacts must use the .parquet suffix") + run_dir = self._run_dir(safe_run_id) + relative_path = artifact_path.relative_to(run_dir).as_posix() + with _STORE_LOCK: + manifest = self._required_manifest(safe_run_id) + artifacts = dict(manifest.get("artifacts") or {}) + artifacts[name] = relative_path + manifest["artifacts"] = artifacts + manifest["updated_at"] = _now_iso() + _atomic_write_json(run_dir / "manifest.json", manifest) + return manifest + + def append_event( + self, + run_id: str, + event_type: str, + payload: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + """Append a compact event, retaining only the most recent ``MAX_EVENTS`` entries.""" + safe_run_id = self._validate_run_id(run_id) + if not isinstance(event_type, str): + raise MiningRunValidationError("event_type must be a string") + clean_event_type = event_type.strip() + if not clean_event_type or len(clean_event_type) > 64: + raise MiningRunValidationError("event_type must contain 1 to 64 characters") + raw_payload: Mapping[str, Any] | Any = {} if payload is None else payload + if not isinstance(raw_payload, Mapping): + raise MiningRunValidationError("event payload must be a mapping") + clean_payload = cast(dict[str, Any], _canonicalize_json_value(raw_payload)) + encoded_payload = json.dumps( + clean_payload, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + if len(encoded_payload) > MAX_EVENT_PAYLOAD_BYTES: + raise MiningRunValidationError( + f"event payload exceeds {MAX_EVENT_PAYLOAD_BYTES} byte limit" + ) + + with _STORE_LOCK: + self._required_manifest(safe_run_id) + path = self._run_dir(safe_run_id) / "events.jsonl" + events = self._read_events_path(path) + next_id = max((event["id"] for event in events), default=0) + 1 + event = { + "id": next_id, + "timestamp": _now_iso(), + "type": clean_event_type, + "payload": clean_payload, + } + events.append(event) + events = events[-MAX_EVENTS:] + text = "".join( + json.dumps(item, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + "\n" + for item in events + ) + _atomic_write_text(path, text) + return event + + def read_events(self, run_id: str, *, after_id: int = 0) -> list[dict[str, Any]]: + """Read retained events whose monotonically increasing ID is greater than ``after_id``.""" + if isinstance(after_id, bool) or not isinstance(after_id, int) or after_id < 0: + raise MiningRunValidationError("after_id must be a non-negative integer") + safe_run_id = self._validate_run_id(run_id) + with _STORE_LOCK: + self._required_manifest(safe_run_id) + events = self._read_events_path(self._run_dir(safe_run_id) / "events.jsonl") + return [event for event in events if event["id"] > after_id] + + def list_runs( + self, + *, + limit: int = 50, + statuses: Collection[MiningRunStatus] | None = None, + ) -> list[dict[str, Any]]: + """Return recent valid manifests without exposing store paths to API callers.""" + if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 200: + raise MiningRunValidationError("limit must be between 1 and 200") + allowed_statuses = None if statuses is None else set(statuses) + if allowed_statuses is not None and not allowed_statuses <= RUN_STATUSES: + raise MiningRunValidationError("statuses contains an unsupported mining run status") + + try: + paths = list(self.runs_root.glob("*/manifest.json")) + except OSError as exc: + raise MiningRunStoreError("failed to scan mining run manifests") from exc + manifests: list[dict[str, Any]] = [] + for path in paths: + run_id = path.parent.name + if not _RUN_ID_PATTERN.fullmatch(run_id): + continue + try: + manifest = self._read_manifest_path( + self._run_dir(run_id) / "manifest.json", + run_id, + ) + except MiningRunStoreError: + continue + if manifest is None: + continue + if allowed_statuses is not None and manifest.get("status") not in allowed_statuses: + continue + manifests.append(manifest) + manifests.sort(key=_manifest_sort_key, reverse=True) + return manifests[:limit] + + def find_by_signature( + self, + run_signature: str, + *, + statuses: Collection[MiningRunStatus] | None = None, + ) -> dict[str, Any] | None: + """Find the newest run with a signature, optionally restricted to selected statuses.""" + if not isinstance(run_signature, str) or not run_signature: + raise MiningRunValidationError("run_signature must not be empty") + allowed_statuses = None if statuses is None else set(statuses) + if allowed_statuses is not None and not allowed_statuses <= RUN_STATUSES: + raise MiningRunValidationError("statuses contains an unsupported mining run status") + + # Directory enumeration and manifest reads stay outside the write lock. Atomic replacements + # make each individual read coherent while avoiding a lock around a potentially slow scan. + try: + paths = list(self.runs_root.glob("*/manifest.json")) + except OSError as exc: + raise MiningRunStoreError("failed to scan mining run manifests") from exc + matches: list[dict[str, Any]] = [] + for path in paths: + run_id = path.parent.name + if not _RUN_ID_PATTERN.fullmatch(run_id): + continue + try: + manifest_path = self._run_dir(run_id) / "manifest.json" + manifest = self._read_manifest_path(manifest_path, run_id) + except MiningRunStoreError: + continue + if manifest is None or manifest.get("run_signature") != run_signature: + continue + if allowed_statuses is not None and manifest.get("status") not in allowed_statuses: + continue + matches.append(manifest) + return max(matches, key=_manifest_sort_key, default=None) + + def recover_interrupted(self) -> int: + """Mark runs without a live in-process worker as interrupted at startup.""" + try: + paths = list(self.runs_root.glob("*/manifest.json")) + except OSError as exc: + raise MiningRunStoreError("failed to scan mining run manifests") from exc + + candidates: list[str] = [] + for path in paths: + run_id = path.parent.name + if not _RUN_ID_PATTERN.fullmatch(run_id): + continue + try: + manifest_path = self._run_dir(run_id) / "manifest.json" + manifest = self._read_manifest_path(manifest_path, run_id) + except MiningRunStoreError: + continue + if manifest is not None and manifest.get("status") in ACTIVE_RUN_STATUSES: + candidates.append(run_id) + + recovered = 0 + for run_id in candidates: + with _STORE_LOCK: + manifest = self._required_manifest(run_id) + if manifest["status"] not in ACTIVE_RUN_STATUSES: + continue + self._transition_locked(manifest, "interrupted", error=None) + recovered += 1 + return recovered + + def _transition_locked( + self, + manifest: dict[str, Any], + status: MiningRunStatus, + *, + error: str | None, + ) -> dict[str, Any]: + previous = manifest["status"] + if previous == status: + return manifest + if status not in _ALLOWED_TRANSITIONS[previous]: + raise InvalidMiningStatusTransitionError( + f"cannot transition from {previous} to {status}" + ) + + now = _now_iso() + manifest["status"] = status + manifest["updated_at"] = now + if status == "running" and not manifest.get("started_at"): + manifest["started_at"] = now + if status == "cancelling": + manifest["cancellation_requested_at"] = now + if status in TERMINAL_RUN_STATUSES: + manifest["finished_at"] = now + if error is not None: + manifest["error"] = str(error) + _atomic_write_json(self._run_dir(manifest["run_id"]) / "manifest.json", manifest) + return manifest + + def _required_manifest(self, run_id: str) -> dict[str, Any]: + manifest = self._read_manifest_path(self._run_dir(run_id) / "manifest.json", run_id) + if manifest is None: + raise KeyError(run_id) + return manifest + + def _read_manifest_path(self, path: Path, run_id: str) -> dict[str, Any] | None: + if not path.exists(): + return None + value = _read_json(path) + if not isinstance(value, dict): + raise MiningRunStoreError(f"invalid manifest for run {run_id}") + return self._normalize_manifest(value, run_id) + + def _normalize_manifest(self, value: dict[str, Any], run_id: str) -> dict[str, Any]: + status = value.get("status", "queued") + if status not in RUN_STATUSES: + raise MiningRunStoreError(f"invalid status in manifest for run {run_id}") + raw_request = value.get("request") if isinstance(value.get("request"), dict) else {} + data_fingerprint = value.get("data_fingerprint") + signature = value.get("run_signature") + if not isinstance(signature, str) or not signature: + signature = compute_run_signature(raw_request, data_fingerprint) + raw_artifacts = value.get("artifacts") if isinstance(value.get("artifacts"), dict) else {} + artifacts: dict[str, str] = {} + run_dir = self._run_dir(run_id) + for name, raw_path in raw_artifacts.items(): + if name not in ARTIFACT_NAMES or not isinstance(raw_path, str): + continue + try: + safe_path = self._safe_artifact_path(run_id, Path(raw_path)) + except MiningRunValidationError: + continue + if safe_path.suffix.lower() == ".parquet": + artifacts[name] = safe_path.relative_to(run_dir).as_posix() + normalized = dict(value) + normalized.update( + { + "schema_version": value.get("schema_version", 0), + "run_id": run_id, + "status": status, + "request": raw_request, + "data_fingerprint": data_fingerprint, + "run_signature": signature, + "artifacts": artifacts, + "created_at": value.get("created_at"), + "updated_at": value.get("updated_at") or value.get("created_at"), + "started_at": value.get("started_at"), + "finished_at": value.get("finished_at"), + "cancellation_requested_at": value.get("cancellation_requested_at"), + "error": value.get("error"), + } + ) + return normalized + + @staticmethod + def _read_events_path(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeError) as exc: + raise MiningRunStoreError(f"failed to read events file: {path}") from exc + events: list[dict[str, Any]] = [] + for line in lines: + if not line.strip(): + continue + try: + event = json.loads(line) + except json.JSONDecodeError as exc: + raise MiningRunStoreError(f"invalid events file: {path}") from exc + if ( + not isinstance(event, dict) + or isinstance(event.get("id"), bool) + or not isinstance(event.get("id"), int) + or event["id"] <= 0 + ): + raise MiningRunStoreError(f"invalid event record: {path}") + if events and event["id"] <= events[-1]["id"]: + raise MiningRunStoreError(f"non-monotonic event IDs: {path}") + events.append(event) + return events[-MAX_EVENTS:] + + def _run_dir(self, run_id: str) -> Path: + safe_run_id = self._validate_run_id(run_id) + candidate = (self.runs_root / safe_run_id).resolve() + if not candidate.is_relative_to(self.runs_root): + raise MiningRunValidationError("run path escapes mining runs root") + return candidate + + def _safe_artifact_path(self, run_id: str, path: Path) -> Path: + run_dir = self._run_dir(run_id) + candidate = path if path.is_absolute() else run_dir / path + resolved = candidate.resolve() + if not resolved.is_relative_to(run_dir): + raise MiningRunValidationError("artifact path escapes its mining run directory") + return resolved + + @staticmethod + def _validate_run_id(run_id: str) -> str: + if not isinstance(run_id, str) or not _RUN_ID_PATTERN.fullmatch(run_id): + raise MiningRunValidationError("run_id contains unsafe characters") + return run_id + + @staticmethod + def _validate_artifact_name(name: str) -> None: + if name not in ARTIFACT_NAMES: + raise MiningRunValidationError(f"unsupported artifact name: {name!r}") + + +def _canonicalize_json_value(value: Any) -> Any: + if value is None or isinstance(value, (str, bool, int)): + return value + if isinstance(value, float): + if not math.isfinite(value): + raise MiningRunValidationError("non-finite numbers are not supported") + return value + if isinstance(value, Enum): + return _canonicalize_json_value(value.value) + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, Path): + return str(value) + if isinstance(value, Mapping): + result: dict[str, Any] = {} + for key, item in value.items(): + if not isinstance(key, str): + raise MiningRunValidationError("JSON mapping keys must be strings") + result[key] = _canonicalize_json_value(item) + return {key: result[key] for key in sorted(result)} + if isinstance(value, (list, tuple)): + return [_canonicalize_json_value(item) for item in value] + if isinstance(value, (set, frozenset)): + items = [_canonicalize_json_value(item) for item in value] + return sorted( + items, + key=lambda item: json.dumps( + item, ensure_ascii=False, allow_nan=False, sort_keys=True, separators=(",", ":") + ), + ) + raise MiningRunValidationError(f"value is not JSON serializable: {type(value).__name__}") + + +def _read_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise MiningRunStoreError(f"failed to read JSON file: {path}") from exc + + +def _atomic_write_json(path: Path, value: Any) -> None: + try: + text = json.dumps(value, ensure_ascii=False, allow_nan=False, indent=2) + "\n" + except (TypeError, ValueError) as exc: + raise MiningRunValidationError("value is not JSON serializable") from exc + _atomic_write_text(path, text) + + +def _atomic_write_text(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + with temporary.open("w", encoding="utf-8", newline="\n") as stream: + stream.write(text) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + except OSError as exc: + temporary.unlink(missing_ok=True) + raise MiningRunStoreError(f"failed to write file: {path}") from exc + + +def _now_iso() -> str: + return datetime.now(UTC).isoformat() + + +def _manifest_sort_key(manifest: dict[str, Any]) -> tuple[str, str]: + return ( + str(manifest.get("updated_at") or manifest.get("created_at") or ""), + str(manifest.get("run_id") or ""), + ) diff --git a/backend/app/services/mining_manager.py b/backend/app/services/mining_manager.py new file mode 100644 index 0000000..0422ac3 --- /dev/null +++ b/backend/app/services/mining_manager.py @@ -0,0 +1,263 @@ +"""Threaded orchestration for persistent mining jobs.""" + +from __future__ import annotations + +import threading +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from app.backtest.worker import make_worker_task, run_worker_task +from app.services.heavy_job_limiter import ( + HeavyJobCancelledError, + shared_heavy_job_limiter, +) +from app.services.mining_jobs import ( + ACTIVE_RUN_STATUSES, + SUCCESS_RUN_STATUSES, + TERMINAL_RUN_STATUSES, + MiningRunStore, + MiningRunValidationError, + compute_run_signature, +) + +WorkerRunner = Callable[ + [dict[str, Any], Callable[[dict[str, Any]], None], threading.Event], + dict[str, Any], +] +TaskFactory = Callable[[str, Path, dict[str, Any]], dict[str, Any]] + +_SUCCESS_STATUSES = {"succeeded", "succeeded_with_budget_exhausted"} +_SHUTDOWN_JOIN_SECONDS = 1.0 + + +class MiningJobManager: + """Coordinate mining persistence, capacity, cancellation, and worker threads.""" + + def __init__( + self, + data_dir: Path | str, + worker_runner: WorkerRunner = run_worker_task, + task_factory: TaskFactory = make_worker_task, + ) -> None: + self._data_dir = Path(data_dir).resolve() + self._store = MiningRunStore(self._data_dir) + self._worker_runner = worker_runner + self._task_factory = task_factory + self._lock = threading.RLock() + self._threads: dict[str, threading.Thread] = {} + self._cancel_events: dict[str, threading.Event] = {} + self._shutdown = False + + @property + def store(self) -> MiningRunStore: + return self._store + + def start( + self, + request: dict[str, Any], + data_fingerprint: Any, + force: bool = False, + source: str = "manual", + run_id: str | None = None, + ) -> dict[str, Any]: + signature = compute_run_signature(request, data_fingerprint) + with self._lock: + if self._shutdown: + raise RuntimeError("mining job manager is shut down") + if not force: + active = self._store.find_by_signature( + signature, + statuses=ACTIVE_RUN_STATUSES, + ) + if active is not None: + return active + succeeded = self._store.find_by_signature( + signature, + statuses=SUCCESS_RUN_STATUSES, + ) + if succeeded is not None: + return succeeded + + try: + manifest = self._store.create( + request, + data_fingerprint, + run_id=run_id, + ) + except MiningRunValidationError: + if run_id is None: + raise + existing = self._store.get(run_id) + if existing is None: + raise + return existing + run_id = manifest["run_id"] + self._store.append_event( + run_id, + "queued", + {"status": "queued", "source": source}, + ) + self._start_thread_locked(run_id, source) + return manifest + + def cancel(self, run_id: str) -> dict[str, Any]: + with self._lock: + manifest = self._store.get(run_id) + if manifest is None: + raise KeyError(run_id) + if manifest["status"] in TERMINAL_RUN_STATUSES: + return manifest + + cancel_event = self._cancel_events.get(run_id) + if cancel_event is None: + cancelled = self._store.transition_status(run_id, "cancelled") + self._store.append_event(run_id, "cancelled", {"status": "cancelled"}) + return cancelled + + cancel_event.set() + if manifest["status"] != "cancelling": + manifest = self._store.transition_status(run_id, "cancelling") + self._store.append_event(run_id, "cancelling", {"status": "cancelling"}) + return manifest + + def shutdown(self) -> None: + with self._lock: + self._shutdown = True + run_ids = list(self._threads) + for run_id in run_ids: + self.cancel(run_id) + + deadline = time.monotonic() + _SHUTDOWN_JOIN_SECONDS + current = threading.current_thread() + for run_id in run_ids: + with self._lock: + thread = self._threads.get(run_id) + if thread is None or thread is current: + continue + remaining = deadline - time.monotonic() + if remaining <= 0: + break + thread.join(timeout=remaining) + + def recover_interrupted(self) -> int: + return self._store.recover_interrupted() + + def _start_thread_locked(self, run_id: str, source: str) -> None: + if run_id in self._threads: + return + cancel_event = threading.Event() + thread = threading.Thread( + target=self._run_job, + args=(run_id, source, cancel_event), + name=f"mining-{run_id}", + daemon=True, + ) + self._cancel_events[run_id] = cancel_event + self._threads[run_id] = thread + thread.start() + + def _run_job( + self, + run_id: str, + source: str, + cancel_event: threading.Event, + ) -> None: + try: + with shared_heavy_job_limiter.slot("mining", cancel_event=cancel_event): + if not self._mark_running(run_id, cancel_event): + return + manifest = self._store.get(run_id) + if manifest is None: + raise KeyError(run_id) + payload = { + "run_id": run_id, + "request": manifest["request"], + "data_fingerprint": manifest["data_fingerprint"], + "source": source, + } + task = self._task_factory("mining", self._data_dir, payload) + result = self._worker_runner( + task, + lambda progress: self._record_progress(run_id, progress, cancel_event), + cancel_event, + ) + if not isinstance(result, dict): + raise TypeError("mining worker result must be a compact dict") + self._finish_success(run_id, result, cancel_event) + except HeavyJobCancelledError: + self._finish_cancelled(run_id) + except Exception as exc: + if cancel_event.is_set(): + self._finish_cancelled(run_id) + else: + self._finish_failed(run_id, exc) + finally: + with self._lock: + self._threads.pop(run_id, None) + self._cancel_events.pop(run_id, None) + + def _mark_running(self, run_id: str, cancel_event: threading.Event) -> bool: + with self._lock: + if cancel_event.is_set(): + self._finish_cancelled_locked(run_id) + return False + self._store.transition_status(run_id, "running") + self._store.append_event(run_id, "running", {"status": "running"}) + return True + + def _record_progress( + self, + run_id: str, + progress: dict[str, Any], + cancel_event: threading.Event, + ) -> None: + if not isinstance(progress, dict): + raise TypeError("mining progress must be a compact dict") + with self._lock: + if cancel_event.is_set(): + return + self._store.append_event(run_id, "progress", progress) + self._store.write_summary(run_id, {"progress": progress}) + + def _finish_success( + self, + run_id: str, + result: dict[str, Any], + cancel_event: threading.Event, + ) -> None: + status = result.get("status", "succeeded") + if status not in _SUCCESS_STATUSES: + raise ValueError(f"unsupported mining worker status: {status!r}") + with self._lock: + if cancel_event.is_set(): + self._finish_cancelled_locked(run_id) + return + self._store.write_summary(run_id, result) + self._store.transition_status(run_id, status) + self._store.append_event(run_id, status, {"status": status}) + + def _finish_cancelled(self, run_id: str) -> None: + with self._lock: + self._finish_cancelled_locked(run_id) + + def _finish_cancelled_locked(self, run_id: str) -> None: + manifest = self._store.get(run_id) + if manifest is None or manifest["status"] in TERMINAL_RUN_STATUSES: + return + self._store.transition_status(run_id, "cancelled") + self._store.append_event(run_id, "cancelled", {"status": "cancelled"}) + + def _finish_failed(self, run_id: str, exc: Exception) -> None: + message = str(exc)[:2000] + with self._lock: + manifest = self._store.get(run_id) + if manifest is None or manifest["status"] in TERMINAL_RUN_STATUSES: + return + self._store.transition_status(run_id, "failed", error=message) + self._store.append_event( + run_id, + "error", + {"status": "failed", "message": message}, + ) diff --git a/backend/app/services/mining_preflight.py b/backend/app/services/mining_preflight.py new file mode 100644 index 0000000..6f234ee --- /dev/null +++ b/backend/app/services/mining_preflight.py @@ -0,0 +1,139 @@ +"""Lightweight mining date availability checks shared by API and workers.""" +from __future__ import annotations + +from dataclasses import asdict, dataclass +from datetime import date +from pathlib import Path +from typing import Any + +from app.backtest.mining import ( + nested_fold_count, + required_outer_folds, + required_trading_bars, + validation_config_for_profile, +) +from app.tickflow.repository import enriched_dirname + + +@dataclass(frozen=True) +class MiningAvailability: + asset_type: str + budget_profile: str + trading_bars: int + required_bars: int + outer_folds: int + required_outer_folds: int + eligible: bool + available_start: date | None + available_end: date | None + effective_start: date | None + effective_end: date | None + suggested_start: date | None + + def to_dict(self) -> dict[str, Any]: + return { + key: value.isoformat() if isinstance(value, date) else value + for key, value in asdict(self).items() + } + + +def enriched_partition_dates( + data_dir: Path, + asset_type: str, + start: date | None = None, + end: date | None = None, +) -> list[date]: + root = data_dir / enriched_dirname(asset_type) + values: set[date] = set() + for partition in root.glob("date=*"): + try: + value = date.fromisoformat(partition.name.removeprefix("date=")) + except ValueError: + continue + if start is not None and value < start: + continue + if end is not None and value > end: + continue + if (partition / "part.parquet").is_file(): + values.add(value) + return sorted(values) + + +def mining_availability( + data_dir: Path, + *, + asset_type: str, + budget_profile: str, + start: date | None = None, + end: date | None = None, +) -> MiningAvailability: + if asset_type not in {"stock", "etf"}: + raise ValueError(f"unsupported mining asset type: {asset_type}") + if start is not None and end is not None and start > end: + raise ValueError("mining start must not be after end") + + config = validation_config_for_profile(budget_profile) + required_folds = required_outer_folds(budget_profile) + required_bars = required_trading_bars(config, required_folds) + all_dates = enriched_partition_dates(data_dir, asset_type) + scoped = [ + value + for value in all_dates + if (start is None or value >= start) and (end is None or value <= end) + ] + dates_through_end = [ + value for value in all_dates if end is None or value <= end + ] + suggested_start = ( + dates_through_end[-required_bars] + if len(dates_through_end) >= required_bars + else None + ) + trading_bars = len(scoped) + return MiningAvailability( + asset_type=asset_type, + budget_profile=budget_profile, + trading_bars=trading_bars, + required_bars=required_bars, + outer_folds=nested_fold_count(trading_bars, config), + required_outer_folds=required_folds, + eligible=trading_bars >= required_bars, + available_start=all_dates[0] if all_dates else None, + available_end=all_dates[-1] if all_dates else None, + effective_start=scoped[0] if scoped else None, + effective_end=scoped[-1] if scoped else None, + suggested_start=suggested_start, + ) + + +def require_mining_availability( + data_dir: Path, + *, + asset_type: str, + budget_profile: str, + start: date | None = None, + end: date | None = None, +) -> MiningAvailability: + availability = mining_availability( + data_dir, + asset_type=asset_type, + budget_profile=budget_profile, + start=start, + end=end, + ) + if availability.eligible: + return availability + + if availability.effective_start is None: + effective_range = "contains no enriched data" + else: + effective_range = ( + f"{availability.effective_start.isoformat()} to " + f"{availability.effective_end.isoformat()}" + ) + fold_label = "outer fold" if availability.required_outer_folds == 1 else "outer folds" + raise ValueError( + f"{budget_profile} mining requires at least {availability.required_bars} " + f"enriched trading bars for {availability.required_outer_folds} {fold_label}; " + f"effective range {effective_range} has {availability.trading_bars}" + ) diff --git a/backend/app/services/mining_process_lock.py b/backend/app/services/mining_process_lock.py new file mode 100644 index 0000000..f0323b0 --- /dev/null +++ b/backend/app/services/mining_process_lock.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import BinaryIO + + +class MiningProcessLockError(RuntimeError): + """Another application process owns mining for this data directory.""" + + +class MiningProcessLock: + def __init__(self, data_dir: Path) -> None: + self._path = Path(data_dir) / ".mining_process.lock" + self._stream: BinaryIO | None = None + + def acquire(self) -> None: + if self._stream is not None: + return + self._path.parent.mkdir(parents=True, exist_ok=True) + stream = self._path.open("a+b") + try: + stream.seek(0, os.SEEK_END) + if stream.tell() == 0: + stream.write(b"0") + stream.flush() + os.set_inheritable(stream.fileno(), False) + _try_lock_file(stream) + except BaseException: + stream.close() + raise + self._stream = stream + + def release(self) -> None: + stream = self._stream + if stream is None: + return + self._stream = None + try: + _unlock_file(stream) + finally: + stream.close() + + +def _try_lock_file(stream: BinaryIO) -> None: + if os.name == "nt": + import msvcrt + + stream.seek(0) + try: + msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) + except OSError as exc: + raise MiningProcessLockError( + "another application process already owns mining for this data directory" + ) from exc + return + + import fcntl + + try: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + raise MiningProcessLockError( + "another application process already owns mining for this data directory" + ) from exc + + +def _unlock_file(stream: BinaryIO) -> None: + if os.name == "nt": + import msvcrt + + stream.seek(0) + msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) + return + + import fcntl + + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) diff --git a/backend/app/services/mining_schedule.py b/backend/app/services/mining_schedule.py new file mode 100644 index 0000000..6a6475d --- /dev/null +++ b/backend/app/services/mining_schedule.py @@ -0,0 +1,391 @@ +"""Weekly scheduled mining orchestration and deterministic data claims.""" + +from __future__ import annotations + +import hashlib +import json +import logging +import threading +from datetime import date, datetime +from pathlib import Path +from typing import Any +from zoneinfo import ZoneInfo + +import polars as pl + +from app.backtest.factor import FACTOR_COLUMNS, FACTOR_METHODOLOGY_VERSION +from app.backtest.mining import ( + required_outer_folds, + required_trading_bars, + validation_config_for_profile, +) +from app.services import preferences +from app.services.mining_preflight import enriched_partition_dates +from app.services.regime_builder import load_regime_history, regime_path + +logger = logging.getLogger(__name__) + +BEIJING_TZ = ZoneInfo("Asia/Shanghai") +MINING_ALGORITHM_VERSION = "mining-v2" +FINGERPRINT_VERSION = "weekly-mining-data-v2" +_PROFILES = frozenset({"balanced", "strict"}) +_CLAIM_LOCK = threading.Lock() + + +def beijing_now(now: datetime | None = None) -> datetime: + """Return an aware Beijing datetime without depending on the server timezone.""" + if now is None: + return datetime.now(BEIJING_TZ) + if now.tzinfo is None: + return now.replace(tzinfo=BEIJING_TZ) + return now.astimezone(BEIJING_TZ) + + +def beijing_date(now: datetime | None = None) -> date: + return beijing_now(now).date() + + +def iso_week(value: date) -> tuple[int, int]: + iso_year, week, _ = value.isocalendar() + return iso_year, week + + +def build_default_request(repo: Any, profile: str) -> dict[str, Any]: + """Build the bounded V1 stock/full-market request used by the scheduler.""" + if profile not in _PROFILES: + raise ValueError(f"unsupported mining profile: {profile}") + latest = repo.latest_enriched_date("stock") + end = latest.isoformat() if latest is not None else None + return { + "factor_names": [item["id"] for item in FACTOR_COLUMNS[:48]], + "strategy_ids": [], + "symbols": None, + "asset_type": "stock", + "start": None, + "end": end, + "budget_profile": profile, + "require_regime": True, + } + + +def build_data_fingerprint( + repo: Any, + app_state: Any, + request: dict[str, Any], +) -> dict[str, Any]: + """Hash one stable managed generation plus source metadata.""" + for _attempt in range(2): + fingerprint = _build_data_fingerprint_once(repo, app_state, request) + if repo.get_matrix_data_generation(fingerprint["asset_type"]) == fingerprint["generation"]: + return fingerprint + raise ValueError("enriched data changed while building the mining fingerprint") + + +def _build_data_fingerprint_once( + repo: Any, + app_state: Any, + request: dict[str, Any], +) -> dict[str, Any]: + data_dir = Path(repo.store.data_dir) + asset_type = str(request.get("asset_type") or "stock") + enriched_root = ( + data_dir / "kline_daily_enriched" + if asset_type == "stock" + else data_dir / f"kline_{asset_type}_enriched" + ) + module_root = Path(__file__).resolve().parents[1] + components = { + "version": FINGERPRINT_VERSION, + "asset_type": asset_type, + "generation": repo.get_matrix_data_generation(asset_type), + "latest_enriched_date": _iso_or_none(repo.latest_enriched_date(asset_type)), + "enriched": _enriched_metadata(enriched_root), + "instruments": _instrument_metadata(repo, asset_type), + "regime": _path_metadata(regime_path(data_dir), root=data_dir), + "algorithm_version": MINING_ALGORITHM_VERSION, + "methodology_version": FACTOR_METHODOLOGY_VERSION, + "implementation": _implementation_metadata(module_root), + "strategies": _selected_strategy_metadata( + app_state, + request.get("strategy_ids") or [], + data_dir, + ), + } + payload = _canonical_json(components) + return { + **components, + "digest": hashlib.sha256(payload.encode("utf-8")).hexdigest(), + } + + +def schedule_claim(day: date) -> str: + iso_year, week = iso_week(day) + return f"weekly-{iso_year}-W{week:02d}" + + +def run_weekly_mining(app_state: Any, *, now: datetime | None = None) -> dict[str, Any]: + """Check the weekly gate and enqueue mining; never perform mining synchronously.""" + config = preferences.get_mining_schedule() + day = beijing_date(now) + if not config["mining_schedule_enabled"]: + return {"status": "disabled"} + weekday = day.weekday() + if weekday > 4 or weekday < config["mining_schedule_weekday"]: + return {"status": "weekday_mismatch"} + + manager = getattr(app_state, "mining_manager", None) + repo = getattr(app_state, "repo", None) + if manager is None or repo is None: + raise RuntimeError("scheduled mining dependencies are not initialized") + store = getattr(manager, "store", None) + if store is None: + raise RuntimeError("scheduled mining manager has no run store") + + request = build_default_request(repo, config["mining_budget_profile"]) + fingerprint = build_data_fingerprint(repo, app_state, request) + claim = schedule_claim(day) + fingerprint = {**fingerprint, "source": "scheduled", "source_claim": claim} + + with _CLAIM_LOCK: + existing = store.get(claim) + if existing is not None: + return {"status": "already_claimed", "run_id": claim} + + prerequisite_error = _prerequisite_error(repo, request) + if prerequisite_error is not None: + _record_skipped_prerequisite(store, claim, request, fingerprint, prerequisite_error) + return { + "status": "skipped_prerequisite", + "run_id": claim, + "error": prerequisite_error, + } + + run = manager.start( + request, + fingerprint, + force=False, + source="scheduled", + run_id=claim, + ) + run_id = run.get("run_id") if isinstance(run, dict) else getattr(run, "run_id", None) + return {"status": "enqueued", "run_id": run_id or claim} + + +def _prerequisite_error(repo: Any, request: dict[str, Any]) -> str | None: + data_dir = Path(repo.store.data_dir) + end = request.get("end") + if end is None: + return "stock enriched data is unavailable" + regime = regime_path(data_dir) + try: + if not regime.is_file() or regime.stat().st_size <= 0: + return "regime data is unavailable" + except OSError: + return "regime data is unavailable" + + start = request.get("start") + start_date = date.fromisoformat(start) if start is not None else None + end_date = date.fromisoformat(end) + partitions = enriched_partition_dates( + data_dir, + "stock", + start_date, + end_date, + ) + covered = [value.isoformat() for value in partitions] + profile = request["budget_profile"] + validation = validation_config_for_profile(profile) + required = required_trading_bars( + validation, + required_outer_folds(profile), + ) + if len(covered) < required: + return ( + "insufficient enriched trading dates: " + f"need {required}, got {len(covered)}" + ) + + regime_history = load_regime_history(data_dir) + if regime_history.is_empty() or "date" not in regime_history.columns: + return "regime data is unavailable" + regime_dates = set( + regime_history.select( + pl.col("date").cast(pl.Utf8).str.slice(0, 10) + ).to_series().to_list() + ) + required_predecessors = set(covered[:-1]) + missing_regime = required_predecessors - regime_dates + if missing_regime: + return ( + "regime coverage is incomplete for T-1 alignment: " + f"missing {len(missing_regime)} trading dates" + ) + return None + + +def _record_skipped_prerequisite( + store: Any, + claim: str, + request: dict[str, Any], + fingerprint: dict[str, Any], + error: str, +) -> None: + try: + store.create(request, fingerprint, run_id=claim) + except Exception: + if store.get(claim) is not None: + return + raise + store.append_event( + claim, + "skipped_prerequisite", + {"source": "scheduled", "reason": error}, + ) + store.transition_status(claim, "skipped_prerequisite", error=error) + + +def _instrument_metadata(repo: Any, asset_type: str) -> dict[str, Any]: + instruments = repo.get_instruments_asset(asset_type) + if instruments is None or instruments.is_empty() or "symbol" not in instruments.columns: + return {"rows": 0, "digest": "no-instruments"} + columns = [ + name + for name in ( + "symbol", + "name", + "total_shares", + "float_shares", + "limit_up", + "limit_down", + ) + if name in instruments.columns + ] + payload = instruments.select(columns).sort("symbol").to_dicts() + digest = hashlib.blake2b( + json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8"), + digest_size=20, + ).hexdigest() + return {"rows": instruments.height, "columns": columns, "digest": digest} + + +def _enriched_metadata(root: Path) -> dict[str, Any]: + records: list[dict[str, Any]] = [] + for partition in sorted(root.glob("date=*"), key=lambda item: item.name): + try: + date.fromisoformat(partition.name.removeprefix("date=")) + except ValueError: + continue + records.append( + { + "partition": partition.name, + "file": _path_metadata(partition / "part.parquet", root=root), + } + ) + return { + "partition_count": len(records), + "first_partition": records[0]["partition"] if records else None, + "last_partition": records[-1]["partition"] if records else None, + "metadata_digest": hashlib.sha256(_canonical_json(records).encode("utf-8")).hexdigest(), + } + + +def _selected_strategy_metadata( + app_state: Any, + strategy_ids: list[str], + data_dir: Path, +) -> list[dict[str, Any]]: + if not strategy_ids: + return [] + engine = getattr(app_state, "strategy_engine", None) + if engine is None: + raise RuntimeError("strategy engine is unavailable for scheduled mining fingerprint") + metadata: list[dict[str, Any]] = [] + for strategy_id in sorted(strategy_ids): + strategy = engine.get(strategy_id) + if strategy.execution_backend != "matrix_native": + raise ValueError(f"scheduled mining strategy is not matrix-native: {strategy_id}") + source_path = Path(strategy.file_path) if strategy.file_path is not None else None + override_path = data_dir / "user_data" / "strategy_overrides" / f"{strategy_id}.json" + metadata.append( + { + "strategy_id": strategy_id, + "source": _content_metadata( + source_path, root=source_path.parent if source_path else data_dir + ), + "source_tree": ( + _implementation_metadata(source_path.parent) + if source_path is not None + else None + ), + "override": _content_metadata(override_path, root=data_dir), + } + ) + return metadata + + +def _path_metadata(path: Path | None, *, root: Path) -> dict[str, Any] | None: + if path is None: + return None + try: + stat = path.stat() + except OSError: + return {"path": _relative_path(path, root), "exists": False} + return { + "path": _relative_path(path, root), + "exists": True, + "size": stat.st_size, + "mtime_ns": stat.st_mtime_ns, + } + + +def _content_metadata(path: Path | None, *, root: Path) -> dict[str, Any] | None: + if path is None: + return None + try: + content = path.read_bytes() + except OSError: + return {"path": _relative_path(path, root), "exists": False} + return { + "path": _relative_path(path, root), + "exists": True, + "size": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + } + + +def _implementation_metadata(module_root: Path) -> dict[str, Any]: + records = [] + for path in sorted(module_root.rglob("*.py"), key=lambda item: item.as_posix()): + try: + content = path.read_bytes() + except OSError: + continue + records.append({ + "path": path.relative_to(module_root).as_posix(), + "sha256": hashlib.sha256(content).hexdigest(), + }) + return { + "file_count": len(records), + "digest": hashlib.sha256(_canonical_json(records).encode("utf-8")).hexdigest(), + } + + +def _relative_path(path: Path, root: Path) -> str: + try: + return path.resolve().relative_to(root.resolve()).as_posix() + except ValueError: + return path.name + + +def _iso_or_none(value: date | None) -> str | None: + return value.isoformat() if value is not None else None + + +def _canonical_json(value: Any) -> str: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) diff --git a/backend/app/services/preferences.py b/backend/app/services/preferences.py index 8bb2fb2..43fc60e 100644 --- a/backend/app/services/preferences.py +++ b/backend/app/services/preferences.py @@ -5,12 +5,19 @@ """ from __future__ import annotations +import copy import json import logging +import re from pathlib import Path logger = logging.getLogger(__name__) +# 进程内缓存: 行情轮询线程一轮会调用 8~12 次 getter, 每次读盘+parse 是纯重复; +# 文件仅在用户改设置时变化, 以 (mtime_ns, size) 签名判断是否重读。 +_cache: dict | None = None +_cache_sig: tuple[int, int] | None = None + def _path() -> Path: from app.config import settings @@ -19,14 +26,32 @@ def _path() -> Path: return p +def _invalidate_cache() -> None: + global _cache, _cache_sig + _cache = None + _cache_sig = None + + def load() -> dict: + """读取 preferences.json (带 mtime 签名缓存)。返回深拷贝, 调用方可自由修改。""" + global _cache, _cache_sig p = _path() - if p.exists(): - try: - return json.loads(p.read_text(encoding="utf-8")) - except Exception as e: # noqa: BLE001 - logger.warning("preferences.json malformed: %s", e) - return {} + try: + sig = (p.stat().st_mtime_ns, p.stat().st_size) + except OSError: + return {} + if _cache is not None and sig == _cache_sig: + return copy.deepcopy(_cache) + try: + data = json.loads(p.read_text(encoding="utf-8")) + except FileNotFoundError: + return {} + except Exception as e: + logger.warning("preferences.json malformed: %s", e) + return {} + _cache = data + _cache_sig = sig + return copy.deepcopy(_cache) def save(updates: dict) -> dict: @@ -36,6 +61,7 @@ def save(updates: dict) -> dict: _path().write_text( json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8", ) + _invalidate_cache() return current @@ -88,6 +114,7 @@ def set_realtime_quote_interval(interval: float) -> float: _path().write_text( json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8", ) + _invalidate_cache() return interval @@ -316,6 +343,74 @@ def get_regime_warmup_days() -> int: return 40 +# ── 市场主线(概念/行业涨停梯队)过滤 ── +# 宽基/风格标签(融资融券 ~7700 成分、深股通/沪股通 ~3300-3700、国企改革 ~2900) +# 会按"家数"霸占主线榜首, 但它们不是可操作的题材主线。默认按成分股数上限过滤。 +# 标定(2026-08 THS 概念): 成员 >600 的 55 个概念几乎全是此类风格标签, +# 真实题材(华为概念 2006/人工智能 2166/固态电池等)均在 600 以下或可自行调整。 +_MAINLINE_MAX_MEMBERS_MIN = 50 +_MAINLINE_MAX_MEMBERS_MAX = 5000 +_MAINLINE_MIN_MEMBERS_MIN = 1 +_MAINLINE_MIN_MEMBERS_MAX = 200 + + +def get_mainline_max_members() -> int: + """主线维度成员数上限, 超过视为宽基/风格标签被过滤。默认 600。""" + v = load().get("mainline_max_members", 600) + try: + return max(_MAINLINE_MAX_MEMBERS_MIN, min(_MAINLINE_MAX_MEMBERS_MAX, int(v))) + except (TypeError, ValueError): + return 600 + + +def get_mainline_min_members() -> int: + """主线维度成员数下限, 过滤微型标签。默认 4。""" + v = load().get("mainline_min_members", 4) + try: + return max(_MAINLINE_MIN_MEMBERS_MIN, min(_MAINLINE_MIN_MEMBERS_MAX, int(v))) + except (TypeError, ValueError): + return 4 + + +def get_mainline_blacklist() -> list[str]: + """用户自定义屏蔽的维度成员名(不论成员数大小)。默认空。 + + 保存时接受 list 或逗号/顿号/分号/空白分隔的字符串。 + """ + v = load().get("mainline_blacklist", []) + if isinstance(v, str): + v = [part for part in re.split(r"[,，、;；\s]+", v) if part] # noqa: RUF001 + if not isinstance(v, list): + return [] + return [str(x).strip() for x in v if str(x).strip()] + + +def get_mainline_filter_config() -> dict: + """主线过滤配置汇总(供 API 返回与计算读取)。""" + return { + "min_members": get_mainline_min_members(), + "max_members": get_mainline_max_members(), + "blacklist": get_mainline_blacklist(), + } + + +def set_mainline_filter_config(cfg: dict) -> dict: + """保存主线过滤配置(白名单字段, 部分更新)。修改后需重算主线生效。""" + updates: dict = {} + if "min_members" in cfg and cfg["min_members"] is not None: + updates["mainline_min_members"] = cfg["min_members"] + if "max_members" in cfg and cfg["max_members"] is not None: + updates["mainline_max_members"] = cfg["max_members"] + if "blacklist" in cfg and cfg["blacklist"] is not None: + raw = cfg["blacklist"] + if isinstance(raw, str): + raw = [part for part in re.split(r"[,，、;；\s]+", raw) if part] # noqa: RUF001 + updates["mainline_blacklist"] = [str(x).strip() for x in (raw or []) if str(x).strip()] + if updates: + save(updates) + return get_mainline_filter_config() + + _PIPELINE_PULL_KEYS = ("pipeline_pull_etf", "pipeline_pull_index") @@ -475,6 +570,43 @@ def set_review_schedule(enabled: bool, hour: int, minute: int) -> dict: return {"enabled": bool(enabled), "hour": h, "minute": m} +MINING_BUDGET_PROFILES = frozenset({"balanced", "strict"}) + + +def get_mining_schedule() -> dict: + """返回周度自动 mining 配置。历史配置缺字段时默认关闭。""" + data = load() + weekday = data.get("mining_schedule_weekday", 4) + if isinstance(weekday, bool) or not isinstance(weekday, int) or not 0 <= weekday <= 4: + weekday = 4 + profile = data.get("mining_budget_profile", "balanced") + if not isinstance(profile, str) or profile not in MINING_BUDGET_PROFILES: + profile = "balanced" + enabled = data.get("mining_schedule_enabled", False) + if not isinstance(enabled, bool): + enabled = False + return { + "mining_schedule_enabled": enabled, + "mining_schedule_weekday": weekday, + "mining_budget_profile": profile, + } + + +def set_mining_schedule(enabled: bool, weekday: int, profile: str) -> dict: + """校验并一次写入周度自动 mining 的整组配置。""" + if isinstance(weekday, bool) or not isinstance(weekday, int) or not 0 <= weekday <= 4: + raise ValueError("mining schedule weekday must be between 0 and 4") + if profile not in MINING_BUDGET_PROFILES: + raise ValueError("mining budget profile must be balanced or strict") + result = { + "mining_schedule_enabled": bool(enabled), + "mining_schedule_weekday": weekday, + "mining_budget_profile": profile, + } + save(result) + return result + + def get_review_push_channels() -> list[str]: """复盘推送渠道(多选) — 选定的外部工具列表, 复盘归档后逐个推送。 diff --git a/backend/app/services/quote_service.py b/backend/app/services/quote_service.py index a8fe985..38f2232 100644 --- a/backend/app/services/quote_service.py +++ b/backend/app/services/quote_service.py @@ -130,18 +130,38 @@ class QuoteSubscriber: self._event.set() +# 落盘节流间隔: last_fetch_ms 仅在进程重启后用于显示"最后获取时间"(运行中读内存值), +# 每 30s 持久化一次足够, 避免 expert 档每秒一轮的全量 preferences 重写磁盘。 +_LAST_FETCH_WRITE_INTERVAL_MS = 30_000.0 +_last_fetch_written_at_ms: float = 0.0 + + def _persist_last_fetch(fetched_at_ms: float) -> None: """把"最后获取"时间戳持久化到 preferences, 使进程重启后仍可显示。 放在锁外调用 (IO); 失败不影响主流程 (内存值已更新, 下次 fetch 再写)。 + 距上次成功落盘不足 30s 时跳过 (节流只影响落盘频率, 内存值不受影响)。 """ + global _last_fetch_written_at_ms + if (fetched_at_ms - _last_fetch_written_at_ms) < _LAST_FETCH_WRITE_INTERVAL_MS: + return try: from app.services import preferences preferences.save({"last_fetch_ms": round(fetched_at_ms, 0)}) + _last_fetch_written_at_ms = fetched_at_ms except Exception as e: # noqa: BLE001 logger.debug("last_fetch_ms 持久化失败 (不影响行情): %s", e) +def _monitor_name_map(repo) -> dict[str, str]: + """监控回填用的 symbol → name 映射 (股票 + ETF + 指数, 股票优先)。 + + 走 repo.get_name_map() 的进程内 memo (三份 instruments 维表刷新时失效), + 避免每轮监控对 ~7000 行维表 iter_rows 重建。过滤空名称与旧行为一致。 + """ + return {s: n for s, n in repo.get_name_map().items() if n} + + class QuoteService: """全局实时行情服务 — 单例。""" @@ -1081,28 +1101,10 @@ class QuoteService: engine = getattr(self._app_state, "monitor_engine", None) if engine and engine.rule_count > 0: # 预构建 symbol → name 映射 (enriched 已 drop name 列, 引擎触发时回填用)。 - # 含股票 + ETF 维表, 保证 ETF 监控告警也能回填名称。 + # 股票 + ETF + 指数三表合并走 _monitor_name_map -> repo.get_name_map() + # 的进程内 memo, 避免每轮监控对 ~7000 行维表 iter_rows 重建。 try: - name_map: dict[str, str] = {} - inst_df = self._app_state.repo.get_instruments() - if not inst_df.is_empty() and "symbol" in inst_df.columns and "name" in inst_df.columns: - for row in inst_df.select(["symbol", "name"]).iter_rows(named=True): - if row.get("name"): - name_map[row["symbol"]] = row["name"] - # 仅当存在 ETF 规则时补 ETF 维表 (股票名优先, setdefault 不覆盖股票) - if engine.has_asset_rules("etf"): - etf_inst = self._app_state.repo.get_etf_instruments() - if not etf_inst.is_empty() and "symbol" in etf_inst.columns and "name" in etf_inst.columns: - for row in etf_inst.select(["symbol", "name"]).iter_rows(named=True): - if row.get("name"): - name_map.setdefault(row["symbol"], row["name"]) - # 仅当存在指数规则时补指数维表 (setdefault 不覆盖股票/ETF) - if engine.has_asset_rules("index"): - idx_inst = self._app_state.repo.get_instruments_asset("index") - if not idx_inst.is_empty() and "symbol" in idx_inst.columns and "name" in idx_inst.columns: - for row in idx_inst.select(["symbol", "name"]).iter_rows(named=True): - if row.get("name"): - name_map.setdefault(row["symbol"], row["name"]) + name_map = _monitor_name_map(self._app_state.repo) if name_map: engine.set_name_map(name_map) except Exception as e: # noqa: BLE001 diff --git a/backend/app/services/regime_builder.py b/backend/app/services/regime_builder.py index 3143779..b78f429 100644 --- a/backend/app/services/regime_builder.py +++ b/backend/app/services/regime_builder.py @@ -150,7 +150,16 @@ def _aggregate_daily(df: pl.DataFrame, index_pct_map: dict | None = None) -> pl. 纯 polars 聚合, 不重算指标(假设 df 已含 signal_*/change_pct/ma20 等列)。 index_pct_map: {date: 指数涨幅} 可选, 由调用方从指数数据预先算好。 + 梯队指标(首板/N板宽度/晋级率)由 market_phase 提供; phase 列不在此算 + (需要完整日序做平滑), 由 refresh_phase_labels 在 upsert 后统一重标。 """ + from app.services.market_phase import ( + finalize_ladder_row, + ladder_daily_aggs, + ladder_promo_aggs, + with_prev_consecutive, + ) + needed = ["date", "change_pct", "amount", "signal_limit_up", "signal_limit_down", "signal_broken_limit_up", "consecutive_limit_ups", "close", "ma20"] @@ -158,6 +167,9 @@ def _aggregate_daily(df: pl.DataFrame, index_pct_map: dict | None = None) -> pl. if "date" not in avail or "change_pct" not in avail: return pl.DataFrame() + if "consecutive_limit_ups" in avail and "symbol" in df.columns: + df = with_prev_consecutive(df) + # 基础聚合 — 全部用 group_by 一次性向量化算出, 避免逐日 filter 扫全表(OOM/超时元凶)。 has_ma20 = "close" in avail and "ma20" in avail grouped = df.group_by("date").agg( @@ -216,6 +228,15 @@ def _aggregate_daily(df: pl.DataFrame, index_pct_map: dict | None = None) -> pl. ] if has_ma20 else [] ), + # 梯队指标(阶段判定所需): 首板/N板宽度/非空档位数; 晋级率需 _prev_consec + *( + ladder_daily_aggs() + if "consecutive_limit_ups" in avail else [] + ), + *( + ladder_promo_aggs() + if "consecutive_limit_ups" in avail and "_prev_consec" in df.columns else [] + ), ).sort("date") # 转成 dict 列表做分类(规则引擎需逐日算, 但只扫 grouped 行数=天数, 不再回扫全表) @@ -289,6 +310,8 @@ def _aggregate_daily(df: pl.DataFrame, index_pct_map: dict | None = None) -> pl. "speculation_score": round(sub["speculation"]), "resilience_score": round(sub["resilience"]), "trend_score": round(sub["trend"]), + # 梯队指标(阶段判定所需); phase 由 refresh_phase_labels 统一重标 + **finalize_ladder_row(r), }) return pl.DataFrame(rows) if rows else pl.DataFrame() @@ -325,6 +348,10 @@ def _compute_batch(repo, enriched_dir, instruments, historical_shares, needed={"signal_limit_up", "signal_limit_down", "signal_broken_limit_up"}, historical_shares=historical_shares, ) + # 晋级率需要昨日连板数: 在裁掉 warmup 之前先按 symbol 平移, + # 保证每批首日的 _prev_consec 来自 warmup 的最后一个交易日而非 null。 + from app.services.market_phase import with_prev_consecutive + df = with_prev_consecutive(df) # 丢弃 warmup 行, 只留目标区间 return df.filter((pl.col("date") >= batch_start) & (pl.col("date") <= batch_end)) @@ -446,6 +473,28 @@ def load_regime_history(data_dir: Path) -> pl.DataFrame: return pl.DataFrame() +def refresh_phase_labels(data_dir: Path) -> int: + """对全量 regime 时序重标情绪周期阶段(冰点/启动/主升/高潮/退潮/修复)。 + + 阶段判定需要完整日序(EMA 平滑 + 持续性确认), 不能在单批内完成, + 因此每次 upsert 后调用本函数整体重标并写回。行数为天数(千级), 开销可忽略。 + 返回标注的天数; 阶段列缺失所需指标(旧 schema 未重算)时返回 0。 + """ + from app.services.market_phase import classify_phase_series + + df = load_regime_history(data_dir) + required = {"date", "max_consecutive", "first_board", "ge2_count", "promo_rate", "seal_rate"} + if df.is_empty() or not required.issubset(df.columns): + return 0 + try: + labeled = classify_phase_series(df) + except Exception as e: + logger.warning("refresh_phase_labels failed: %s", e) + return 0 + labeled.write_parquet(regime_path(data_dir)) + return labeled.height + + def upsert_regime_history(data_dir: Path, new_rows: pl.DataFrame) -> None: """按 date 覆盖(upsert): 重算的天覆盖旧行, 新天追加。 @@ -553,6 +602,7 @@ def compute_regime_incremental(repo, data_dir: Path, *, today: date | None = Non new_rows = run_regime_batch(repo, start=to_compute[0], end=to_compute[-1]) if not new_rows.is_empty(): upsert_regime_history(data_dir, new_rows) + refresh_phase_labels(data_dir) return new_rows diff --git a/backend/app/strategy/ai_generator.py b/backend/app/strategy/ai_generator.py index 648cb69..0e3dba9 100644 --- a/backend/app/strategy/ai_generator.py +++ b/backend/app/strategy/ai_generator.py @@ -343,6 +343,7 @@ META = {{...}}，{entrypoint_requirement}。只输出完整 Python 代码。 "polars", "numpy", "app.backtest.matrix", + "app.strategy.builtin.factor_rank_research", "datetime", "__future__", }) diff --git a/backend/app/strategy/builtin/factor_rank_research.py b/backend/app/strategy/builtin/factor_rank_research.py new file mode 100644 index 0000000..bfe1ae3 --- /dev/null +++ b/backend/app/strategy/builtin/factor_rank_research.py @@ -0,0 +1,209 @@ +"""Fixed matrix-native strategy for controlled factor-rank research.""" +from __future__ import annotations + +import numpy as np + +from app.backtest.matrix import ( + MarketDataMatrix, + SignalMatrix, + build_matrix_score, + make_signal_matrix, +) + +META = { + "id": "factor_rank_research", + "name": "因子排名研究", + "description": "受控多因子截面评分、阈值与排名选股策略", + "tags": ["因子", "研究", "截面排名"], + "asset_types": ["stock", "etf"], + "timeframes": ["1d"], + "research_only": True, + "params": [ + { + "id": "entry_score", + "label": "入场最低分", + "type": "float", + "default": 70.0, + "min": 0.0, + "max": 100.0, + "step": 5.0, + }, + { + "id": "exit_score", + "label": "离场最高分", + "type": "float", + "default": 40.0, + "min": 0.0, + "max": 100.0, + "step": 5.0, + }, + { + "id": "top_rank", + "label": "每日最多入选", + "type": "int", + "default": 20, + "min": 1, + "max": 100, + "step": 1, + }, + ], + # Research-generated scoring is supplied in params. Keeping META scoring + # empty prevents the framework pipeline from replacing the strategy score. + "scoring": {}, + "order_by": "score", + "descending": True, + "limit": 100, +} + +EXECUTION_BACKEND = "matrix_native" +ENTRY_SIGNALS = ["signal_factor_rank_entry"] +EXIT_SIGNALS = ["signal_factor_rank_exit"] +STOP_LOSS = -0.08 +MAX_HOLD_DAYS = 30 + +_MAX_FACTORS = 4 +_VALID_DIRECTIONS = {"high", "low"} + + +class FactorRankResearchMatrixStrategy: + def __init__( + self, + scoring: dict[str, float] | None = None, + directions: dict[str, str] | None = None, + ) -> None: + self._scoring = _validated_scoring(scoring) if scoring is not None else None + self._directions = ( + _validated_directions(directions, self._scoring) + if directions is not None and self._scoring is not None + else None + ) + + def required_fields(self) -> frozenset[str]: + return frozenset({ + "open", + "high", + "low", + "close", + "volume", + "amount", + "turnover_rate", + }) + + def required_warmup_bars(self, params: dict) -> int: + del params + return 60 + + def required_fields_for_params(self, params: dict) -> frozenset[str]: + if self._scoring is not None: + return frozenset(self._scoring) + raw = params.get("scoring") + if raw is None: + return frozenset() + return frozenset(_validated_scoring(raw)) + + def compute_signals(self, market: MarketDataMatrix, params: dict) -> SignalMatrix: + scoring = self._scoring or _validated_scoring(params.get("scoring")) + directions = self._directions or _validated_directions( + params.get("directions"), scoring + ) + entry_score = _bounded_float(params.get("entry_score", 70.0), "entry_score") + exit_score = _bounded_float(params.get("exit_score", 40.0), "exit_score") + top_rank = int(params.get("top_rank", 20)) + if not 1 <= top_rank <= 100: + raise ValueError("top_rank must be between 1 and 100") + if exit_score > entry_score: + raise ValueError("exit_score must not exceed entry_score") + + universe = np.isfinite(market.close) + score = build_matrix_score( + market, + universe, + scoring, + "score", + True, + fallback=np.zeros(market.shape, dtype=np.float32), + directions=directions, + ) + entry = universe & (score >= np.float32(entry_score)) + entry = _limit_top_rank(entry, score, top_rank) + exit_ = universe & (score <= np.float32(exit_score)) + return make_signal_matrix( + market.shape, + entry=entry.astype(np.uint8), + exit=exit_.astype(np.uint8), + score=score, + entry_signal_code=np.where(entry, 0, -1).astype(np.int16), + exit_signal_code=np.where(exit_, 0, -1).astype(np.int16), + entry_signal_ids=("signal_factor_rank_entry",), + exit_signal_ids=("signal_factor_rank_exit",), + ) + + +def _validated_scoring(raw: object) -> dict[str, float]: + if not isinstance(raw, dict) or not raw: + raise ValueError("factor-rank research requires a non-empty scoring mapping") + if len(raw) > _MAX_FACTORS: + raise ValueError(f"factor-rank research supports at most {_MAX_FACTORS} factors") + scoring: dict[str, float] = {} + for name, value in raw.items(): + if not isinstance(name, str) or not name: + raise ValueError("scoring factor names must be non-empty strings") + try: + weight = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"scoring weight for {name!r} must be numeric") from exc + if not np.isfinite(weight) or weight <= 0.0: + raise ValueError(f"scoring weight for {name!r} must be finite and positive") + scoring[name] = weight + return scoring + + +def _validated_directions( + raw: object, + scoring: dict[str, float], +) -> dict[str, str]: + if raw is None: + return {} + if not isinstance(raw, dict): + raise ValueError("directions must be a mapping") + unknown = sorted(set(raw) - set(scoring)) + if unknown: + raise ValueError(f"directions contain factors absent from scoring: {unknown}") + directions: dict[str, str] = {} + for name, value in raw.items(): + if value not in _VALID_DIRECTIONS: + raise ValueError( + f"direction for {name!r} must be one of {sorted(_VALID_DIRECTIONS)}" + ) + directions[str(name)] = str(value) + return directions + + +def _bounded_float(value: object, name: str) -> float: + try: + number = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be numeric") from exc + if not np.isfinite(number) or not 0.0 <= number <= 100.0: + raise ValueError(f"{name} must be between 0 and 100") + return number + + +def _limit_top_rank( + eligible: np.ndarray, + score: np.ndarray, + top_rank: int, +) -> np.ndarray: + result = np.zeros(eligible.shape, dtype=bool) + for time_id in range(eligible.shape[0]): + asset_ids = np.flatnonzero(eligible[time_id]) + if asset_ids.size <= top_rank: + result[time_id, asset_ids] = True + continue + # mergesort preserves asset-axis order for equal scores. + order = np.argsort(-score[time_id, asset_ids], kind="stable")[:top_rank] + result[time_id, asset_ids[order]] = True + return result + + +MATRIX_STRATEGY = FactorRankResearchMatrixStrategy() diff --git a/backend/app/strategy/config.py b/backend/app/strategy/config.py index 3c594f7..a22e2c5 100644 --- a/backend/app/strategy/config.py +++ b/backend/app/strategy/config.py @@ -6,12 +6,25 @@ """ from __future__ import annotations +import copy import json import logging from pathlib import Path logger = logging.getLogger(__name__) +# 进程内缓存: 监控引擎每轮对每条策略规则调用 load_override, 每次读盘+parse 纯重复; +# override 仅在用户编辑时变化, 以 (mtime_ns, size) 签名判断是否重读。 +# 键为 override 文件路径 (进程内可能有多个 data_dir, 如测试)。 +_override_cache: dict[str, dict] = {} +_override_cache_sig: dict[str, tuple[int, int]] = {} + + +def _invalidate_override_cache(path: Path) -> None: + key = str(path) + _override_cache.pop(key, None) + _override_cache_sig.pop(key, None) + def _overrides_dir(data_dir: Path) -> Path: d = data_dir / "user_data" / "strategy_overrides" @@ -19,15 +32,29 @@ def _overrides_dir(data_dir: Path) -> Path: return d -def _path(data_dir: Path, strategy_id: str) -> Path: - return _overrides_dir(data_dir) / f"{strategy_id}.json" +def _path(data_dir: Path, strategy_id: str, *, ensure_dir: bool = True) -> Path: + # ensure_dir=False 供热路径读取: mkdir 系统调用在 Windows 上 ~0.07ms, + # 读缓存命中时跳过它 (目录由写路径保证存在)。 + if ensure_dir: + d = _overrides_dir(data_dir) + else: + d = data_dir / "user_data" / "strategy_overrides" + return d / f"{strategy_id}.json" def load_override(data_dir: Path, strategy_id: str) -> dict: - """读取策略的用户覆盖配置，不存在返回空 dict""" - p = _path(data_dir, strategy_id) - if not p.exists(): + """读取策略的用户覆盖配置，不存在返回空 dict (带 mtime 签名缓存, 返回深拷贝)""" + p = _path(data_dir, strategy_id, ensure_dir=False) + key = str(p) + try: + st = p.stat() + sig = (st.st_mtime_ns, st.st_size) + except OSError: + _invalidate_override_cache(p) return {} + cached = _override_cache.get(key) + if cached is not None and sig == _override_cache_sig.get(key): + return copy.deepcopy(cached) try: data = json.loads(p.read_text(encoding="utf-8")) # 清理 basic_filter 中值为 None/空的键（避免固化无意义的空值） @@ -38,7 +65,9 @@ def load_override(data_dir: Path, strategy_id: str) -> dict: data["basic_filter"] = cleaned else: del data["basic_filter"] - return data + _override_cache[key] = data + _override_cache_sig[key] = sig + return copy.deepcopy(data) except Exception as e: logger.warning("load override %s failed: %s", strategy_id, e) return {} @@ -49,11 +78,13 @@ def save_override(data_dir: Path, strategy_id: str, overrides: dict) -> None: p = _path(data_dir, strategy_id) p.parent.mkdir(parents=True, exist_ok=True) p.write_text(json.dumps(overrides, ensure_ascii=False, indent=2), encoding="utf-8") + _invalidate_override_cache(p) def delete_override(data_dir: Path, strategy_id: str) -> None: """删除策略的用户覆盖配置（重置为默认值）""" p = _path(data_dir, strategy_id) + _invalidate_override_cache(p) if p.exists(): p.unlink() diff --git a/backend/app/strategy/engine.py b/backend/app/strategy/engine.py index fca60e7..5ee42d0 100644 --- a/backend/app/strategy/engine.py +++ b/backend/app/strategy/engine.py @@ -554,10 +554,12 @@ class StrategyEngine: # 查询 # ================================================================ - def list_strategies(self) -> list[dict]: - """返回所有策略的元信息""" + def list_strategies(self, *, include_research: bool = False) -> list[dict]: + """Return public strategy metadata unless research templates are requested.""" result = [] for s in self._strategies.values(): + if s.meta.get("research_only") and not include_research: + continue result.append({ **s.meta, "source": s.source, @@ -732,7 +734,11 @@ class StrategyEngine: ), ) field_columns.update( - self._matrix_field_columns(strategy, overrides_map.get(strategy_id)) + self._matrix_field_columns( + strategy, + overrides_map.get(strategy_id), + params, + ) ) if not matrix_ids: return None @@ -1027,7 +1033,11 @@ class StrategyEngine: field_columns: set[str] = set() for sid, strategy in matrix_strats: field_columns.update( - self._matrix_field_columns(strategy, overrides_map.get(sid)) + self._matrix_field_columns( + strategy, + overrides_map.get(sid), + params_map.get(sid), + ) ) shared_matrix = build_market_data_matrix( shared_history, @@ -1052,8 +1062,26 @@ class StrategyEngine: return results @staticmethod - def _matrix_field_columns(strategy: StrategyDef, overrides: dict | None = None) -> set[str]: + def _matrix_field_columns( + strategy: StrategyDef, + overrides: dict | None = None, + params: dict | None = None, + ) -> set[str]: fields = set(strategy.matrix_strategy.required_fields()) + # 参数评分字段 (如挖掘策略的因子组合) 需展开为实际数据依赖, + # 与 backtest._resolve_matrix_native 保持同一语义, 否则虚拟因子 + # (limit_up_count_* -> consecutive_limit_ups) 在矩阵里缺字段。 + parameter_fields = getattr( + strategy.matrix_strategy, + "required_fields_for_params", + None, + ) + if callable(parameter_fields): + fields.update( + scoring_dependencies( + {str(name): 1.0 for name in parameter_fields(params or {})} + ) + ) basic_filter = dict(strategy.basic_filter or {}) if (overrides or {}).get("basic_filter"): basic_filter.update(overrides["basic_filter"]) @@ -1102,7 +1130,7 @@ class StrategyEngine: return StrategyResult(as_of=as_of, strategy_id=strategy_id) market = build_market_data_matrix( source_panel, - field_columns=self._matrix_field_columns(strategy, overrides), + field_columns=self._matrix_field_columns(strategy, overrides, params), ) if source_panel is None or source_panel.is_empty(): diff --git a/backend/app/strategy/scoring.py b/backend/app/strategy/scoring.py index 4456983..dca28ca 100644 --- a/backend/app/strategy/scoring.py +++ b/backend/app/strategy/scoring.py @@ -35,12 +35,31 @@ VIRTUAL_SCORING_DEPENDENCIES: dict[str, frozenset[str]] = { "close_position": frozenset({"high", "low", "close"}), "distance_to_high_60d": frozenset({"close", "high_60d"}), "distance_from_low_60d": frozenset({"close", "low_60d"}), + "max_ret_20d": frozenset({"close"}), + "ret_skew_20d": frozenset({"close"}), + "up_days_20d": frozenset({"close"}), + "amihud_20d": frozenset({"close", "amount"}), + "turnover_z_60d": frozenset({"turnover_rate"}), + "vol_price_corr_20d": frozenset({"close", "volume"}), + "vwap_bias": frozenset({"close", "volume", "amount"}), + "vol_trend_5_60": frozenset({"volume"}), + "limit_up_count_20d": frozenset({"consecutive_limit_ups"}), + "limit_up_count_60d": frozenset({"consecutive_limit_ups"}), } _ROLLING_SCORING_WARMUP: dict[str, int] = { "vol_ratio_10d": 11, "turnover_ratio_5d": 6, "amount_ratio_5d": 6, + "max_ret_20d": 21, + "ret_skew_20d": 21, + "up_days_20d": 21, + "amihud_20d": 21, + "turnover_z_60d": 61, + "vol_price_corr_20d": 21, + "vol_trend_5_60": 60, + "limit_up_count_20d": 21, + "limit_up_count_60d": 61, } @@ -143,9 +162,73 @@ def scoring_value_expr(columns: Collection[str], name: str) -> pl.Expr | None: return _relative(pl.col("close"), pl.col("high_60d")) if name == "distance_from_low_60d": return _relative(pl.col("close"), pl.col("low_60d")) + if name in { + "max_ret_20d", "ret_skew_20d", "up_days_20d", + "amihud_20d", "vol_price_corr_20d", + }: + change = _daily_change_expr() + if name == "max_ret_20d": + return change.rolling_max(20, min_samples=20).over("symbol") + if name == "ret_skew_20d": + return change.rolling_skew(20, bias=True).over("symbol") + if name == "up_days_20d": + return ( + (change > 0).cast(pl.Float64) + .rolling_sum(20, min_samples=20).over("symbol") + ) + if name == "amihud_20d": + illiquidity = _ratio(change.abs(), pl.col("amount") / 1e8) + return illiquidity.rolling_mean(20, min_samples=20).over("symbol") + volume = pl.col("volume") + product = change * volume + return _rolling_corr_expr(change, volume, product, 20).over("symbol") + if name == "turnover_z_60d": + baseline = pl.col("turnover_rate").shift(1) + mean = baseline.rolling_mean(60, min_samples=60) + std = baseline.rolling_std(60, min_samples=60) + return ( + pl.when(std > 0).then((pl.col("turnover_rate") - mean) / std) + .otherwise(None) + .over("symbol") + ) + if name == "vwap_bias": + vwap = _ratio(pl.col("amount"), pl.col("volume") * 100.0) + return _relative(pl.col("close"), vwap) + if name == "vol_trend_5_60": + fast = pl.col("volume").rolling_mean(5) + slow = pl.col("volume").rolling_mean(60) + return _relative(fast, slow).over("symbol") + if name in {"limit_up_count_20d", "limit_up_count_60d"}: + window = 20 if name == "limit_up_count_20d" else 60 + hit = (pl.col("consecutive_limit_ups").fill_null(0) > 0).cast(pl.Float64) + return hit.rolling_sum(window, min_samples=window).over("symbol") return None +def _daily_change_expr() -> pl.Expr: + previous = pl.col("close").shift(1) + return _ratio(pl.col("close"), previous) - 1.0 + + +def _rolling_corr_expr( + left: pl.Expr, right: pl.Expr, product: pl.Expr, window: int +) -> pl.Expr: + """Pearson correlation over a rolling window, matching the matrix kernel formula.""" + mean_left = left.rolling_mean(window, min_samples=window) + mean_right = right.rolling_mean(window, min_samples=window) + mean_product = product.rolling_mean(window, min_samples=window) + mean_left_sq = (left * left).rolling_mean(window, min_samples=window) + mean_right_sq = (right * right).rolling_mean(window, min_samples=window) + covariance = mean_product - mean_left * mean_right + variance_left = mean_left_sq - mean_left * mean_left + variance_right = mean_right_sq - mean_right * mean_right + return pl.when( + (variance_left > 0) & (variance_right > 0) + ).then( + covariance / (variance_left * variance_right).sqrt() + ).otherwise(None) + + def materialize_scoring_columns( frame: pl.DataFrame, names: Collection[str], diff --git a/backend/app/tickflow/repository.py b/backend/app/tickflow/repository.py index 10c874f..7f6705c 100644 --- a/backend/app/tickflow/repository.py +++ b/backend/app/tickflow/repository.py @@ -26,6 +26,13 @@ import duckdb import polars as pl from app.config import settings +from app.enriched_generation import ( + EnrichedGenerationUnavailableError, + EnrichedPublication, + bump_enriched_generation, + get_enriched_generation, +) +from app.market_time import cn_today from app.parquet import scan_enriched_parquet logger = logging.getLogger(__name__) @@ -318,6 +325,7 @@ class KlineRepository: # 完整 enriched 历史 (含所有指标, 供 filter_history 策略使用) self._enriched_history_cache: pl.DataFrame | None = None # ~100万行 self._enriched_history_start: date | None = None + self._enriched_history_generation: str | None = None self._index_instruments_cache: pl.DataFrame | None = None self._etf_enriched_cache: pl.DataFrame | None = None self._etf_enriched_cache_date: date | None = None @@ -485,6 +493,7 @@ class KlineRepository: self._enriched_cache_date = None self._enriched_history_cache = None self._enriched_history_start = None + self._enriched_history_generation = None self._live_agg_cache = None self._live_agg_cache_date = None self._live_agg_check_date = None @@ -497,6 +506,7 @@ class KlineRepository: self._etf_instruments_cache = None self._index_symbol_set_cache = None self._etf_symbol_set_cache = None + self._name_map_cache = None self._index_enriched_cache = None self._index_enriched_cache_date = None @@ -510,6 +520,7 @@ class KlineRepository: """ try: started = time.perf_counter() + refresh_generation = self.get_matrix_data_generation("stock") logger.info("enriched refresh start") step = time.perf_counter() @@ -596,8 +607,13 @@ class KlineRepository: logger.info("enriched refresh step done: join instruments (%.2fs)", time.perf_counter() - step) # 缓存完整历史 (含指标+必要基础信息) 供 filter_history/backtest 直接复用 + if self.get_matrix_data_generation("stock") != refresh_generation: + raise EnrichedGenerationUnavailableError( + "enriched data changed while refreshing its history cache" + ) self._enriched_history_cache = df_full self._enriched_history_start = df_full["date"].min() + self._enriched_history_generation = refresh_generation logger.info("enriched 历史缓存: %d rows, %s ~ %s", len(df_full), self._enriched_history_start, latest) @@ -626,6 +642,8 @@ class KlineRepository: logger.info("enriched 缓存已计算: %d 只, 日期 %s (即时计算)", len(df_today), latest) logger.info("enriched refresh done (%.2fs)", time.perf_counter() - started) return + except EnrichedGenerationUnavailableError: + raise except Exception as e: # noqa: BLE001 logger.warning("enriched 即时计算失败, 使用原始 14 列缓存: %s", e) @@ -908,7 +926,7 @@ class KlineRepository: def _live_agg_baseline_date(self, latest: date) -> date: """盘中递推基准日期。当天实时分区存在时使用上一可用交易日。""" - if latest != date.today(): + if latest != cn_today(): return latest try: row = self.execute_one( @@ -1051,6 +1069,7 @@ class KlineRepository: df = pl.scan_parquet(self._inst_glob).collect() if not df.is_empty(): self._instruments_cache = df + self._name_map_cache = None logger.info("instruments 缓存已加载: %d 只", len(df)) except Exception as e: # noqa: BLE001 logger.warning("instruments 缓存刷新失败: %s", e) @@ -1062,6 +1081,7 @@ class KlineRepository: if not df.is_empty(): self._index_instruments_cache = df self._index_symbol_set_cache = None + self._name_map_cache = None logger.info("index instruments 缓存已加载: %d 只", len(df)) except Exception as e: # noqa: BLE001 logger.debug("index instruments 缓存刷新跳过: %s", e) @@ -1087,6 +1107,7 @@ class KlineRepository: df_all = pl.concat(parts, how="diagonal_relaxed").unique(subset=["symbol"], keep="last").sort("symbol") self._etf_instruments_cache = df_all self._etf_symbol_set_cache = None + self._name_map_cache = None logger.info("ETF instruments 缓存已加载: %d 只", len(df_all)) def get_enriched_latest(self) -> tuple[pl.DataFrame, date | None]: @@ -1164,8 +1185,21 @@ class KlineRepository: ) -> pl.DataFrame | None: """从预计算 enriched 历史缓存返回完整区间；缓存不覆盖时返回 None。""" if self._enriched_history_cache is None: + if self._enriched_warming: + # 后台预热中: 返回 None (缓存不覆盖), 调用方各自走慢路径; + # 否则请求线程会与预热线程并发重复 300 天全量重算 + # (同 get_enriched_latest 的守卫语义)。 + return None self._refresh_enriched() cache = self._enriched_history_cache + data_dir = getattr(getattr(self, "store", None), "data_dir", None) + if data_dir is not None: + try: + current_generation = self.get_matrix_data_generation("stock") + except EnrichedGenerationUnavailableError: + return None + if self._enriched_history_generation != current_generation: + return None if cache is None or cache.is_empty() or "date" not in cache.columns: return None @@ -1204,9 +1238,9 @@ class KlineRepository: # 后台预热中: 返回空表, 不触发同步重算 (同 get_enriched_latest 守卫) return pl.DataFrame() self._refresh_enriched() - self._live_agg_check_date = date.today() # 刚建过, 当天不必再查磁盘 + self._live_agg_check_date = cn_today() # 刚建过, 当天不必再查磁盘 else: - today = date.today() + today = cn_today() if self._live_agg_check_date != today: # today 翻天了 (次日开盘首次轮询): 校验基准日是否需要前移重建。 # 同一天内多次调用直接跳过, 避免每轮都扫 parquet。 @@ -1306,16 +1340,28 @@ class KlineRepository: 自选列表/名称批查等场景的统一名称解析入口, 避免各调用方自行合并两份缓存。 symbols 非 None 时只返回命中的条目。 + 全量结果缓存在 _name_map_cache (随三份 instruments 维表刷新失效), + 避免每请求对 ~7000 行维表做 iter_rows 重建。 """ + if self._name_map_cache is not None: + if symbols is None: + return dict(self._name_map_cache) + wanted = set(symbols) + return {s: n for s, n in self._name_map_cache.items() if s in wanted} + # 只构建并缓存全量映射; symbols 过滤只作用于返回值。 + # 若把过滤后的结果写入缓存, 后续不同 symbols 的查询会命中残缺缓存, + # 导致新加入自选的标的查不到名称。 name_map: dict[str, str] = {} for df in (self.get_instruments(), self.get_etf_instruments(), self.get_instruments_asset("index")): if df.is_empty() or "symbol" not in df.columns or "name" not in df.columns: continue - if symbols is not None: - df = df.filter(pl.col("symbol").is_in(symbols)) for symbol, name in df.select(["symbol", "name"]).iter_rows(): name_map.setdefault(symbol, name) - return name_map + self._name_map_cache = name_map + if symbols is None: + return dict(name_map) + wanted = set(symbols) + return {s: n for s, n in name_map.items() if s in wanted} def enriched_latest_date(self) -> date | None: """返回缓存中的 enriched 最新日期。""" @@ -1355,10 +1401,26 @@ class KlineRepository: # 扩展范围用于指标预热 (MA60 需要 ~60 交易日 ≈ 120 日历日) warmup_start = start - timedelta(days=150) - # 扫描14列 parquet - df = self._scan_daily_symbol(symbol, warmup_start, end, None) - if not df.is_empty(): - df = self._compute_enriched_range(df) + # 优先复用预计算 enriched 历史缓存 (300 天全指标, 与回测引擎同源): + # 个股对话框打开时本接口每个行情 tick 被调一次, 逐请求 150 天扫描 + 全套 + # 指标重算是热路径上最大的重复计算。缓存最新日可能不含当日实时行, + # 由下方 get_enriched_latest 覆盖逻辑补齐; 覆盖不足时回退单股计算路径。 + df = pl.DataFrame() + hist = self._enriched_history_cache + if hist is not None and not hist.is_empty() and "date" in hist.columns: + hist_min = self._enriched_history_start + hist_max = hist["date"].max() + if hist_min is not None and hist_min <= start and hist_max >= start: + df = hist.filter( + (pl.col("symbol") == symbol) + & (pl.col("date") >= start) + & (pl.col("date") <= end) + ) + if df.is_empty(): + # 扫描14列 parquet + df = self._scan_daily_symbol(symbol, warmup_start, end, None) + if not df.is_empty(): + df = self._compute_enriched_range(df) # 尝试用缓存数据覆盖最新日 (盘中更准确) cached, cache_date = self.get_enriched_latest() @@ -1829,30 +1891,11 @@ class KlineRepository: return latest def get_matrix_data_generation(self, asset_type: str = "stock") -> str: - """Return a persistent generation bumped by every managed enriched write.""" - path = self.store.data_dir / f".matrix_generation_{asset_type}.json" - try: - payload = json.loads(path.read_text(encoding="utf-8")) - generation = str(payload.get("generation") or "") - if generation: - return generation - except (OSError, TypeError, ValueError, json.JSONDecodeError): - pass - return self._bump_matrix_data_generation(asset_type) + """Return the stable generation for managed enriched readers.""" + return get_enriched_generation(self.store.data_dir, asset_type) def _bump_matrix_data_generation(self, asset_type: str) -> str: - generation = uuid.uuid4().hex - path = self.store.data_dir / f".matrix_generation_{asset_type}.json" - temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - temporary.write_text( - json.dumps({ - "generation": generation, - "updated_at_ns": time.time_ns(), - }, separators=(",", ":")), - encoding="utf-8", - ) - temporary.replace(path) - return generation + return bump_enriched_generation(self.store.data_dir, asset_type) def symbols_lagging(self, reference_date: date, min_gap_days: int = 3) -> list[str]: """返回日K覆盖落后的标的: 其最新 bar 早于 reference_date - min_gap_days。 @@ -1962,6 +2005,7 @@ class KlineRepository: self._atomic_write_parquet(df.unique(subset=["symbol"], keep="last").sort("symbol"), out) self._index_instruments_cache = None self._etf_instruments_cache = None + self._name_map_cache = None self._refresh_index_instruments() def save_etf_instruments(self, df: pl.DataFrame) -> None: @@ -1974,6 +2018,7 @@ class KlineRepository: out.parent.mkdir(parents=True, exist_ok=True) self._atomic_write_parquet(df.unique(subset=["symbol"], keep="last").sort("symbol"), out) self._etf_instruments_cache = None + self._name_map_cache = None self._refresh_etf_instruments() def refresh_index_views(self) -> None: @@ -2052,25 +2097,36 @@ class KlineRepository: def _write_daily_partition(self, df: pl.DataFrame, table: str) -> None: """按 date 分区写入 parquet，每个日期一个文件，支持 merge-upsert。""" base = self.store.data_dir / table + generation_asset = { + "kline_daily_enriched": "stock", + "kline_etf_enriched": "etf", + }.get(table) + publication = ( + EnrichedPublication(self.store.data_dir, generation_asset) + if generation_asset is not None + else None + ) with self._write_lock: for date_df in df.partition_by("date"): dt = date_df["date"][0] ds = dt.isoformat() if hasattr(dt, "isoformat") else str(dt) out = base / f"date={ds}" / "part.parquet" out.parent.mkdir(parents=True, exist_ok=True) + existing = pl.DataFrame() if out.exists(): existing = pl.read_parquet(out) date_df = pl.concat([existing, date_df], how="diagonal_relaxed").unique( subset=["symbol", "date"], keep="last" ) date_df = date_df.sort(["symbol", "date"]) - self._atomic_write_parquet(date_df, out) - generation_asset = { - "kline_daily_enriched": "stock", - "kline_etf_enriched": "etf", - }.get(table) - if generation_asset is not None: - self._bump_matrix_data_generation(generation_asset) + if not existing.is_empty() and existing.equals(date_df): + continue + if publication is None: + self._atomic_write_parquet(date_df, out) + else: + publication.write_parquet(date_df, out) + if publication is not None: + publication.commit() def merge_live_daily_asset(self, asset_type: str, df: pl.DataFrame) -> None: """按 symbol 合并当天指定资产日K分区。用于少量自选实时，不覆盖全市场。""" @@ -2139,6 +2195,35 @@ class KlineRepository: subset=["symbol", "date"], keep="last" ) merged_cache = merged_cache.sort(["symbol"]) + + from app.indicators.pipeline import ENRICHED_STORAGE_COLS + storage_cols = [c for c in ENRICHED_STORAGE_COLS if c in df.columns] + df_storage = df.select(storage_cols).sort(["symbol"]) + base = self.store.data_dir / table + ds = dt.isoformat() if hasattr(dt, "isoformat") else str(dt) + out = base / f"date={ds}" / "part.parquet" + out.parent.mkdir(parents=True, exist_ok=True) + publication = ( + EnrichedPublication(self.store.data_dir, asset_type) + if asset_type in {"stock", "etf"} + else None + ) + with self._write_lock: + existing = pl.DataFrame() + if out.exists(): + existing = pl.read_parquet(out) + df_storage = pl.concat([existing, df_storage], how="diagonal_relaxed").unique( + subset=["symbol", "date"], keep="last" + ) + df_storage = df_storage.sort(["symbol"]) + if existing.is_empty() or not existing.equals(df_storage): + if publication is None: + self._atomic_write_parquet(df_storage, out) + else: + publication.write_parquet(df_storage, out) + if publication is not None: + publication.commit() + if asset_type == "stock": self._enriched_cache = merged_cache self._enriched_cache_date = dt @@ -2149,23 +2234,6 @@ class KlineRepository: self._index_enriched_cache = merged_cache self._index_enriched_cache_date = dt - from app.indicators.pipeline import ENRICHED_STORAGE_COLS - storage_cols = [c for c in ENRICHED_STORAGE_COLS if c in df.columns] - df_storage = df.select(storage_cols).sort(["symbol"]) - base = self.store.data_dir / table - ds = dt.isoformat() if hasattr(dt, "isoformat") else str(dt) - out = base / f"date={ds}" / "part.parquet" - out.parent.mkdir(parents=True, exist_ok=True) - with self._write_lock: - if out.exists(): - existing = pl.read_parquet(out) - df_storage = pl.concat([existing, df_storage], how="diagonal_relaxed").unique( - subset=["symbol", "date"], keep="last" - ) - self._atomic_write_parquet(df_storage.sort(["symbol"]), out) - if asset_type in {"stock", "etf"}: - self._bump_matrix_data_generation(asset_type) - def flush_live_daily(self, df: pl.DataFrame) -> None: """覆写当天 kline_daily 分区 (实时行情落盘, 非merge)。""" if df.is_empty() or "date" not in df.columns: @@ -2205,16 +2273,10 @@ class KlineRepository: dt = df["date"][0] cache_df = self._with_instrument_metadata(asset_type, df).sort(["symbol"]) if asset_type == "stock": - self._enriched_cache = cache_df - self._enriched_cache_date = dt table = "kline_daily_enriched" elif asset_type == "etf": - self._etf_enriched_cache = cache_df - self._etf_enriched_cache_date = dt table = "kline_etf_enriched" elif asset_type == "index": - self._index_enriched_cache = cache_df - self._index_enriched_cache_date = dt table = "kline_index_enriched" else: return @@ -2226,7 +2288,27 @@ class KlineRepository: ds = dt.isoformat() if hasattr(dt, "isoformat") else str(dt) out = base / f"date={ds}" / "part.parquet" out.parent.mkdir(parents=True, exist_ok=True) + publication = ( + EnrichedPublication(self.store.data_dir, asset_type) + if asset_type in {"stock", "etf"} + else None + ) with self._write_lock: - self._atomic_write_parquet(df_storage, out) - if asset_type in {"stock", "etf"}: - self._bump_matrix_data_generation(asset_type) + existing = pl.read_parquet(out) if out.exists() else pl.DataFrame() + if existing.is_empty() or not existing.equals(df_storage): + if publication is None: + self._atomic_write_parquet(df_storage, out) + else: + publication.write_parquet(df_storage, out) + if publication is not None: + publication.commit() + + if asset_type == "stock": + self._enriched_cache = cache_df + self._enriched_cache_date = dt + elif asset_type == "etf": + self._etf_enriched_cache = cache_df + self._etf_enriched_cache_date = dt + elif asset_type == "index": + self._index_enriched_cache = cache_df + self._index_enriched_cache_date = dt diff --git a/backend/tests/backtest/test_dependencies.py b/backend/tests/backtest/test_dependencies.py index d5d44ee..5983d69 100644 --- a/backend/tests/backtest/test_dependencies.py +++ b/backend/tests/backtest/test_dependencies.py @@ -1,5 +1,8 @@ from __future__ import annotations +import types +from datetime import date, timedelta + import polars as pl from app.backtest.strategy import StrategyDependencyResolver @@ -38,11 +41,79 @@ def test_resolver_merges_signals_scoring_filter_and_execution_columns(): assert {"macd_dif", "macd_dea", "ma20", "momentum_20d", "rsi_14"} <= set(plan.indicator_columns) assert {"signal_macd_golden", "signal_ma20_breakdown", "signal_limit_up", "signal_limit_down"} <= set(plan.signal_columns) assert {"symbol", "date", "open", "high", "low", "close", "volume", "raw_close", "raw_high"} <= set(plan.base_columns) - assert "raw_low" not in plan.base_columns + # 涨跌停信号族统一加载不复权三价 (翘板用 raw_low 判定"曾触及跌停") + assert "raw_low" in plan.base_columns assert "rsi_6" not in plan.indicator_columns assert plan.full_feature_fallback is False +def test_resolver_includes_raw_low_for_limit_signal_family(): + """回归: 翘板信号 (signal_limit_down_recovery) 依赖不复权 raw_low。 + + 旧 bug: 涨跌停基础列只声明 raw_close/raw_high, panel 加载缺 raw_low, + 因子用到翘板信号时 compute_limit_signals 抛 + "unable to find column raw_low" → 回测特征准备失败。 + """ + plan = StrategyDependencyResolver().resolve( + _strategy(), + params={"rsi_max": 30}, + basic_filter={"enabled": False}, + entry_signals=["signal_limit_down_recovery"], + exit_signals=[], + ) + + assert "signal_limit_down_recovery" in plan.signal_columns + assert {"raw_close", "raw_high", "raw_low"} <= set(plan.base_columns) + + +def test_load_panel_for_backtest_supplies_raw_low_for_recovery(monkeypatch, tmp_path): + """回归 (端到端): resolver → load_panel_for_backtest → compute_limit_signals 全链路。 + + 旧 bug: 涨跌停基础列漏 raw_low, 因子用到翘板信号时回测特征准备抛 + "unable to find column raw_low"。 + """ + from app.backtest.engine import BacktestEngine + + n_days = 40 + start = date(2024, 1, 1) + dates = [start + timedelta(days=i) for i in range(n_days)] + px = [10.0 + 0.01 * i for i in range(n_days)] + panel_lf = pl.LazyFrame({ + "symbol": ["600001.SH"] * n_days, + "date": dates, + "open": px, "high": px, "low": px, "close": px, + "volume": [100_000.0] * n_days, + "amount": [1_000_000.0] * n_days, + "raw_close": px, "raw_high": px, "raw_low": px, + }) + monkeypatch.setattr("app.backtest.engine.pl.scan_parquet", lambda path, *a, **k: panel_lf) + + instruments = pl.DataFrame({ + "symbol": ["600001.SH"], "name": ["普通股"], + "limit_up": [11.0], "limit_down": [9.0], + }) + repo = types.SimpleNamespace( + store=types.SimpleNamespace(data_dir=tmp_path), + get_enriched_range=lambda *a, **k: None, + get_instruments_asset=lambda at: instruments, + get_historical_shares=lambda: pl.DataFrame(), + ) + plan = StrategyDependencyResolver().resolve( + _strategy(), + params={"rsi_max": 30}, + basic_filter={"enabled": False}, + entry_signals=["signal_limit_down_recovery"], + exit_signals=[], + ) + + df = BacktestEngine(repo).load_panel_for_backtest( + ["600001.SH"], start, dates[-1], plan, asset_type="stock", + ) + + assert "raw_low" in df.columns + assert "signal_limit_down_recovery" in df.columns + + def test_resolver_expands_virtual_scoring_dependencies(): strategy = _strategy(meta={ "id": "deps", diff --git a/backend/tests/backtest/test_factor_batch.py b/backend/tests/backtest/test_factor_batch.py index debdb4f..f945d78 100644 --- a/backend/tests/backtest/test_factor_batch.py +++ b/backend/tests/backtest/test_factor_batch.py @@ -79,10 +79,10 @@ def test_batch_isolates_a_single_factor_failure(monkeypatch): service = FactorBacktestService(engine) original = service._evaluate_panel - def evaluate(panel, config, run_id, started_at): + def evaluate(panel, config, run_id, started_at, **kwargs): if config.factor_name == "turnover_rate": raise ValueError("broken factor") - return original(panel, config, run_id, started_at) + return original(panel, config, run_id, started_at, **kwargs) monkeypatch.setattr(service, "_evaluate_panel", evaluate) result = service.run_batch(_batch_config(["change_pct", "turnover_rate"])) @@ -135,6 +135,20 @@ def test_factor_catalog_covers_normalized_indicator_families(): "log_amount", "gap_return", "distance_to_high_60d", + "max_ret_20d", + "ret_skew_20d", + "up_days_20d", + "amihud_20d", + "turnover_z_60d", + "vol_price_corr_20d", + "vwap_bias", + "vol_trend_5_60", + "limit_up_count_20d", + "limit_up_count_60d", + "pb_latest", + "roe_latest", + "revenue_yoy_latest", + "debt_ratio_latest", } <= set(factor_ids) assert set(DERIVED_FACTOR_DEPENDENCIES) <= set(factor_ids) diff --git a/backend/tests/backtest/test_factor_metrics.py b/backend/tests/backtest/test_factor_metrics.py new file mode 100644 index 0000000..4607a48 --- /dev/null +++ b/backend/tests/backtest/test_factor_metrics.py @@ -0,0 +1,391 @@ +from __future__ import annotations + +from datetime import date, timedelta +from types import SimpleNamespace + +import polars as pl +import pytest + +from app.backtest.factor import ( + FactorBacktestService, + FactorBatchConfig, + FactorBatchItem, + FactorConfig, + FactorResult, +) +from app.backtest.regime_alignment import ( + align_regime_t_minus_one, + build_regime_filter_mask, +) + + +class _Engine: + def __init__(self, panel: pl.DataFrame, data_dir=None) -> None: + self.panel = panel + self.calls = 0 + self.repo = ( + SimpleNamespace(store=SimpleNamespace(data_dir=data_dir)) + if data_dir is not None + else None + ) + + def load_panel(self, symbols, start, end, columns, asset_type): + self.calls += 1 + selected = [column for column in columns if column in self.panel.columns] + return self.panel.select(selected) + + +def _config( + factor_name: str = "turnover_rate", + **overrides, +) -> FactorConfig: + values = { + "factor_name": factor_name, + "symbols": None, + "start": date(2026, 1, 5), + "end": date(2026, 1, 12), + "n_groups": 2, + "rebalance": "daily", + "fees_pct": 0.0, + "slippage_bps": 0.0, + } + values.update(overrides) + return FactorConfig(**values) + + +def _batch_config(**overrides) -> FactorBatchConfig: + values = { + "factor_names": ["turnover_rate"], + "symbols": None, + "start": date(2026, 1, 5), + "end": date(2026, 1, 12), + "n_groups": 2, + "rebalance": "daily", + "fees_pct": 0.0, + "slippage_bps": 0.0, + } + values.update(overrides) + return FactorBatchConfig(**values) + + +def _daily_panel(days: int = 8, symbols: int = 4) -> pl.DataFrame: + rows = [] + start = date(2026, 1, 5) + for day in range(days): + for index in range(symbols): + rows.append({ + "symbol": f"S{index}", + "date": start + timedelta(days=day), + "open": 10.0 + index, + "high": 10.5 + index, + "low": 9.5 + index, + "close": (10.0 + index) * (1.0 + (index - 1) * day * 0.01), + "volume": 1_000.0, + "amount": 10_000.0, + "turnover_rate": float(index + 1), + }) + return pl.DataFrame(rows) + + +def test_single_and_batch_use_same_full_price_axis_with_internal_factor_null(): + panel = _daily_panel(days=5, symbols=3).with_columns( + pl.when((pl.col("symbol") == "S2") & (pl.col("date") == date(2026, 1, 6))) + .then(None) + .otherwise(pl.col("turnover_rate")) + .alias("turnover_rate") + ) + engine = _Engine(panel) + service = FactorBacktestService(engine) + + single = service.run(_config(end=date(2026, 1, 9))) + batch = service.run_batch(_batch_config(end=date(2026, 1, 9))) + + assert single.error is None + assert batch.results[0].error is None + assert single.ic_mean == batch.results[0].ic_mean + assert single.ir == batch.results[0].ir + assert single.long_short_stats["total_return"] == batch.results[0].long_short_return + + +def test_daily_forward_returns_join_exact_global_trading_dates_on_suspension(): + dates = [date(2026, 1, 5) + timedelta(days=offset) for offset in range(6)] + rows = [ + {"symbol": "B", "date": current, "close": 20.0 + index} + for index, current in enumerate(dates) + ] + rows.extend( + {"symbol": "A", "date": current, "close": 10.0 + index} + for index, current in enumerate(dates) + if index != 3 + ) + panel = pl.DataFrame(rows) + + prepared = FactorBacktestService._attach_shared_next_return( + panel, + _batch_config(end=dates[-1]), + ) + first = prepared.filter( + (pl.col("symbol") == "A") & (pl.col("date") == dates[0]) + ).row(0, named=True) + second = prepared.filter( + (pl.col("symbol") == "A") & (pl.col("date") == dates[1]) + ).row(0, named=True) + + assert first["_forward_return_1d"] == pytest.approx(11.0 / 10.0 - 1.0) + assert first["_forward_return_3d"] is None + assert first["_forward_return_5d"] == pytest.approx(15.0 / 10.0 - 1.0) + assert second["_forward_return_3d"] == pytest.approx(14.0 / 11.0 - 1.0) + + +def test_service_forward_axis_uses_market_partitions_when_selected_universe_has_gap( + tmp_path, +): + dates = [date(2026, 1, 5) + timedelta(days=offset) for offset in range(3)] + panel = pl.DataFrame({ + "symbol": ["A", "A"], + "date": [dates[0], dates[2]], + "open": [10.0, 12.0], + "high": [10.0, 12.0], + "low": [10.0, 12.0], + "close": [10.0, 12.0], + "volume": [1_000.0, 1_000.0], + "amount": [10_000.0, 12_000.0], + "turnover_rate": [1.0, 2.0], + }) + for current in dates: + partition = tmp_path / "kline_daily_enriched" / f"date={current.isoformat()}" + partition.mkdir(parents=True) + pl.DataFrame({"date": [current]}).write_parquet(partition / "part.parquet") + service = FactorBacktestService(_Engine(panel, tmp_path)) + config = _batch_config(start=dates[0], end=dates[-1]) + + loaded = service._load_factor_panel(config, ["turnover_rate"]) + prepared = service._attach_shared_next_return( + loaded, + config, + trading_dates=service._global_trading_dates(config), + ) + + first = prepared.filter(pl.col("date") == dates[0]).row(0, named=True) + assert first["_forward_return_1d"] is None + assert first["_forward_return_3d"] is None + + +def test_tie_aware_groups_do_not_split_constant_factor_by_symbol_order(): + panel = pl.DataFrame({ + "symbol": ["C", "A", "D", "B"], + "date": [date(2026, 1, 5)] * 4, + "factor": [7.0] * 4, + }) + + first = FactorBacktestService._add_groups(panel, "factor", 5).sort("symbol") + second = FactorBacktestService._add_groups( + panel.reverse(), "factor", 5 + ).sort("symbol") + + assert first["_group"].n_unique() == 1 + assert first["_group"].to_list() == second["_group"].to_list() + assert first["_factor_strength"].to_list() == second["_factor_strength"].to_list() + + +def test_weekly_uses_first_actual_trading_day_not_monday(): + panel = pl.DataFrame({ + "symbol": ["A", "A", "A", "A"], + "date": [ + date(2026, 1, 6), + date(2026, 1, 7), + date(2026, 1, 13), + date(2026, 1, 14), + ], + "close": [10.0, 11.0, 12.0, 13.0], + }) + + result = FactorBacktestService._calc_period_return(panel, "weekly") + + assert result.filter(pl.col("date") == date(2026, 1, 6))["_next_return"][0] == pytest.approx(0.2) + assert result.filter(pl.col("date") == date(2026, 1, 7))["_next_return"][0] is None + + +def test_monthly_uses_first_actual_trading_day(): + panel = pl.DataFrame({ + "symbol": ["A", "A", "A", "A"], + "date": [ + date(2026, 1, 6), + date(2026, 1, 7), + date(2026, 2, 3), + date(2026, 2, 4), + ], + "close": [10.0, 11.0, 12.0, 13.0], + }) + + result = FactorBacktestService._calc_period_return(panel, "monthly") + + assert result.filter(pl.col("date") == date(2026, 1, 6))["_next_return"][0] == pytest.approx(0.2) + assert result.filter(pl.col("date") == date(2026, 1, 7))["_next_return"][0] is None + + +def test_factor_weight_and_decomposed_costs_change_group_results(): + panel = _daily_panel(days=2, symbols=4).with_columns( + pl.when(pl.col("date") == date(2026, 1, 6)) + .then( + pl.when(pl.col("symbol") == "S0").then(9.0) + .when(pl.col("symbol") == "S1").then(10.0) + .when(pl.col("symbol") == "S2").then(14.3) + .otherwise(16.9) + ) + .otherwise(pl.col("close")) + .alias("close") + ) + service = FactorBacktestService(_Engine(panel)) + + equal = service.run(_config(end=date(2026, 1, 6), weight="equal")) + weighted = service.run(_config(end=date(2026, 1, 6), weight="factor_weight")) + costly = service.run(_config( + end=date(2026, 1, 6), + weight="factor_weight", + fees_pct=0.009, + commission_pct=0.001, + stamp_tax_pct=0.002, + slippage_bps=10.0, + )) + + equal_q2 = next(item for item in equal.group_stats if item["label"] == "Q2") + weighted_q2 = next(item for item in weighted.group_stats if item["label"] == "Q2") + costly_q2 = next(item for item in costly.group_stats if item["label"] == "Q2") + assert weighted_q2["total_return"] != equal_q2["total_return"] + assert weighted_q2["total_return"] - costly_q2["total_return"] == pytest.approx(0.006) + assert costly.long_short_stats["total_return"] < weighted.long_short_stats["total_return"] + assert costly.config["commission_pct"] == 0.001 + assert costly.config["stamp_tax_pct"] == 0.002 + + +def test_factor_v2_metrics_and_defaults_are_backward_compatible(): + default_result = FactorResult(run_id="r", config={}) + default_item = FactorBatchItem(factor_name="f", label="F", group="G") + + assert default_result.methodology_version == "factor_v2" + assert default_result.yearly_ic == [] + assert default_result.ic_decay == [] + assert default_result.regime_stats == [] + assert default_item.methodology_version == "factor_v2" + assert default_item.yearly_ic == [] + + result = FactorBacktestService(_Engine(_daily_panel())).run(_config()) + assert result.methodology_version == "factor_v2" + assert result.coverage == 1.0 + assert result.turnover is not None + assert result.long_short_sharpe is not None + assert [item["horizon"] for item in result.ic_decay] == [1, 3, 5] + assert result.yearly_ic[0]["year"] == 2026 + assert result.long_short_stats["portfolio_type"] == "theoretical_factor_spread" + assert result.long_short_stats["executable_short"] is False + + +def test_factor_regime_stats_accept_injected_t_minus_one_mapping(tmp_path): + panel = _daily_panel(days=5) + market_dates = [date(2026, 1, 2), *panel["date"].unique().sort().to_list()] + for current in market_dates: + partition = tmp_path / "kline_daily_enriched" / f"date={current.isoformat()}" + partition.mkdir(parents=True) + pl.DataFrame({"date": [current]}).write_parquet(partition / "part.parquet") + regimes = { + date(2026, 1, 2): {"state": "range", "score": 50}, + date(2026, 1, 5): {"state": "weak", "score": 20}, + date(2026, 1, 6): {"state": "strong", "score": 80}, + date(2026, 1, 7): {"state": "strong", "score": 85}, + date(2026, 1, 8): {"state": "range", "score": 50}, + } + + result = FactorBacktestService(_Engine(panel, tmp_path)).run( + _config(end=date(2026, 1, 9)), + regime_by_date=regimes, + ) + + assert result.error is None + assert {item["state"] for item in result.regime_stats} == {"range", "strong", "weak"} + + +def test_factor_regime_stats_reject_missing_actual_market_predecessor(tmp_path): + panel = _daily_panel(days=3) + market_dates = [date(2026, 1, 2), *panel["date"].unique().sort().to_list()] + for current in market_dates: + partition = tmp_path / "kline_daily_enriched" / f"date={current.isoformat()}" + partition.mkdir(parents=True) + pl.DataFrame({"date": [current]}).write_parquet(partition / "part.parquet") + regimes = { + date(2026, 1, 1): {"state": "range", "score": 50}, + date(2026, 1, 5): {"state": "weak", "score": 20}, + date(2026, 1, 6): {"state": "strong", "score": 80}, + } + + with pytest.raises(ValueError, match="2026-01-02"): + FactorBacktestService(_Engine(panel, tmp_path)).run( + _config(end=date(2026, 1, 7)), + regime_by_date=regimes, + ) + + +def test_align_regime_t_minus_one_is_pure_and_fail_closed_in_required_range(): + labels = ("2026-01-05", "2026-01-06", "2026-01-07") + regimes = { + "2026-01-05": ("weak", 20), + "2026-01-06": {"state": "strong", "score": 80}, + } + + aligned = align_regime_t_minus_one( + labels, + regimes, + required_start=date(2026, 1, 6), + required_end=date(2026, 1, 7), + ) + mask = build_regime_filter_mask( + labels, + {"states": ["strong"], "min_score": 60}, + regimes, + required_start=date(2026, 1, 6), + required_end=date(2026, 1, 7), + ) + + assert aligned == [None, ("weak", 20.0), ("strong", 80.0)] + assert mask is not None + assert mask.tolist() == [True, False, True] + + with pytest.raises(ValueError, match="正式首日"): + align_regime_t_minus_one( + labels, + regimes, + required_start=date(2026, 1, 5), + required_end=date(2026, 1, 7), + ) + + lean_regimes = { + "2026-01-05": ("lean_strong", 60), + "2026-01-06": ("range", 50), + } + strong_only_mask = build_regime_filter_mask( + labels, + {"states": ["strong"]}, + lean_regimes, + required_start=date(2026, 1, 6), + required_end=date(2026, 1, 7), + ) + assert strong_only_mask is not None + assert strong_only_mask.tolist() == [True, False, False] + + aggregated_mask = build_regime_filter_mask( + labels, + {"states": ["strong", "lean_strong"]}, + lean_regimes, + required_start=date(2026, 1, 6), + required_end=date(2026, 1, 7), + ) + assert aggregated_mask is not None + assert aggregated_mask.tolist() == [True, True, False] + + with pytest.raises(ValueError, match="缺少前一交易日环境"): + align_regime_t_minus_one( + labels, + {"2026-01-05": ("weak", 20)}, + required_start=date(2026, 1, 6), + required_end=date(2026, 1, 7), + ) diff --git a/backend/tests/backtest/test_factor_rank_research.py b/backend/tests/backtest/test_factor_rank_research.py new file mode 100644 index 0000000..26379b8 --- /dev/null +++ b/backend/tests/backtest/test_factor_rank_research.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +from datetime import date +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import polars as pl +import pytest +from fastapi import HTTPException + +from app.api import screener as screener_api +from app.api import strategy as strategy_api +from app.backtest.matrix import build_market_data_matrix, validate_signal_matrix +from app.backtest.optimizer import expand_param_grid +from app.backtest.strategy import StrategyDependencyResolver +from app.strategy.engine import StrategyEngine + +STRATEGY_PATH = ( + Path(__file__).resolve().parents[2] + / "app" + / "strategy" + / "builtin" + / "factor_rank_research.py" +) + + +def _market(): + panel = pl.DataFrame({ + "symbol": ["000001.SZ", "000002.SZ", "000003.SZ", "000004.SZ"] * 2, + "date": [date(2024, 1, 2)] * 4 + [date(2024, 1, 3)] * 4, + "open": [10.0] * 8, + "high": [10.5] * 8, + "low": [9.5] * 8, + "close": [10.0] * 8, + "volume": [1_000.0] * 8, + "amount": [1.0, 2.0, 3.0, 4.0, 4.0, 3.0, 2.0, 1.0], + "turnover_rate": [4.0, 3.0, 2.0, 1.0, 1.0, 2.0, 3.0, 4.0], + }) + return build_market_data_matrix( + panel, + field_columns={"amount", "turnover_rate"}, + ) + + +def test_strategy_loads_as_builtin_matrix_native_and_grid_params_validate(): + strategy = StrategyEngine._load_file(STRATEGY_PATH) + + assert strategy.meta["id"] == "factor_rank_research" + assert strategy.meta["research_only"] is True + assert strategy.execution_backend == "matrix_native" + assert strategy.matrix_strategy is not None + assert strategy.meta["scoring"] == {} + combos = expand_param_grid( + strategy.meta["params"], + { + "entry_score": [50.0, 75.0], + "exit_score": [20.0], + "top_rank": [1, 2], + }, + ) + assert len(combos) == 4 + assert strategy.matrix_strategy.required_warmup_bars({}) == 60 + assert {"amount", "turnover_rate", "close"}.issubset( + strategy.matrix_strategy.required_fields() + ) + + +def test_research_template_is_hidden_from_ordinary_strategy_apis(tmp_path): + engine = StrategyEngine(strategy_dirs=[STRATEGY_PATH.parent]) + repo = SimpleNamespace(store=SimpleNamespace(data_dir=tmp_path)) + request = SimpleNamespace( + app=SimpleNamespace(state=SimpleNamespace(strategy_engine=engine, repo=repo)) + ) + + screener_payload = screener_api.strategies(request) + strategy_payload = strategy_api.list_strategies(request) + + assert engine.has("factor_rank_research") + assert "factor_rank_research" in { + item["id"] for item in engine.list_strategies(include_research=True) + } + assert "factor_rank_research" not in { + item["id"] for item in screener_payload["presets"] + } + assert "factor_rank_research" not in { + item["id"] for item in strategy_payload["strategies"] + } + + with pytest.raises(HTTPException) as screener_error: + screener_api.run_preset( + screener_api.PresetRequest( + strategy_id="factor_rank_research", + as_of=date(2024, 1, 2), + ), + request, + ) + assert screener_error.value.status_code == 404 + + with pytest.raises(HTTPException) as strategy_error: + strategy_api.run_strategy( + strategy_api.RunRequest( + strategy_id="factor_rank_research", + as_of=date(2024, 1, 2), + ), + request, + ) + assert strategy_error.value.status_code == 404 + + +def test_dependency_resolver_includes_parameter_scoring_fields(): + strategy = StrategyEngine._load_file(STRATEGY_PATH) + + plan = StrategyDependencyResolver().resolve( + strategy, + params={"scoring": {"amount": 1.0, "ma20_bias": 1.0}}, + basic_filter={"enabled": False}, + entry_signals=strategy.entry_signals, + exit_signals=strategy.exit_signals, + ) + + assert {"amount", "close"}.issubset(plan.base_columns) + assert plan.indicator_columns == frozenset() + assert {"amount", "close"}.issubset(plan.matrix_columns) + + +def test_strategy_uses_controlled_scoring_directions_thresholds_and_top_rank(): + strategy = StrategyEngine._load_file(STRATEGY_PATH).matrix_strategy + market = _market() + + signals = strategy.compute_signals( + market, + { + "scoring": {"amount": 1.0, "turnover_rate": 1.0}, + "directions": {"amount": "high", "turnover_rate": "low"}, + "entry_score": 60.0, + "exit_score": 25.0, + "top_rank": 1, + }, + ) + + validate_signal_matrix(signals, market.shape) + assert signals.entry.sum(axis=1).tolist() == [1, 1] + assert signals.entry.tolist() == [[0, 0, 0, 1], [1, 0, 0, 0]] + assert signals.exit.tolist() == [[1, 0, 0, 0], [0, 0, 0, 1]] + assert signals.entry_signal_ids == ("signal_factor_rank_entry",) + assert signals.exit_signal_ids == ("signal_factor_rank_exit",) + assert not signals.score.flags.writeable + + +def test_strategy_direction_changes_score_without_dynamic_formula_execution(): + strategy = StrategyEngine._load_file(STRATEGY_PATH).matrix_strategy + market = _market() + + high = strategy.compute_signals( + market, + { + "scoring": {"amount": 1.0}, + "directions": {"amount": "high"}, + "entry_score": 0.0, + "exit_score": 0.0, + "top_rank": 4, + }, + ) + low = strategy.compute_signals( + market, + { + "scoring": {"amount": 1.0}, + "directions": {"amount": "low"}, + "entry_score": 0.0, + "exit_score": 0.0, + "top_rank": 4, + }, + ) + + np.testing.assert_allclose(high.score + low.score, 100.0) + with pytest.raises(ValueError, match="unsupported matrix feature"): + strategy.compute_signals( + market, + { + "scoring": {"__import__('os').system('bad')": 1.0}, + "entry_score": 50.0, + "exit_score": 20.0, + "top_rank": 1, + }, + ) + + +@pytest.mark.parametrize( + ("params", "message"), + [ + ({"scoring": {}}, "non-empty scoring"), + ( + {"scoring": {f"factor_{index}": 1.0 for index in range(5)}}, + "at most 4 factors", + ), + ( + { + "scoring": {"amount": 1.0}, + "directions": {"turnover_rate": "low"}, + }, + "absent from scoring", + ), + ( + { + "scoring": {"amount": 1.0}, + "entry_score": 20.0, + "exit_score": 30.0, + }, + "exit_score must not exceed", + ), + ], +) +def test_strategy_rejects_uncontrolled_or_invalid_research_params(params, message): + strategy = StrategyEngine._load_file(STRATEGY_PATH).matrix_strategy + + with pytest.raises(ValueError, match=message): + strategy.compute_signals(_market(), params) diff --git a/backend/tests/backtest/test_matrix_strategy.py b/backend/tests/backtest/test_matrix_strategy.py index 91c3519..c1db050 100644 --- a/backend/tests/backtest/test_matrix_strategy.py +++ b/backend/tests/backtest/test_matrix_strategy.py @@ -1,6 +1,7 @@ from __future__ import annotations import gc +import threading from dataclasses import replace from datetime import date, timedelta from pathlib import Path @@ -87,6 +88,7 @@ def test_research_factor_catalog_matches_matrix_features(): for offset in range(120): for asset_id, symbol in enumerate(("000001.SZ", "600000.SH")): close = 10.0 + asset_id * 5.0 + offset * (0.02 + asset_id * 0.01) + np.sin(offset / 5.0) + volume = 1000.0 + asset_id * 250.0 + (offset % 9) * 80.0 rows.append({ "symbol": symbol, "date": start + timedelta(days=offset), @@ -94,14 +96,35 @@ def test_research_factor_catalog_matches_matrix_features(): "high": close + 0.35 + asset_id * 0.03, "low": close - 0.3, "close": close, - "volume": 1000.0 + asset_id * 250.0 + (offset % 9) * 80.0, - "amount": (1000.0 + asset_id * 250.0 + (offset % 9) * 80.0) * close, + "volume": volume, + "amount": volume * close, "turnover_rate": 1.0 + asset_id * 0.2 + (offset % 7) * 0.05, + "consecutive_limit_ups": ( + (offset % 17) + 1 if (asset_id == 0 and offset % 17 == 0) + else (2 if (asset_id == 1 and offset % 23 == 0) else 0) + ), }) panel = pl.DataFrame(rows) factor_names = {item["id"] for item in FACTOR_COLUMNS} expected = FactorBacktestService._compute_missing_factors(panel, factor_names) - market = build_market_data_matrix(panel, field_columns={"amount", "turnover_rate"}) + from app.backtest.fundamentals import ( + FUNDAMENTAL_FACTOR_NAMES, + attach_matrix_fundamental_fields, + ) + + fundamental_names = sorted(factor_names & FUNDAMENTAL_FACTOR_NAMES) + if fundamental_names: + # 夹具无财务数据: 两条路径都应得到全 null/NaN 列, 而不是报错。 + expected = expected.with_columns([ + pl.lit(None, dtype=pl.Float64).alias(name) + for name in fundamental_names + if name not in expected.columns + ]) + market = build_market_data_matrix( + panel, + field_columns={"amount", "turnover_rate", "consecutive_limit_ups"}, + ) + market = attach_matrix_fundamental_fields(market, None, fundamental_names) for name in sorted(factor_names): expected_values = expected.sort(["date", "symbol"])[name].to_numpy().reshape(market.shape) @@ -296,7 +319,7 @@ def test_builtin_matrix_strategies_use_their_declared_formula_modules(): path for path in strategy_dir.glob("*.py") if path.name != "__init__.py" ) - assert len(strategy_files) == 18 + assert len(strategy_files) == 19 for strategy_path in strategy_files: strategy = StrategyEngine._load_file(strategy_path) assert strategy.execution_backend == "matrix_native" @@ -603,6 +626,61 @@ def test_matrix_cache_can_be_disabled(tmp_path): assert not isinstance(market.close, np.memmap) +def test_matrix_cache_cancellation_removes_staging_output(tmp_path, monkeypatch): + market_root = tmp_path / "kline_daily_enriched" + current = date(2024, 3, 2) + partition = market_root / f"date={current.isoformat()}" + partition.mkdir(parents=True) + pl.DataFrame({ + "symbol": ["000001.SZ"], + "date": [current], + "open": [10.0], + "high": [10.0], + "low": [10.0], + "close": [10.0], + "volume": [1_000.0], + }).write_parquet(partition / "part.parquet") + cache_root = tmp_path / "matrix_cache" + cancel_event = threading.Event() + original_scan = matrix_module._scan_matrix_values + + def cancel_after_scan(*args, **kwargs): + original_scan(*args, **kwargs) + cancel_event.set() + + monkeypatch.setattr(matrix_module, "_scan_matrix_values", cancel_after_scan) + + with pytest.raises( + matrix_module.MatrixPrewarmCancelledError, + match="prewarm cancelled", + ): + load_market_data_matrix_from_parquet( + market_root, + current, + current, + field_columns=set(), + cache_root=cache_root, + cancel_event=cancel_event, + ) + + assert list(cache_root.glob("v*-*")) == [] + assert list(cache_root.glob(".*.tmp")) == [] + + +def test_matrix_cache_can_be_cancelled_before_scan(tmp_path): + cancel_event = threading.Event() + cancel_event.set() + + with pytest.raises(matrix_module.MatrixPrewarmCancelledError): + load_market_data_matrix_from_parquet( + tmp_path / "missing", + date(2024, 3, 1), + date(2024, 3, 1), + field_columns=set(), + cancel_event=cancel_event, + ) + + def test_matrix_cache_prunes_by_bytes_and_leaves_no_staging_directory(tmp_path): market_root = tmp_path / "kline_daily_enriched" current = date(2024, 3, 4) @@ -708,7 +786,7 @@ def test_registered_builtin_matrix_strategies_share_one_cache_profile(): profile = build_matrix_cache_profile(engine, "stock") strategies = engine.strategy_definitions() - assert len(strategies) == 18 + assert len(strategies) == 19 assert all(strategy.execution_backend == "matrix_native" for strategy in strategies) assert profile.warmup_bars > 0 assert profile.forward_bars == max(int(strategy.max_hold_days or 0) for strategy in strategies) diff --git a/backend/tests/backtest/test_mining.py b/backend/tests/backtest/test_mining.py new file mode 100644 index 0000000..5c73e23 --- /dev/null +++ b/backend/tests/backtest/test_mining.py @@ -0,0 +1,941 @@ +from __future__ import annotations + +import json +from dataclasses import asdict, replace +from datetime import date, timedelta + +import numpy as np +import polars as pl +import pytest + +import app.backtest.mining as mining_module +from app.backtest.mining import ( + CandidateEvaluation, + CorrelationResult, + FactorMetric, + MiningBudget, + MiningRequest, + MiningService, + NestedValidationConfig, + _searchable_factors, + beam_search_factor_combinations, + compute_rank_correlation, + generate_nested_folds, + nested_fold_count, + prune_correlated_factors, + required_outer_folds, + required_trading_bars, + validation_config_for_profile, +) + + +def _panel(days: int = 12, assets: int = 6) -> pl.DataFrame: + rows = [] + start = date(2024, 1, 2) + for day_id in range(days): + current = start + timedelta(days=day_id * 2) + for asset_id in range(assets): + target = float(asset_id + (day_id % 2) * 0.1) + rows.append({ + "symbol": f"{asset_id:06d}.SZ", + "date": current, + "good": target, + "inverse": -target, + "copy": target * 10.0, + "noise": float((asset_id * 7 + day_id * 3) % assets), + "_next_return": target, + }) + return pl.DataFrame(rows) + + +def _metrics() -> tuple[FactorMetric, ...]: + return ( + FactorMetric("good", 1.0, 1.0, 1.0, 0.2, 1.0), + FactorMetric("copy", 0.9, 0.9, 1.0, 0.1, 1.0), + FactorMetric("inverse", 0.8, 0.8, 1.0, 0.1, -1.0), + FactorMetric("noise", 0.2, 0.1, 0.9, 0.5, 0.0), + ) + + +def test_profiles_enforce_hard_limits_and_are_json_serializable(): + request = MiningRequest.for_profile("exploratory", ["good", "noise"]) + + assert request.budget.beam_width == 8 + assert request.validation.outer_train_bars == 126 + assert request.validation.outer_test_bars == 63 + assert NestedValidationConfig.balanced().outer_train_bars == 504 + assert NestedValidationConfig.strict().outer_train_bars == 756 + assert json.loads(request.to_json())["factor_names"] == ["good", "noise"] + assert asdict(request)["validation"]["purge_bars"] == 30 + with pytest.raises(ValueError, match="max_factors"): + MiningBudget(max_factors=49) + with pytest.raises(ValueError, match="beam_width"): + MiningBudget(beam_width=33) + with pytest.raises(ValueError, match="max_trials"): + MiningBudget(max_trials=257) + with pytest.raises(ValueError, match="existing strategy count"): + MiningRequest( + factor_names=("good",), + existing_strategy_ids=tuple(str(index) for index in range(9)), + ) + + +@pytest.mark.parametrize( + ("profile", "required_bars", "folds"), + [ + ("exploratory", 219, 1), + ("balanced", 786, 3), + ("strict", 1164, 3), + ], +) +def test_profile_trading_bar_requirements(profile, required_bars, folds): + config = validation_config_for_profile(profile) + + assert required_outer_folds(profile) == folds + assert required_trading_bars(config, folds) == required_bars + assert nested_fold_count(required_bars - 1, config) == folds - 1 + assert nested_fold_count(required_bars, config) == folds + + +def test_rank_correlation_is_pairwise_finite_symmetric_and_average_ranked(): + panel = pl.DataFrame({ + "date": [date(2024, 1, 2)] * 5 + [date(2024, 1, 3)] * 5, + "a": [1.0, 1.0, 3.0, np.nan, 5.0, 5.0, 4.0, 3.0, 2.0, 1.0], + "b": [2.0, 2.0, 6.0, 8.0, np.inf, 1.0, 2.0, 3.0, 4.0, 5.0], + "c": [1.0, None, 2.0, 3.0, 4.0, None, None, None, None, None], + "constant": [7.0] * 10, + }) + + result = compute_rank_correlation( + panel, + ["a", "b", "c", "constant"], + date(2024, 1, 2), + date(2024, 1, 3), + ) + + matrix = np.asarray(result.matrix) + counts = np.asarray(result.pair_counts) + np.testing.assert_allclose(matrix, matrix.T) + np.testing.assert_array_equal(np.diag(matrix), np.ones(4)) + assert counts[0, 1] == 2 + assert counts[0, 2] == 1 + assert counts[2, 2] == 1 + assert counts[3, 3] == 2 + assert counts[0, 3] == 0 + assert np.isnan(matrix[0, 3]) + assert result.n_dates == 2 + assert set(result.timing_ms) == {"filter", "rank_accumulate", "finalize", "total"} + + daily_correlations = [] + for daily in panel.partition_by("date"): + valid = daily.filter(pl.col("a").is_finite() & pl.col("b").is_finite()) + daily_correlations.append(np.corrcoef( + valid["a"].rank(method="average").to_numpy(), + valid["b"].rank(method="average").to_numpy(), + )[0, 1]) + assert daily_correlations == pytest.approx([1.0, -1.0]) + assert matrix[0, 1] == pytest.approx(np.mean(daily_correlations)) + + +def test_rank_correlation_reranks_after_pairwise_finite_intersection(): + panel = pl.DataFrame({ + "date": [date(2024, 1, 2)] * 4, + "a": [1.0, 2.0, 3.0, 4.0], + "b": [10.0, None, 20.0, 30.0], + "c": [None, 10.0, 20.0, 30.0], + }) + + result = compute_rank_correlation(panel, ["a", "b", "c"]) + matrix = np.asarray(result.matrix) + + assert matrix[0, 1] == pytest.approx(1.0) + assert matrix[0, 2] == pytest.approx(1.0) + assert matrix[1, 2] == pytest.approx(1.0) + + +def test_pruning_uses_deterministic_metric_order_and_reports_representative(): + correlation = CorrelationResult( + factor_names=("a", "b", "c"), + matrix=((1.0, 0.9, 0.1), (0.9, 1.0, 0.2), (0.1, 0.2, 1.0)), + pair_counts=((10, 10, 10), (10, 10, 10), (10, 10, 10)), + elapsed_ms=1.0, + n_dates=2, + n_rows=10, + ) + metrics = ( + FactorMetric("b", 1.0, 2.0, 0.8, 0.1), + FactorMetric("c", 0.5, 1.0, 1.0, 0.1), + FactorMetric("a", 1.0, 2.0, 0.8, 0.2), + ) + + result = prune_correlated_factors(metrics, correlation, 0.8) + + assert result.selected == ("b", "c") + assert result.excluded[0].factor_id == "a" + assert result.excluded[0].representative == "b" + assert result.excluded[0].rho == pytest.approx(0.9) + + +def test_pruning_ignores_unestimable_factor_pairs(): + correlation = CorrelationResult( + factor_names=("a", "b"), + matrix=((1.0, float("nan")), (float("nan"), 1.0)), + pair_counts=((10, 0), (0, 10)), + elapsed_ms=1.0, + n_dates=2, + n_rows=10, + ) + metrics = ( + FactorMetric("a", 1.0, 1.0, 1.0, 0.1), + FactorMetric("b", 0.9, 0.9, 1.0, 0.1), + ) + + result = prune_correlated_factors(metrics, correlation, 0.8) + + assert result.selected == ("a", "b") + assert result.excluded == () + + +def test_beam_search_learns_direction_from_train_and_honors_real_proxy_budget(): + panel = _panel() + + first = beam_search_factor_combinations( + panel, + ["noise", "inverse", "good"], + max_combination_size=4, + beam_width=32, + max_trials=7, + ) + second = beam_search_factor_combinations( + panel, + ["good", "noise", "inverse"], + max_combination_size=4, + beam_width=32, + max_trials=7, + ) + reversed_rows = beam_search_factor_combinations( + panel.reverse(), + ["good", "noise", "inverse"], + max_combination_size=4, + beam_width=32, + max_trials=7, + ) + + assert first.trials_used == 7 + assert first.budget_exhausted is True + assert first.candidates == second.candidates == reversed_rows.candidates + inverse = next( + candidate + for candidate in first.candidates + if candidate.factor_names == ("inverse",) + ) + assert inverse.directions == (-1,) + assert all(len(candidate.factor_names) <= 4 for candidate in first.candidates) + assert all( + set(candidate.weights) <= {1.0, 2.0} + and sum(weight == 2.0 for weight in candidate.weights) <= 1 + for candidate in first.candidates + ) + + +def test_beam_search_does_not_access_rows_outside_explicit_train_range(): + panel = _panel(days=10) + train_end = sorted(panel["date"].unique().to_list())[5] + changed = panel.with_columns( + pl.when(pl.col("date") > train_end) + .then(-pl.col("_next_return") * 1_000.0) + .otherwise(pl.col("_next_return")) + .alias("_next_return") + ) + + original_result = beam_search_factor_combinations( + panel, + ["good", "inverse", "noise"], + train_end=train_end, + max_trials=30, + ) + changed_result = beam_search_factor_combinations( + changed, + ["good", "inverse", "noise"], + train_end=train_end, + max_trials=30, + ) + + assert original_result.candidates == changed_result.candidates + + +def test_nested_folds_use_trading_labels_with_explicit_purge_and_embargo(): + labels = [f"T{index:02d}" for index in range(22)] + config = NestedValidationConfig( + outer_train_bars=12, + outer_test_bars=4, + outer_step_bars=4, + inner_train_bars=5, + inner_test_bars=2, + inner_step_bars=2, + purge_bars=1, + embargo_bars=2, + min_train_bars=5, + ) + + folds = generate_nested_folds(labels, config) + + assert folds[0].outer.train_labels == tuple(labels[:12]) + assert folds[0].outer.purge_labels == ("T12",) + assert folds[0].outer.test_labels == tuple(labels[13:17]) + assert folds[0].outer.embargo_labels == ("T17", "T18") + assert folds[0].inner[0].train_labels == tuple(labels[:5]) + assert folds[0].inner[0].purge_labels == ("T05",) + assert folds[0].inner[0].test_labels == ("T06", "T07") + assert set(folds[0].inner[-1].test_labels).issubset(folds[0].outer.train_labels) + with pytest.raises(ValueError, match="insufficient trading bars"): + generate_nested_folds(labels[:10], config) + + +def _frame_labels(frame) -> tuple[str, ...]: + return tuple( + frame.select(pl.col("date").cast(pl.Utf8).str.slice(0, 10).unique().sort()) + .to_series() + .to_list() + ) + + +class _Evaluator: + def __init__(self) -> None: + self.calls: list[tuple[tuple[str, ...], tuple[str, ...], dict]] = [] + + def evaluate_candidate(self, train, test, definition): + train_labels = _frame_labels(train) + test_labels = _frame_labels(test) + self.calls.append((train_labels, test_labels, dict(definition))) + return {"score": len(definition.get("factor_names", ())) or 0.1} + + +class _AlternatingWinnerEvaluator(_Evaluator): + """Prefer the good factor only in windows before 2024-01-25; every later window prefers the runner-up. + + With a 20-day panel (dates spaced two calendar days apart) and a + non-overlapping outer step, every inner and outer window of the first + fold falls in January while every window of the second fold falls after + the boundary, so the per-fold winner flips and cross-fold evaluation + has two distinct definitions to score. + """ + + def evaluate_candidate(self, train, test, definition): + self.calls.append((_frame_labels(train), _frame_labels(test), dict(definition))) + prefer_good = _frame_labels(test)[0] < "2024-01-25" + names = definition.get("factor_names") or () + is_good = bool(names) and names[0] == "good" + return {"score": 2.0 if is_good == prefer_good else 1.0} + + +class _LabelEvaluator(_Evaluator): + def __init__(self) -> None: + super().__init__() + self.label_calls: list[tuple[tuple[str, ...], tuple[str, ...], dict]] = [] + + def evaluate_candidate(self, train, test, definition): + raise AssertionError("label evaluator must not receive fold DataFrames") + + def evaluate_candidate_labels(self, train_labels, test_labels, definition): + self.label_calls.append(( + tuple(train_labels), + tuple(test_labels), + dict(definition), + )) + return {"score": len(definition.get("factor_names", ())) or 0.1} + + +@pytest.mark.parametrize("evaluator_type", [_Evaluator, _LabelEvaluator]) +def test_mining_service_reselects_per_inner_fold_and_keeps_tests_out_of_selection( + evaluator_type, +): + panel = _panel(days=10) + validation = NestedValidationConfig( + outer_train_bars=8, + outer_test_bars=2, + outer_step_bars=2, + inner_train_bars=4, + inner_test_bars=2, + inner_step_bars=2, + purge_bars=0, + embargo_bars=0, + min_train_bars=4, + ) + request = MiningRequest( + factor_names=("good", "copy", "inverse", "noise"), + correlation_threshold=0.95, + budget=MiningBudget( + max_combination_size=2, + beam_width=4, + max_proxy_trials=18, + max_trials=3, + ), + validation=validation, + profile="exploratory", + ) + folds = generate_nested_folds( + sorted(str(value) for value in panel["date"].unique().to_list()), + validation, + ) + assert len(folds) == 1 + metric_calls: list[tuple[str, ...]] = [] + + def metric_provider(train, factor_names): + assert tuple(factor_names) == request.factor_names + labels = tuple( + train.select(pl.col("date").cast(pl.Utf8).str.slice(0, 10).unique().sort()) + .to_series() + .to_list() + ) + metric_calls.append(labels) + return _metrics() + + evaluator = evaluator_type() + result = MiningService().run( + panel, + request, + metric_provider=metric_provider, + evaluator=evaluator, + ) + + nested = folds[0] + expected_selection_labels = [ + *(inner.train_labels for inner in nested.inner), + nested.outer.train_labels, + ] + assert metric_calls == expected_selection_labels + assert result.trials_used == request.budget.max_trials + assert result.proxy_trials_used <= request.budget.max_proxy_trials + evaluator_calls = ( + evaluator.label_calls + if isinstance(evaluator, _LabelEvaluator) + else evaluator.calls + ) + assert len(evaluator_calls) == len(nested.inner) + 1 + for call, inner in zip(evaluator_calls[:-1], nested.inner, strict=True): + train_labels, test_labels, _ = call + assert train_labels == inner.train_labels + assert test_labels == inner.test_labels + assert not set(nested.outer.test_labels).intersection(train_labels + test_labels) + outer_train, outer_test, outer_definition = evaluator_calls[-1] + assert outer_train == nested.outer.train_labels + assert outer_test == nested.outer.test_labels + assert result.folds[0].selected_candidate_id is not None + selected = next( + candidate + for candidate in result.folds[0].candidates + if candidate.candidate_id == result.folds[0].selected_candidate_id + ) + assert outer_definition == selected.definition() + + changed = panel.with_columns( + pl.when( + pl.col("date") + .cast(pl.Utf8) + .str.slice(0, 10) + .is_in(nested.outer.test_labels) + ) + .then(-pl.col("good") * 999.0) + .otherwise(pl.col("good")) + .alias("good") + ) + without_evaluation = MiningService().run( + panel, + request, + factor_metrics=_metrics(), + ) + changed_result = MiningService().run( + changed, + request, + factor_metrics=_metrics(), + ) + assert without_evaluation.folds[0].selected_factors == ( + changed_result.folds[0].selected_factors + ) + assert without_evaluation.folds[0].candidates == changed_result.folds[0].candidates + + with pytest.raises(ValueError, match="requires at least 3 outer folds"): + MiningService().run( + panel, + replace(request, profile="balanced"), + factor_metrics=_metrics(), + ) + + +def test_mining_service_requires_fold_local_metrics_with_evaluator(): + panel = _panel(days=10) + request = MiningRequest( + factor_names=("good",), + budget=MiningBudget(max_combination_size=1, max_proxy_trials=12, max_trials=3), + validation=NestedValidationConfig( + outer_train_bars=8, + outer_test_bars=2, + outer_step_bars=2, + inner_train_bars=4, + inner_test_bars=2, + inner_step_bars=2, + purge_bars=0, + embargo_bars=0, + min_train_bars=4, + ), + profile="exploratory", + ) + + with pytest.raises(ValueError, match="metric_provider is required"): + MiningService().run(panel, request, evaluator=_Evaluator()) + with pytest.raises(ValueError, match="factor_metrics must not be supplied"): + MiningService().run( + panel, + request, + factor_metrics=(FactorMetric("good", 1.0, 1.0, 1.0, 0.1),), + metric_provider=lambda _train, _names: ( + FactorMetric("good", 1.0, 1.0, 1.0, 0.1), + ), + evaluator=_Evaluator(), + ) + + +def test_target_endpoint_is_removed_from_every_train_phase_but_not_tests(monkeypatch): + panel = _panel(days=10) + labels = sorted(panel["date"].unique().to_list()) + target_dates = pl.DataFrame({ + "date": labels, + "_target_date": [*labels[1:], labels[-1] + timedelta(days=2)], + }) + panel = panel.join(target_dates, on="date", how="left") + validation = NestedValidationConfig( + outer_train_bars=8, + outer_test_bars=2, + outer_step_bars=2, + inner_train_bars=4, + inner_test_bars=2, + inner_step_bars=2, + purge_bars=0, + embargo_bars=0, + min_train_bars=4, + ) + request = MiningRequest( + factor_names=("good",), + budget=MiningBudget(max_combination_size=1, max_proxy_trials=12, max_trials=3), + validation=validation, + profile="exploratory", + ) + nested = generate_nested_folds([str(label) for label in labels], validation)[0] + train_ends = [*(inner.train_end for inner in nested.inner), nested.outer.train_end] + metric_frames = [] + correlation_frames = [] + beam_frames = [] + + def assert_target_bounded(frame, train_end): + assert frame.filter( + pl.col("_target_date").cast(pl.Utf8).str.slice(0, 10) > train_end + ).is_empty() + + def metric_provider(train, factor_names): + train_end = train_ends[len(metric_frames)] + assert tuple(factor_names) == request.factor_names + assert_target_bounded(train, train_end) + metric_frames.append(train) + return (FactorMetric("good", 1.0, 1.0, 1.0, 0.1),) + + original_correlation = mining_module.compute_rank_correlation + original_beam = mining_module.beam_search_factor_combinations + + def checked_correlation(frame, *args, **kwargs): + train_end = train_ends[len(correlation_frames)] + assert_target_bounded(frame, train_end) + correlation_frames.append(frame) + return original_correlation(frame, *args, **kwargs) + + def checked_beam(frame, *args, **kwargs): + train_end = train_ends[len(beam_frames)] + assert_target_bounded(frame, train_end) + beam_frames.append(frame) + return original_beam(frame, *args, **kwargs) + + class EndpointEvaluator(_Evaluator): + def __init__(self): + super().__init__() + self.test_frames = [] + + def evaluate_candidate(self, train, test, definition): + self.test_frames.append(test) + return super().evaluate_candidate(train, test, definition) + + monkeypatch.setattr(mining_module, "compute_rank_correlation", checked_correlation) + monkeypatch.setattr(mining_module, "beam_search_factor_combinations", checked_beam) + evaluator = EndpointEvaluator() + + result = MiningService().run( + panel, + request, + metric_provider=metric_provider, + evaluator=evaluator, + ) + + assert result.folds[0].error is None + assert len(metric_frames) == len(correlation_frames) == len(beam_frames) == 3 + evaluation_folds = [*nested.inner, nested.outer] + assert len(evaluator.calls) == len(evaluation_folds) + for (train_labels, test_labels, _), test_frame, fold in zip( + evaluator.calls, + evaluator.test_frames, + evaluation_folds, + strict=True, + ): + assert train_labels == fold.train_labels[:-1] + assert test_labels == fold.test_labels + assert test_frame.height == len(fold.test_labels) * 6 + assert not test_frame.filter( + pl.col("_target_date").cast(pl.Utf8).str.slice(0, 10) > fold.test_end + ).is_empty() + + +def test_outer_refit_fails_when_selected_factor_structure_is_missing(): + panel = _panel(days=10) + request = MiningRequest( + factor_names=("good", "copy"), + correlation_threshold=0.8, + budget=MiningBudget(max_combination_size=1, max_proxy_trials=24, max_trials=8), + validation=NestedValidationConfig( + outer_train_bars=8, + outer_test_bars=2, + outer_step_bars=2, + inner_train_bars=4, + inner_test_bars=2, + inner_step_bars=2, + purge_bars=0, + embargo_bars=0, + min_train_bars=4, + ), + profile="exploratory", + ) + + def metric_provider(train, _factor_names): + if train["date"].n_unique() < 8: + return ( + FactorMetric("good", 1.0, 1.0, 1.0, 0.1), + FactorMetric("copy", 0.5, 0.5, 1.0, 0.1), + ) + return ( + FactorMetric("good", 0.5, 0.5, 1.0, 0.1), + FactorMetric("copy", 1.0, 1.0, 1.0, 0.1), + ) + + evaluator = _Evaluator() + result = MiningService().run( + panel, + request, + metric_provider=metric_provider, + evaluator=evaluator, + ) + + fold = result.folds[0] + assert len(evaluator.calls) == 2 + assert fold.selected_candidate_id is None + assert fold.outer_evaluation is None + assert fold.error == "outer retraining did not reproduce selected candidate structure" + assert {candidate.factor_names for candidate in fold.candidates} == {("copy",)} + + +def test_candidate_evaluation_dataclass_score_must_be_finite(): + panel = _panel(days=10) + request = MiningRequest( + factor_names=("good",), + budget=MiningBudget(max_combination_size=1, max_proxy_trials=12, max_trials=8), + validation=NestedValidationConfig( + outer_train_bars=8, + outer_test_bars=2, + outer_step_bars=2, + inner_train_bars=4, + inner_test_bars=2, + inner_step_bars=2, + purge_bars=0, + embargo_bars=0, + min_train_bars=4, + ), + profile="exploratory", + ) + + class NonFiniteEvaluator: + def evaluate_candidate(self, train, test, definition): + return CandidateEvaluation(score=float("nan"), metrics={"source": "test"}) + + result = MiningService().run( + panel, + request, + metric_provider=lambda _train, _names: ( + FactorMetric("good", 1.0, 1.0, 1.0, 0.1), + ), + evaluator=NonFiniteEvaluator(), + ) + + assert result.folds[0].selected_candidate_id is None + assert result.folds[0].error == "no candidate completed inner validation within budget" + + +def test_beam_search_skips_factors_without_sufficient_valid_observations(): + panel = _panel().with_columns( + pl.lit(None).cast(pl.Float64).alias("all_null"), + pl.lit(7.0).alias("constant"), + pl.col("inverse").alias("valid_inverse"), + pl.when(pl.col("symbol").is_in(["000000.SZ", "000001.SZ"])) + .then(pl.col("good")) + .otherwise(None) + .alias("sparse"), + ) + + result = beam_search_factor_combinations( + panel, + ["all_null", "constant", "good", "sparse", "valid_inverse"], + max_combination_size=2, + max_trials=30, + ) + invalid_only = beam_search_factor_combinations( + panel, + ["all_null", "constant", "sparse"], + max_combination_size=2, + max_trials=30, + ) + + assert result.candidates + assert all( + set(candidate.factor_names) <= {"good", "valid_inverse"} + for candidate in result.candidates + ) + assert {candidate.factor_names for candidate in result.candidates} >= { + ("good",), + ("valid_inverse",), + ("good", "valid_inverse"), + } + assert all(candidate.dates > 0 and candidate.observations >= 3 for candidate in result.candidates) + assert invalid_only.candidates == () + + +def test_small_real_allowance_evaluates_factor_and_finalists_stay_capped(): + panel = _panel(days=10) + request = MiningRequest( + factor_names=("good", "noise"), + existing_strategy_ids=tuple(f"existing-{index}" for index in range(8)), + budget=MiningBudget(max_combination_size=1, max_proxy_trials=12, max_trials=1), + validation=NestedValidationConfig( + outer_train_bars=8, + outer_test_bars=2, + outer_step_bars=2, + inner_train_bars=4, + inner_test_bars=2, + inner_step_bars=2, + purge_bars=0, + embargo_bars=0, + min_train_bars=4, + ), + profile="exploratory", + ) + evaluator = _Evaluator() + + result = MiningService().run( + panel, + request, + metric_provider=lambda _train, _names: ( + FactorMetric("good", 1.0, 1.0, 1.0, 0.1), + FactorMetric("noise", 0.5, 0.5, 1.0, 0.1), + ), + evaluator=evaluator, + ) + + assert len(evaluator.calls) == 1 + assert evaluator.calls[0][2]["kind"] == "factor_rank" + finalists = result.folds[0].candidates + assert all(candidate.kind == "factor_rank" for candidate in finalists) + assert len(finalists) == 2 + assert result.folds[0].selected_candidate_id is not None + + +def test_existing_strategies_are_benchmarked_on_every_outer_fold(): + panel = _panel(days=18) + request = MiningRequest( + factor_names=("good",), + existing_strategy_ids=("alpha", "beta"), + budget=MiningBudget(max_combination_size=1, max_proxy_trials=24, max_trials=64), + validation=NestedValidationConfig( + outer_train_bars=8, + outer_test_bars=2, + outer_step_bars=4, + inner_train_bars=4, + inner_test_bars=2, + inner_step_bars=2, + purge_bars=0, + embargo_bars=0, + min_train_bars=4, + ), + profile="exploratory", + ) + evaluator = _Evaluator() + + result = MiningService().run( + panel, + request, + metric_provider=lambda _train, _names: (FactorMetric("good", 1.0, 1.0, 1.0, 0.1),), + evaluator=evaluator, + ) + + assert len(result.folds) == 3 + for fold in result.folds: + assert all(candidate.kind == "factor_rank" for candidate in fold.candidates) + signatures = [candidate_id for candidate_id, _ in fold.benchmark_evaluations] + assert signatures == ["strategy:alpha", "strategy:beta"] + for _, evaluation in fold.benchmark_evaluations: + assert evaluation.error is None + evaluated_strategy_kinds = [ + definition["kind"] + for *_, definition in evaluator.calls + if definition["kind"] == "existing_strategy" + ] + assert len(evaluated_strategy_kinds) == 6 + + +def test_winner_definitions_are_cross_evaluated_on_all_outer_folds(): + panel = _panel(days=20) + request = MiningRequest( + factor_names=("good", "noise"), + budget=MiningBudget(max_combination_size=1, max_proxy_trials=24, max_trials=64), + validation=NestedValidationConfig( + outer_train_bars=8, + outer_test_bars=2, + outer_step_bars=10, + inner_train_bars=4, + inner_test_bars=2, + inner_step_bars=2, + purge_bars=0, + embargo_bars=0, + min_train_bars=4, + ), + profile="exploratory", + ) + evaluator = _AlternatingWinnerEvaluator() + + result = MiningService().run( + panel, + request, + metric_provider=lambda _train, _names: ( + FactorMetric("good", 1.0, 1.0, 1.0, 0.1), + FactorMetric("noise", 0.9, 0.9, 1.0, 0.1), + ), + evaluator=evaluator, + ) + + assert len(result.folds) == 2 + winners = {fold.selected_candidate_id for fold in result.folds} + assert len(winners) == 2 + for fold in result.folds: + cross_ids = [candidate_id for candidate_id, _ in fold.cross_evaluations] + assert cross_ids == sorted(winners - {fold.selected_candidate_id}) + for _, evaluation in fold.cross_evaluations: + assert evaluation.error is None + + +def test_benchmarks_run_even_when_factor_track_fails_on_a_fold(): + panel = _panel(days=18) + request = MiningRequest( + factor_names=("good",), + existing_strategy_ids=("alpha",), + budget=MiningBudget(max_combination_size=1, max_proxy_trials=24, max_trials=64), + validation=NestedValidationConfig( + outer_train_bars=8, + outer_test_bars=2, + outer_step_bars=4, + inner_train_bars=4, + inner_test_bars=2, + inner_step_bars=2, + purge_bars=0, + embargo_bars=0, + min_train_bars=4, + ), + profile="exploratory", + ) + + class _FailingEvaluator(_Evaluator): + def evaluate_candidate(self, train, test, definition): + raise RuntimeError("backtest exploded") + + result = MiningService().run( + panel, + request, + metric_provider=lambda _train, _names: (FactorMetric("good", 1.0, 1.0, 1.0, 0.1),), + evaluator=_FailingEvaluator(), + ) + + assert len(result.folds) == 1 + fold = result.folds[0] + assert fold.error == "no candidate completed inner validation within budget" + assert fold.selected_candidate_id is None + assert [candidate_id for candidate_id, _ in fold.benchmark_evaluations] == [ + "strategy:alpha" + ] + benchmark = fold.benchmark_evaluations[0][1] + assert benchmark.error == "backtest exploded" + + +def test_searchable_factors_are_capped_by_beam_width(): + budget = MiningBudget(beam_width=3) + assert _searchable_factors(("a", "b", "c", "d", "e"), budget) == ("a", "b", "c") + assert _searchable_factors(("a",), MiningBudget(beam_width=8)) == ("a",) + + +def test_beam_search_only_receives_capped_factor_inputs(): + names = tuple(f"factor_{index:02d}" for index in range(20)) + rows = [] + start = date(2024, 1, 2) + for day_id in range(12): + for asset_id in range(6): + row = { + "symbol": f"{asset_id:06d}.SZ", + "date": start + timedelta(days=day_id), + "_next_return": float(asset_id), + } + for factor_id, name in enumerate(names): + row[name] = float((asset_id * (factor_id + 1)) % 7) + factor_id * 0.5 + rows.append(row) + panel = pl.DataFrame(rows) + request = MiningRequest( + factor_names=names, + correlation_threshold=1.0, + budget=MiningBudget( + max_combination_size=2, + beam_width=4, + max_proxy_trials=96, + max_trials=8, + ), + validation=NestedValidationConfig( + outer_train_bars=8, + outer_test_bars=2, + outer_step_bars=4, + inner_train_bars=4, + inner_test_bars=2, + inner_step_bars=2, + purge_bars=0, + embargo_bars=0, + min_train_bars=4, + ), + profile="exploratory", + ) + evaluator = _Evaluator() + + result = MiningService().run( + panel, + request, + metric_provider=lambda _train, factor_names: tuple( + FactorMetric(name, 1.0 - index * 0.01, 1.0, 1.0, 0.1) + for index, name in enumerate(factor_names) + ), + evaluator=evaluator, + ) + + searched = { + factor_name + for candidate in result.folds[0].candidates + for factor_name in candidate.factor_names + } + assert searched <= set(names[:4]) + assert any(len(candidate.factor_names) == 2 for candidate in result.folds[0].candidates) diff --git a/backend/tests/backtest/test_mining_runtime.py b/backend/tests/backtest/test_mining_runtime.py new file mode 100644 index 0000000..b982700 --- /dev/null +++ b/backend/tests/backtest/test_mining_runtime.py @@ -0,0 +1,314 @@ +from __future__ import annotations + +from datetime import date, timedelta +from types import SimpleNamespace + +import polars as pl +import pytest + +from app.backtest.mining import MiningCandidate +from app.backtest.mining_runtime import ( + TrainingMetricProvider, + _decode_runtime_request, + _load_compact_factor_panel, + _prepare_base_market, + _rank_artifact_candidates, + _regime_date_count, + attach_single_forward_return, +) +from app.services import regime_builder + + +def test_runtime_rejects_insufficient_balanced_range_before_loading_panel( + tmp_path, +) -> None: + first = date(2023, 10, 13) + dates = [first + timedelta(days=offset) for offset in range(690)] + for value in dates: + partition = tmp_path / "kline_daily_enriched" / f"date={value.isoformat()}" + partition.mkdir(parents=True) + (partition / "part.parquet").touch() + + payload = { + "run_id": "insufficient-balanced", + "request": { + "factor_names": ["turnover_rate"], + "strategy_ids": [], + "asset_type": "stock", + "budget_profile": "balanced", + "start": dates[0].isoformat(), + "end": dates[-1].isoformat(), + }, + } + + with pytest.raises( + ValueError, + match=( + r"balanced mining requires at least 786 enriched trading bars for " + r"3 outer folds; effective range .* has 690" + ), + ): + _decode_runtime_request(payload, tmp_path, SimpleNamespace()) + + +def test_single_forward_label_uses_global_trading_axis_without_jump() -> None: + first = date(2024, 1, 2) + missing = first + timedelta(days=1) + resumed = first + timedelta(days=2) + panel = pl.DataFrame({ + "symbol": ["000001.SZ", "000001.SZ"], + "date": [first, resumed], + "close": [10.0, 12.0], + "turnover_rate": [1.0, 2.0], + "unused_factor": [9.0, 10.0], + }) + + result = attach_single_forward_return( + panel, + start=first, + end=resumed, + horizon=1, + trading_dates=[first, missing, resumed], + factor_names=["turnover_rate"], + ) + + first_row = result.filter(pl.col("date") == first).row(0, named=True) + assert first_row["_target_date"] == missing + assert first_row["_next_return"] is None + assert "_forward_return_1d" not in result.columns + assert "close" not in result.columns + assert "unused_factor" not in result.columns + assert result.columns == [ + "symbol", + "date", + "turnover_rate", + "_next_return", + "_target_date", + ] + assert result.schema["_next_return"] == pl.Float32 + assert result.schema["turnover_rate"] == pl.Float32 + + fast = attach_single_forward_return( + panel.sort(["date", "symbol"]), + start=first, + end=resumed, + horizon=1, + trading_dates=[first, missing, resumed], + factor_names=["turnover_rate"], + assume_unique_symbol_date=True, + ) + assert fast.equals(result) + + +def test_compact_factor_panel_matches_full_symbol_independent_calculation( + monkeypatch, +) -> None: + first = date(2024, 1, 2) + rows = [] + for symbol, offset in (("a", 0.0), ("b", 2.0), ("c", 4.0)): + for day in range(70): + close = 10.0 + offset + day * 0.1 + rows.append({ + "symbol": symbol, + "date": first + timedelta(days=day), + "open": close - 0.1, + "high": close + 0.2, + "low": close - 0.2, + "close": close, + "volume": 1000.0 + day, + "amount": close * (1000.0 + day), + "turnover_rate": 1.0 + day / 100.0, + }) + raw = pl.DataFrame(rows).sort(["symbol", "date"]) + + class Engine: + def load_panel(self, *_args, **_kwargs): + return raw + + engine = Engine() + from app.backtest.factor import FactorBacktestService + + factor_service = FactorBacktestService(engine) + config = SimpleNamespace( + symbols=None, + start=first, + end=first + timedelta(days=69), + asset_type="stock", + ) + names = ("momentum_20d", "rsi_14", "ma20_bias") + full = factor_service._compute_missing_factors( + raw, + set(names), + assume_sorted=True, + ).select(["symbol", "date", "close", *names]).with_columns([ + pl.col(name).cast(pl.Float32) for name in names + ]).sort(["date", "symbol"]) + + monkeypatch.setattr("app.backtest.mining_runtime._SYMBOL_BATCH_SIZE", 1) + compact = _load_compact_factor_panel( + factor_service, + config, + names, + expected_generation="generation", + cancel_check=None, + ) + + assert compact.equals(full) + + +def test_compact_factor_panel_rejects_noncanonical_symbol_date_keys() -> None: + first = date(2024, 1, 2) + canonical = pl.DataFrame({ + "symbol": ["a", "a", "b"], + "date": [first, first + timedelta(days=1), first], + "open": [1.0, 1.0, 1.0], + "high": [1.0, 1.0, 1.0], + "low": [1.0, 1.0, 1.0], + "close": [1.0, 1.0, 1.0], + "volume": [1.0, 1.0, 1.0], + "amount": [1.0, 1.0, 1.0], + "turnover_rate": [1.0, 1.0, 1.0], + }) + config = SimpleNamespace( + symbols=None, + start=first, + end=first + timedelta(days=1), + asset_type="stock", + ) + + class Engine: + def __init__(self, panel): + self.panel = panel + + def load_panel(self, *_args, **_kwargs): + return self.panel + + from app.backtest.factor import FactorBacktestService + + for invalid in ( + canonical.with_columns(pl.Series( + "date", + [first + timedelta(days=1), first, first], + )), + pl.concat([canonical.slice(0, 1), canonical]), + ): + with pytest.raises(ValueError, match="unique symbol/date"): + _load_compact_factor_panel( + FactorBacktestService(Engine(invalid)), + config, + ("turnover_rate",), + expected_generation="generation", + cancel_check=None, + ) + + +def test_artifact_finalists_are_truncated_by_oos_sharpe_before_signature() -> None: + low = MiningCandidate(candidate_id="a-low", kind="existing_strategy", strategy_id="low") + high = MiningCandidate(candidate_id="z-high", kind="existing_strategy", strategy_id="high") + rows = [ + {"candidate_signature": "a-low", "sharpe": 0.2, "skipped": False}, + {"candidate_signature": "z-high", "sharpe": 1.4, "skipped": False}, + ] + + assert _rank_artifact_candidates([low, high], rows, limit=1) == [high] + + +def test_prepare_base_market_forwards_cancel_event(monkeypatch, tmp_path) -> None: + cancel_event = object() + captured = {} + plan = SimpleNamespace( + base_columns=frozenset(), + intermediate_columns=frozenset(), + indicator_columns=frozenset(), + signal_columns=frozenset(), + matrix_columns=frozenset(), + instrument_columns=frozenset(), + warmup_bars=1, + full_feature_fallback=False, + execution_backend="matrix_native", + fundamental_columns=frozenset(), + ) + research = SimpleNamespace(entry_signals=[], exit_signals=[]) + strategy_engine = SimpleNamespace(get=lambda _strategy_id: research) + service = SimpleNamespace( + _effective_basic_filter=lambda *_args: {}, + engine=SimpleNamespace(), + ) + request = SimpleNamespace( + factor_names=("turnover_rate",), + strategy_ids=(), + asset_type="stock", + forward_horizon=1, + start=date(2024, 1, 2), + end=date(2024, 1, 3), + symbols=None, + ) + + monkeypatch.setattr( + "app.backtest.mining_runtime.StrategyDependencyResolver.resolve", + lambda *_args, **_kwargs: plan, + ) + monkeypatch.setattr( + "app.backtest.mining_runtime.build_matrix_cache_profile", + lambda *_args, **_kwargs: SimpleNamespace(), + ) + + def load_matrix(*_args, **kwargs): + captured.update(kwargs) + return "market" + + service.engine.load_market_data_matrix_for_backtest = load_matrix + + result = _prepare_base_market( + service, + strategy_engine, + tmp_path, + request, + expected_generation="generation", + cancel_check=cancel_event, + ) + + assert result == "market" + assert captured["cancel_event"] is cancel_event + + +def test_training_metric_provider_uses_only_supplied_fold() -> None: + start = date(2024, 1, 2) + rows = [] + for day_offset in range(3): + for asset_id in range(4): + rows.append({ + "symbol": f"{asset_id:06d}.SZ", + "date": start + timedelta(days=day_offset), + "factor": float(asset_id), + "_next_return": ( + float(asset_id) if day_offset < 2 else float(-asset_id) + ), + }) + panel = pl.DataFrame(rows) + train = panel.filter(pl.col("date") < start + timedelta(days=2)) + + provider = TrainingMetricProvider("_next_return") + metric = provider(train, ["factor"])[0] + + assert metric.rank_ic == pytest.approx(1.0) + assert metric.coverage == pytest.approx(1.0) + assert provider.calls[0]["end"] == (start + timedelta(days=1)).isoformat() + assert provider.calls[0]["rows"] == 8 + + +def test_regime_date_count_uses_t_minus_one_market_labels(tmp_path) -> None: + labels = [date(2024, 1, 2) + timedelta(days=offset) for offset in range(4)] + panel = pl.DataFrame({"date": labels}) + regime_builder.upsert_regime_history(tmp_path, pl.DataFrame({ + "date": labels[:3], + "state": ["weak", "strong", "lean_strong"], + "score": [20, 80, 70], + })) + fold = SimpleNamespace( + test_start=labels[1].isoformat(), + test_end=labels[3].isoformat(), + ) + + assert _regime_date_count(panel, fold, "strong", tmp_path) == 2 + assert _regime_date_count(panel, fold, "weak", tmp_path) == 1 diff --git a/backend/tests/backtest/test_research_candidates.py b/backend/tests/backtest/test_research_candidates.py index b391e2c..d9f4d3a 100644 --- a/backend/tests/backtest/test_research_candidates.py +++ b/backend/tests/backtest/test_research_candidates.py @@ -38,6 +38,38 @@ def test_candidate_crud_and_atomic_file(tmp_path): assert store.list() == [] +def test_candidate_mining_provenance_is_idempotent_and_conflict_safe(tmp_path): + store = CandidateStore(tmp_path) + kwargs = { + "origin_run_id": "mining-run-idempotent", + "candidate_signature": "factor-signature", + "kind": "strategy", + "name": "挖掘组合候选", + "source_id": "mined_factor_example", + "config": { + "strategy_id": "mined_factor_example", + "origin_run_id": "mining-run-idempotent", + "candidate_signature": "factor-signature", + "factor_names": ["turnover_rate"], + "directions": ["high"], + "weights": [1.0], + }, + "metrics": {"oos_sharpe": 0.9}, + "data_as_of": "2026-08-11", + } + + first = store.create_or_get_by_provenance(**kwargs) + second = store.create_or_get_by_provenance(**kwargs) + + assert second == first + assert len(store.list()) == 1 + with pytest.raises(CandidateValidationError, match="冲突"): + store.create_or_get_by_provenance( + **{**kwargs, "metrics": {"oos_sharpe": 1.1}} + ) + assert store.list() == [first] + + def test_candidate_rejects_full_result_fields(tmp_path): store = CandidateStore(tmp_path) @@ -52,6 +84,111 @@ def test_candidate_rejects_full_result_fields(tmp_path): ) +def test_candidate_stores_factor_mining_summary(tmp_path): + store = CandidateStore(tmp_path) + config = { + "factor_name": "momentum_20d", + "origin_run_id": "mining-run-001", + "candidate_signature": "factor-signature", + "regime_state": "bull", + "algorithm_version": "mining-v1", + "methodology_version": "factor-v2", + } + metrics = { + "oos_sharpe": 1.24, + "oos_return": 0.18, + "oos_max_drawdown": -0.09, + "oos_positive_fold_ratio": 0.75, + "oos_n_trades": 48, + "valid_folds": 4, + "skipped_folds": 1, + "confidence": 0.9, + "coverage": 0.82, + "turnover": 0.36, + "long_short_sharpe": 1.11, + } + + created = store.create( + kind="factor", + name="挖掘因子候选", + source_id="momentum_20d", + config=config, + metrics=metrics, + data_as_of="2026-08-11", + ) + + assert created["config"] == config + assert created["metrics"] == metrics + assert store.list()[0]["metrics"] == metrics + + +def test_candidate_stores_strategy_mining_factor_combination(tmp_path): + store = CandidateStore(tmp_path) + config = { + "strategy_id": "mined-factor-combination", + "origin_run_id": "mining-run-002", + "candidate_signature": "strategy-signature", + "regime_state": "sideways", + "algorithm_version": "mining-v1", + "methodology_version": "strategy-v1", + "factor_names": ["momentum_20d", "rsi_14"], + "directions": ["high", "low"], + "weights": [0.6, 0.4], + } + metrics = { + "oos_sharpe": 0.98, + "oos_return": 0.12, + "oos_max_drawdown": -0.07, + "oos_positive_fold_ratio": 0.8, + "oos_n_trades": 31, + "valid_folds": 5, + "skipped_folds": 0, + "confidence": 0.86, + } + + created = store.create( + kind="strategy", + name="挖掘组合候选", + source_id="mined-factor-combination", + config=config, + metrics=metrics, + data_as_of="2026-08-11", + ) + + assert created["config"] == config + assert created["metrics"] == metrics + assert store.list()[0]["config"] == config + + +@pytest.mark.parametrize("nested_value", [{"fold_1": 1.2}, [1.2, 0.8]]) +def test_candidate_rejects_nested_mining_metrics(tmp_path, nested_value): + store = CandidateStore(tmp_path) + + with pytest.raises(CandidateValidationError, match="只允许保存标量"): + store.create( + kind="factor", + name="挖掘因子候选", + source_id="momentum_20d", + config={"factor_name": "momentum_20d"}, + metrics={"oos_sharpe": nested_value}, + data_as_of=None, + ) + + +def test_candidate_rejects_unknown_metric_field(tmp_path): + store = CandidateStore(tmp_path) + + with pytest.raises(CandidateValidationError, match="不允许的字段"): + store.create( + kind="factor", + name="挖掘因子候选", + source_id="momentum_20d", + config={"factor_name": "momentum_20d"}, + metrics={"fold_metrics": 1.0}, + data_as_of=None, + ) + + def test_candidate_rejects_non_json_config(tmp_path): store = CandidateStore(tmp_path) @@ -69,13 +206,20 @@ def test_candidate_rejects_non_json_config(tmp_path): def test_candidate_loads_legacy_missing_optional_fields(tmp_path): path = tmp_path / "user_data" / "research_candidates.json" path.parent.mkdir(parents=True) - path.write_text(json.dumps([{ - "id": "legacy", - "kind": "factor", - "name": "旧候选", - "config": {"factor_name": "rsi_14", "equity_curve": [1, 2]}, - "metrics": {"ic_mean": 0.03, "trades": [{"symbol": "000001.SZ"}]}, - }]), encoding="utf-8") + path.write_text( + json.dumps( + [ + { + "id": "legacy", + "kind": "factor", + "name": "旧候选", + "config": {"factor_name": "rsi_14", "equity_curve": [1, 2]}, + "metrics": {"ic_mean": 0.03, "trades": [{"symbol": "000001.SZ"}]}, + } + ] + ), + encoding="utf-8", + ) item = CandidateStore(tmp_path).list()[0] assert item["source_id"] == "rsi_14" diff --git a/backend/tests/backtest/test_strategy_backtest_correctness.py b/backend/tests/backtest/test_strategy_backtest_correctness.py index 1b941f0..4dbde70 100644 --- a/backend/tests/backtest/test_strategy_backtest_correctness.py +++ b/backend/tests/backtest/test_strategy_backtest_correctness.py @@ -181,12 +181,17 @@ def test_non_matrix_strategy_applies_regime_filter_and_reports_config(tmp_path): "signal_limit_up": False, "signal_limit_down": False, } - for offset in range(3) + for offset in range(-1, 3) ]).sort(["symbol", "date"]) regime_builder.upsert_regime_history(tmp_path, pl.DataFrame({ - "date": [start, start + timedelta(days=1)], - "state": ["weak", "strong"], - "score": [10, 85], + "date": [ + start - timedelta(days=1), + start, + start + timedelta(days=1), + start + timedelta(days=2), + ], + "state": ["weak", "weak", "strong", "strong"], + "score": [10, 10, 85, 85], })) engine = _EngineStub(panel, data_dir=tmp_path) service = StrategyBacktestService(engine=engine, strategy_engine=_StrategyEngineStub(_strategy())) @@ -204,16 +209,70 @@ def test_non_matrix_strategy_applies_regime_filter_and_reports_config(tmp_path): assert result.error is None assert engine.sim_matrix is not None - assert engine.sim_matrix.entry[:, 0].tolist() == [1, 0, 1] + assert engine.sim_matrix.entry[:, 0].tolist() == [0, 0, 1] assert result.config["regime_filter"] == regime_filter assert result.stats["selection"] == { - "strategy_matches": 2, - "entry_candidates": 2, + "strategy_matches": 1, + "entry_candidates": 1, "entry_trigger_filtered": 0, "entry_trigger_enabled": False, } +def test_regime_filter_matches_raw_five_level_states(tmp_path): + start = date(2024, 1, 1) + panel = pl.DataFrame([ + { + "symbol": "A", + "name": "A", + "date": start + timedelta(days=offset), + "open": 10.0, + "high": 10.0, + "low": 10.0, + "close": 10.0, + "volume": 1000.0, + "amount": 1000.0, + "signal_limit_up": False, + "signal_limit_down": False, + } + for offset in range(-1, 3) + ]).sort(["symbol", "date"]) + regime_builder.upsert_regime_history(tmp_path, pl.DataFrame({ + "date": [ + start - timedelta(days=1), + start, + start + timedelta(days=1), + start + timedelta(days=2), + ], + "state": ["weak", "lean_strong", "strong", "strong"], + "score": [10, 60, 85, 85], + })) + + def run_with(states: list[str]): + engine = _EngineStub(panel, data_dir=tmp_path) + service = StrategyBacktestService( + engine=engine, + strategy_engine=_StrategyEngineStub(_strategy()), + ) + result = service.run(StrategyBacktestConfig( + strategy_id="test", + symbols=None, + start=start, + end=start + timedelta(days=2), + matching="close_t", + mode="position", + regime_filter={"states": states}, + )) + assert result.error is None + assert engine.sim_matrix is not None + return engine.sim_matrix.entry[:, 0].tolist() + + # 强势与偏强是两个独立档位; 只选强势时偏强日不入场 + assert run_with(["strong"]) == [0, 0, 1] + assert run_with(["strong", "lean_strong"]) == [0, 1, 1] + assert run_with(["lean_strong"]) == [0, 1, 0] + + def test_selection_stats_explain_entry_trigger_filtering(): start = date(2024, 1, 1) panel = pl.DataFrame([ diff --git a/backend/tests/backtest/test_worker_process.py b/backend/tests/backtest/test_worker_process.py index 94c3c84..18cd68f 100644 --- a/backend/tests/backtest/test_worker_process.py +++ b/backend/tests/backtest/test_worker_process.py @@ -1,13 +1,21 @@ from __future__ import annotations +import queue +import threading from datetime import date, timedelta +from types import SimpleNamespace import polars as pl +import pytest +from app.backtest import worker as worker_module +from app.backtest.mining import benchmark_candidate from app.backtest.optimizer import OptimizeConfig from app.backtest.strategy import StrategyBacktestConfig from app.backtest.walkforward import WalkForwardConfig -from app.backtest.worker import make_worker_task, run_worker_task +from app.backtest.worker import BacktestWorkerError, make_worker_task, run_worker_task +from app.enriched_generation import bump_enriched_generation, get_enriched_generation +from app.services.mining_jobs import MiningRunStore def _write_worker_strategy(data_dir) -> None: @@ -88,6 +96,49 @@ def _write_market_data(data_dir, start: date, days: int = 3) -> None: }).write_parquet(instruments_dir / "part.parquet") +def _write_mining_market_data( + data_dir, + start: date, + *, + days: int = 219, + assets: int = 4, +) -> None: + symbols = [f"60000{asset}.SH" for asset in range(assets)] + for offset in range(days): + current = start + timedelta(days=offset) + rows = [] + for asset_id, symbol in enumerate(symbols): + close = 10.0 + asset_id + offset * (0.01 + asset_id * 0.002) + rows.append({ + "symbol": symbol, + "date": current, + "open": close, + "high": close * 1.01, + "low": close * 0.99, + "close": close, + "volume": 1000.0 + asset_id * 100.0, + "amount": close * (100000.0 + asset_id * 1000.0), + "raw_close": close, + "raw_high": close * 1.01, + "raw_low": close * 0.99, + "turnover_rate": 1.0 + asset_id * 0.5 + offset * 0.001, + "consecutive_limit_ups": 0, + "consecutive_limit_downs": 0, + }) + partition = data_dir / "kline_daily_enriched" / f"date={current.isoformat()}" + partition.mkdir(parents=True) + pl.DataFrame(rows).write_parquet(partition / "part.parquet") + + instruments_dir = data_dir / "instruments" + instruments_dir.mkdir(parents=True) + pl.DataFrame({ + "symbol": symbols, + "name": [f"测试{asset}" for asset in range(assets)], + "total_shares": [1_000_000_000.0] * assets, + "float_shares": [1_000_000_000.0] * assets, + }).write_parquet(instruments_dir / "part.parquet") + + def test_spawn_worker_returns_compact_result_and_memory_metrics(tmp_path): start = date(2024, 1, 1) data_dir = tmp_path / "data" @@ -187,6 +238,216 @@ def test_spawn_walkforward_reuses_shared_matrix_across_folds(tmp_path): assert result["worker"]["worker_exitcode"] == 0 +def test_spawn_mining_writes_four_artifacts_and_returns_compact_summary(tmp_path): + start = date(2023, 1, 2) + data_dir = tmp_path / "data" + _write_mining_market_data(data_dir, start) + store = MiningRunStore(data_dir) + manifest = store.create( + { + "factor_names": ["turnover_rate"], + "strategy_ids": [], + "symbols": None, + "asset_type": "stock", + "start": (start - timedelta(days=7)).isoformat(), + "end": (start + timedelta(days=225)).isoformat(), + "budget_profile": "exploratory", + "forward_horizon": 1, + "commission_pct": 0.0, + "stamp_tax_pct": 0.0, + "slippage_bps": 0.0, + "correlation_threshold": 0.75, + "max_combination_factors": 1, + "beam_width": 2, + "max_finalists": 2, + "require_regime": False, + }, + {"generation": get_enriched_generation(data_dir, "stock")}, + run_id="spawn_mining", + ) + payload = { + "run_id": manifest["run_id"], + "request": manifest["request"], + "data_fingerprint": manifest["data_fingerprint"], + "source": "manual", + } + + result = run_worker_task(make_worker_task("mining", data_dir, payload)) + + assert result["status"] in {"succeeded", "succeeded_with_budget_exhausted"} + assert result["factor_count"] == 1 + assert result["data_as_of"] == (start + timedelta(days=218)).isoformat() + assert result["panel_scans"] == 1 + assert result["matrix_bytes"] > 0 + assert result["worker"]["worker_exitcode"] == 0 + assert result["worker"]["serialized_result_bytes"] < 100_000 + registered = store.get("spawn_mining")["artifacts"] # type: ignore[index] + assert set(registered) == {"factors", "correlation", "candidates", "folds"} + for name in registered: + artifact = store.artifact_path("spawn_mining", name) + assert artifact.is_file() + frame = pl.read_parquet(artifact) + assert frame.columns + if name == "folds": + assert "n_dates" in frame.columns + assert frame.filter(pl.col("regime_state") == "overall")["n_dates"].min() > 0 + +def test_spawn_mining_benchmarks_strategy_on_every_outer_fold(tmp_path): + start = date(2023, 1, 2) + data_dir = tmp_path / "data" + _write_mining_market_data(data_dir, start) + store = MiningRunStore(data_dir) + manifest = store.create( + { + "factor_names": ["turnover_rate"], + "strategy_ids": ["low_volatility_leader"], + "symbols": None, + "asset_type": "stock", + "start": (start - timedelta(days=7)).isoformat(), + "end": (start + timedelta(days=225)).isoformat(), + "budget_profile": "exploratory", + "forward_horizon": 1, + "commission_pct": 0.0, + "stamp_tax_pct": 0.0, + "slippage_bps": 0.0, + "correlation_threshold": 0.75, + "max_combination_factors": 1, + "beam_width": 2, + "max_finalists": 2, + "require_regime": False, + }, + {"generation": get_enriched_generation(data_dir, "stock")}, + run_id="spawn_mining_benchmark", + ) + payload = { + "run_id": manifest["run_id"], + "request": manifest["request"], + "data_fingerprint": manifest["data_fingerprint"], + "source": "manual", + } + + result = run_worker_task(make_worker_task("mining", data_dir, payload)) + + assert result["status"] in {"succeeded", "succeeded_with_budget_exhausted"} + folds = pl.read_parquet(store.artifact_path("spawn_mining_benchmark", "folds")) + benchmark_signature = benchmark_candidate("low_volatility_leader").candidate_id + benchmark_rows = folds.filter( + (pl.col("evaluation_kind") == "benchmark") + & (pl.col("candidate_signature") == benchmark_signature) + & (pl.col("regime_state") == "overall") + ) + outer_folds = result["valid_fold_count"] + result["skipped_fold_count"] + assert outer_folds >= 1 + assert benchmark_rows.height == outer_folds + selected_rows = folds.filter( + (pl.col("evaluation_kind") == "selected") + & (pl.col("regime_state") == "overall") + ) + assert selected_rows.height == outer_folds + candidates = pl.read_parquet(store.artifact_path("spawn_mining_benchmark", "candidates")) + assert benchmark_signature in candidates["signature"].to_list() + assert "existing_strategy" in candidates["kind"].to_list() + + +def test_spawn_mining_rejects_generation_change_after_queue(tmp_path): + start = date(2023, 1, 2) + data_dir = tmp_path / "data" + _write_mining_market_data(data_dir, start) + store = MiningRunStore(data_dir) + queued_generation = get_enriched_generation(data_dir, "stock") + manifest = store.create( + { + "factor_names": ["turnover_rate"], + "strategy_ids": [], + "symbols": None, + "asset_type": "stock", + "start": (start - timedelta(days=7)).isoformat(), + "end": (start + timedelta(days=225)).isoformat(), + "budget_profile": "exploratory", + "forward_horizon": 1, + "commission_pct": 0.0, + "stamp_tax_pct": 0.0, + "slippage_bps": 0.0, + "correlation_threshold": 0.75, + "max_combination_factors": 1, + "beam_width": 2, + "max_finalists": 2, + "require_regime": False, + }, + {"generation": queued_generation}, + run_id="stale_generation_mining", + ) + bump_enriched_generation(data_dir, "stock") + payload = { + "run_id": manifest["run_id"], + "request": manifest["request"], + "data_fingerprint": manifest["data_fingerprint"], + "source": "manual", + } + + with pytest.raises( + BacktestWorkerError, + match="changed after the run was queued", + ): + run_worker_task(make_worker_task("mining", data_dir, payload)) + + assert store.get("stale_generation_mining")["artifacts"] == {} # type: ignore[index] + + +def test_worker_terminates_child_after_cancel_grace(monkeypatch, tmp_path): + class FakeQueue: + def get(self, timeout): + raise queue.Empty + + def close(self): + pass + + def join_thread(self): + pass + + class FakeEvent: + def set(self): + pass + + class FakeProcess: + def __init__(self): + self.alive = True + self.exitcode = None + + def start(self): + pass + + def is_alive(self): + return self.alive + + def join(self, timeout=None): + pass + + def terminate(self): + self.alive = False + self.exitcode = -15 + + process = FakeProcess() + context = SimpleNamespace( + Queue=FakeQueue, + Event=FakeEvent, + Process=lambda **_kwargs: process, + ) + clock = iter([0.0, 0.0, 0.0, 6.0]) + monkeypatch.setattr(worker_module.mp, "get_context", lambda _method: context) + monkeypatch.setattr(worker_module.time, "monotonic", lambda: next(clock)) + cancel_event = threading.Event() + cancel_event.set() + + with pytest.raises(BacktestWorkerError, match="after cancellation"): + run_worker_task( + {"kind": "mining", "data_dir": str(tmp_path), "config": {}}, + cancel_event=cancel_event, + ) + + assert process.exitcode == -15 + + def test_spawn_walkforward_skips_folds_before_available_matrix_data(tmp_path): configured_start = date(2024, 1, 1) market_start = configured_start + timedelta(days=4) diff --git a/backend/tests/test_data_clear_generation.py b/backend/tests/test_data_clear_generation.py new file mode 100644 index 0000000..6d0096d --- /dev/null +++ b/backend/tests/test_data_clear_generation.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import json +from datetime import date +from pathlib import Path +from types import SimpleNamespace + +import polars as pl +import pytest + +from app.api import data as data_api +from app.backtest.engine import PanelCache +from app.enriched_generation import ( + EnrichedGenerationUnavailableError, + get_enriched_generation, +) + + +class _RepoStub: + def __init__(self, data_dir: Path) -> None: + self.store = SimpleNamespace(data_dir=data_dir) + self.calls: list[str] = [] + + def clear_cache(self) -> None: + self.calls.append("clear_cache") + + def refresh_cache(self) -> None: + self.calls.append("refresh_cache") + + def rebuild_views(self) -> None: + self.calls.append("rebuild_views") + + +def _request(repo: _RepoStub) -> SimpleNamespace: + return SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(repo=repo))) + + +def _write_parquet_placeholder(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"parquet-placeholder") + + +def _stub_clear_side_effects(monkeypatch: pytest.MonkeyPatch) -> None: + from app.api import overview + from app.services import alert_store + from app.services.pipeline_jobs import job_store + from app.services.screener import ScreenerService + + monkeypatch.setattr(job_store, "clear", lambda: None) + monkeypatch.setattr(alert_store, "clear", lambda _data_dir: None) + monkeypatch.setattr(ScreenerService, "clear_history_cache", lambda: None) + monkeypatch.setattr(overview, "invalidate_overview_cache", lambda: None) + monkeypatch.setattr(data_api, "invalidate_data_cache", lambda _table=None: None) + + +def test_clear_data_bumps_enriched_generations_and_invalidates_panel_cache( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _stub_clear_side_effects(monkeypatch) + repo = _RepoStub(tmp_path) + stock_file = tmp_path / "kline_daily_enriched" / "date=2026-08-14" / "part.parquet" + etf_file = tmp_path / "kline_etf_enriched" / "date=2026-08-14" / "part.parquet" + _write_parquet_placeholder(stock_file) + _write_parquet_placeholder(etf_file) + stock_before = get_enriched_generation(tmp_path, "stock") + etf_before = get_enriched_generation(tmp_path, "etf") + + cache = PanelCache() + cache_args = (["000001.SZ"], date(2026, 8, 14), date(2026, 8, 14), None) + computes: list[int] = [] + + def compute(*_args) -> pl.DataFrame: + computes.append(len(computes) + 1) + return pl.DataFrame({"value": [computes[-1]]}) + + cache.get_or_compute(*cache_args, compute, "stock", stock_before) + result = data_api.clear_data(_request(repo)) + stock_after = get_enriched_generation(tmp_path, "stock") + etf_after = get_enriched_generation(tmp_path, "etf") + cached_after = cache.get_or_compute(*cache_args, compute, "stock", stock_after) + + assert result == {"deleted_files": 2} + assert not stock_file.exists() + assert not etf_file.exists() + assert stock_after != stock_before + assert etf_after != etf_before + assert cached_after["value"].item() == 2 + assert cache.stats()["compute_count"] == 2 + assert repo.calls == ["clear_cache", "refresh_cache", "rebuild_views"] + + +def test_clear_data_restores_ready_generation_when_first_delete_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = _RepoStub(tmp_path) + target = tmp_path / "kline_daily_enriched" / "date=2026-08-14" / "part.parquet" + _write_parquet_placeholder(target) + generation_before = get_enriched_generation(tmp_path, "stock") + original_unlink = Path.unlink + + def fail_target(path: Path, *args, **kwargs) -> None: + if path == target: + raise PermissionError("injected delete failure") + original_unlink(path, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", fail_target) + + with pytest.raises(PermissionError, match="injected delete failure"): + data_api.clear_data(_request(repo)) + + assert target.is_file() + assert get_enriched_generation(tmp_path, "stock") == generation_before + marker = json.loads( + (tmp_path / ".matrix_generation_stock.json").read_text(encoding="utf-8") + ) + assert marker["state"] == "ready" + + +def test_clear_data_keeps_generation_publishing_after_partial_delete( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = _RepoStub(tmp_path) + enriched_dir = tmp_path / "kline_daily_enriched" + first = enriched_dir / "date=2026-08-13" / "part.parquet" + second = enriched_dir / "date=2026-08-14" / "part.parquet" + _write_parquet_placeholder(first) + _write_parquet_placeholder(second) + get_enriched_generation(tmp_path, "stock") + original_unlink = Path.unlink + parquet_unlinks = 0 + + def fail_second_parquet(path: Path, *args, **kwargs) -> None: + nonlocal parquet_unlinks + if path.suffix == ".parquet" and enriched_dir in path.parents: + parquet_unlinks += 1 + if parquet_unlinks == 2: + raise PermissionError("injected partial delete failure") + original_unlink(path, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", fail_second_parquet) + + with pytest.raises(PermissionError, match="injected partial delete failure"): + data_api.clear_data(_request(repo)) + + assert sum(path.exists() for path in (first, second)) == 1 + marker = json.loads( + (tmp_path / ".matrix_generation_stock.json").read_text(encoding="utf-8") + ) + assert marker["state"] == "publishing" + with pytest.raises(EnrichedGenerationUnavailableError, match="being published"): + get_enriched_generation(tmp_path, "stock") diff --git a/backend/tests/test_data_status_storage.py b/backend/tests/test_data_status_storage.py new file mode 100644 index 0000000..70e95b8 --- /dev/null +++ b/backend/tests/test_data_status_storage.py @@ -0,0 +1,61 @@ +"""storage 统计求和不变量测试 — total_size_mb 必须等于各部分之和, 不得重复累加。 + +历史 bug: other_dirs 循环被复制了两遍, 且 financials 既在 other_dirs 里又有专属 +统计块 —— financials 被计入 3 次、pools/backtest_results/screener_results/ai_cache +各计入 2 次, total_size_mb 虚高。此测试用精确整 MB 文件锁定总和不变量。 +""" +from __future__ import annotations + +from pathlib import Path + +from app.api.data import _compute_storage + +MB = 1024 * 1024 + + +def _write_mb(path: Path, mb: float) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"\0" * int(mb * MB)) + + +def _build_tree(data_dir: Path) -> None: + _write_mb(data_dir / "kline_daily" / "part.parquet", 1.0) + _write_mb(data_dir / "financials" / "metrics" / "part.parquet", 2.0) + _write_mb(data_dir / "pools" / "p.json", 1.0) + _write_mb(data_dir / "backtest_results" / "r.json", 1.0) + _write_mb(data_dir / "screener_results" / "s.json", 1.0) + _write_mb(data_dir / "ai_cache" / "c.json", 1.0) + _write_mb(data_dir / "capabilities.json", 0.5) # 根目录散文件 + + +def test_total_equals_sum_of_parts(tmp_path): + _build_tree(tmp_path) + stats = _compute_storage(tmp_path) + + parts_mb = ( + stats["daily_size_mb"] # 1.0 (subdirs 表) + + stats["financials_size_mb"] # 2.0 (专属统计块) + + 1.0 + 1.0 + 1.0 + 1.0 # pools/backtest/screener/ai_cache (单次循环) + + 0.5 # 根目录散文件 + ) + assert stats["total_size_mb"] == parts_mb == 7.5 + + +def test_financials_counted_once_not_three_times(tmp_path): + _build_tree(tmp_path) + stats = _compute_storage(tmp_path) + + assert stats["financials_files"] == 1 + assert stats["financials_size_mb"] == 2.0 + # 旧 bug: financials 计 3 次 (6MB) + 其余 4 目录各计 2 次 (8MB) → 15.5 + assert stats["total_size_mb"] != 15.5 + + +def test_missing_dirs_contribute_zero(tmp_path): + _write_mb(tmp_path / "kline_daily" / "part.parquet", 1.0) + stats = _compute_storage(tmp_path) + + # financials 目录不存在时不含其明细键 (既有行为), 且不贡献任何体积 + assert stats.get("financials_files", 0) == 0 + assert stats.get("financials_size_mb", 0.0) == 0.0 + assert stats["total_size_mb"] == 1.0 diff --git a/backend/tests/test_enriched_generation.py b/backend/tests/test_enriched_generation.py new file mode 100644 index 0000000..a48ccfb --- /dev/null +++ b/backend/tests/test_enriched_generation.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import json +from datetime import date +from types import SimpleNamespace + +import polars as pl +import pytest + +from app.backtest.engine import BacktestEngine, PanelCache +from app.enriched_generation import ( + EnrichedGenerationUnavailableError, + EnrichedPublication, + get_enriched_generation, +) +from app.tickflow.repository import DataStore, KlineRepository + + +def _frame(value: float = 10.0) -> pl.DataFrame: + return pl.DataFrame({ + "symbol": ["000001.SZ"], + "date": [date(2026, 8, 14)], + "open": [value], + "high": [value], + "low": [value], + "close": [value], + "volume": [1_000.0], + }) + + +def test_repository_enriched_noop_does_not_bump_generation(tmp_path) -> None: + repo = KlineRepository(DataStore(tmp_path)) + frame = _frame() + + repo.append_enriched(frame) + first = repo.get_matrix_data_generation("stock") + repo.append_enriched(frame) + + assert repo.get_matrix_data_generation("stock") == first + + +def test_failed_multi_partition_publication_remains_fail_closed( + tmp_path, + monkeypatch, +) -> None: + publication = EnrichedPublication(tmp_path, recover=True) + first = tmp_path / "kline_daily_enriched" / "date=2026-08-13" / "part.parquet" + second = tmp_path / "kline_daily_enriched" / "date=2026-08-14" / "part.parquet" + publication.write_parquet(_frame(10.0), first) + + original_write = pl.DataFrame.write_parquet + + def fail_second(self, path, *args, **kwargs): + if "2026-08-14" in str(path): + raise OSError("injected write failure") + return original_write(self, path, *args, **kwargs) + + monkeypatch.setattr(pl.DataFrame, "write_parquet", fail_second) + with pytest.raises(OSError, match="injected"): + publication.write_parquet(_frame(11.0), second) + + marker = json.loads( + (tmp_path / ".matrix_generation_stock.json").read_text(encoding="utf-8") + ) + assert marker["state"] == "publishing" + assert first.is_file() + assert not second.is_file() + with pytest.raises(EnrichedGenerationUnavailableError, match="being published"): + get_enriched_generation(tmp_path, "stock") + + +def test_recovery_replaces_stale_publication_but_not_active_owner(tmp_path) -> None: + first = EnrichedPublication(tmp_path, recover=True) + out = tmp_path / "kline_daily_enriched" / "date=2026-08-14" / "part.parquet" + first.write_parquet(_frame(10.0), out) + + with pytest.raises(EnrichedGenerationUnavailableError, match="active"): + EnrichedPublication(tmp_path, recover=True).write_parquet(_frame(11.0), out) + + del first + recovered = EnrichedPublication(tmp_path, recover=True) + recovered.write_parquet(_frame(12.0), out) + recovered_generation = recovered.commit() + + assert recovered_generation == get_enriched_generation(tmp_path, "stock") + assert pl.read_parquet(out)["close"].item() == pytest.approx(12.0) + + +def test_panel_cache_generation_change_forces_recompute() -> None: + cache = PanelCache() + calls: list[int] = [] + args = (["000001.SZ"], date(2026, 8, 13), date(2026, 8, 14), None) + + def compute(*_args): + calls.append(len(calls) + 1) + return pl.DataFrame({"value": [calls[-1]]}) + + first = cache.get_or_compute(*args, compute, "stock", "generation-a") + second = cache.get_or_compute(*args, compute, "stock", "generation-b") + + assert first["value"].item() == 1 + assert second["value"].item() == 2 + assert cache.stats()["compute_count"] == 2 + + +def test_panel_reader_retries_when_generation_changes_during_scan(tmp_path) -> None: + generations = iter(["generation-a", "generation-b", "generation-b", "generation-b"]) + repo = SimpleNamespace( + store=SimpleNamespace(data_dir=tmp_path), + get_matrix_data_generation=lambda _asset_type: next(generations), + ) + engine = BacktestEngine(repo) + calls: list[str] = [] + + def load(*_args): + calls.append("scan") + return _frame(float(len(calls))) + + engine._load_panel_inner = load + panel = engine.load_panel( + None, + date(2026, 8, 14), + date(2026, 8, 14), + columns=["symbol", "date", "close"], + ) + + assert calls == ["scan", "scan"] + assert panel["close"].item() == pytest.approx(2.0) + + +def test_matrix_reader_retries_when_generation_changes_during_build( + tmp_path, + monkeypatch, +) -> None: + generations = iter(["generation-a", "generation-b", "generation-b", "generation-b"]) + repo = SimpleNamespace( + store=SimpleNamespace(data_dir=tmp_path), + get_matrix_data_generation=lambda _asset_type: next(generations), + get_instruments_asset=lambda _asset_type: pl.DataFrame(), + ) + engine = BacktestEngine(repo) + calls: list[str | None] = [] + expected = SimpleNamespace( + execution_backend="matrix_native", + base_columns={"open", "high", "low", "close", "volume"}, + instrument_columns=set(), + matrix_columns=set(), + ) + market = SimpleNamespace() + + def load_matrix(*_args, source_generation=None, **_kwargs): + calls.append(source_generation) + return market + + monkeypatch.setattr("app.backtest.engine.load_market_data_matrix_from_parquet", load_matrix) + + result = engine.load_market_data_matrix_for_backtest( + None, + date(2026, 8, 14), + date(2026, 8, 14), + expected, + ) + + assert result is market + assert calls == ["generation-a", "generation-b"] diff --git a/backend/tests/test_enriched_range_warming_guard.py b/backend/tests/test_enriched_range_warming_guard.py new file mode 100644 index 0000000..ece7f2b --- /dev/null +++ b/backend/tests/test_enriched_range_warming_guard.py @@ -0,0 +1,56 @@ +"""get_enriched_range 的预热守卫测试 — 预热期间不得触发同步全量重算。 + +启动后台预热线程正在 _refresh_enriched (300 天 scan + compute, 低配机 50s+) +时, 请求线程进入 get_enriched_range 应返回 None (缓存不覆盖语义), +不能在请求线程里并发跑第二次全量重算。与 get_enriched_latest 守卫对齐。 +""" +from __future__ import annotations + +from datetime import date + +import polars as pl + +from app.tickflow.repository import KlineRepository + + +def _bare_repo() -> KlineRepository: + """跳过 __init__ (避免 DataStore/目录依赖), 只装配守卫涉及的属性。""" + repo = KlineRepository.__new__(KlineRepository) + repo._enriched_history_cache = None + repo._enriched_warming = True + return repo + + +def test_get_enriched_range_returns_none_while_warming(): + repo = _bare_repo() + refresh_calls: list[int] = [] + + def _spy_refresh(): + refresh_calls.append(1) + + repo._refresh_enriched = _spy_refresh # type: ignore[method-assign] + + result = repo.get_enriched_range(date(2026, 1, 1), date(2026, 8, 14)) + + assert result is None + assert refresh_calls == [], "预热期间不得触发 _refresh_enriched" + + +def test_get_enriched_range_rebuilds_when_cold_and_not_warming(): + repo = _bare_repo() + repo._enriched_warming = False + built = pl.DataFrame({ + "symbol": ["600000.SH", "600000.SH"], + "date": [date(2026, 1, 1), date(2026, 8, 14)], + }) + + def _fake_refresh(): + repo._enriched_history_cache = built + + repo._refresh_enriched = _fake_refresh # type: ignore[method-assign] + + result = repo.get_enriched_range(date(2026, 1, 1), date(2026, 8, 14)) + + assert result is not None + assert result.height == 2 + assert result["symbol"].unique().to_list() == ["600000.SH"] diff --git a/backend/tests/test_ext_config_load_all_cache.py b/backend/tests/test_ext_config_load_all_cache.py new file mode 100644 index 0000000..7cbec33 --- /dev/null +++ b/backend/tests/test_ext_config_load_all_cache.py @@ -0,0 +1,99 @@ +"""ExtConfigStore.load_all 签名缓存测试 — 配置未变不重复读盘, 变更后立即可见, 返回副本。""" +from __future__ import annotations + +import json + +import pytest + +from app.services import ext_data +from app.services.ext_data import ExtConfig, ExtConfigStore, ExtField + + +@pytest.fixture(autouse=True) +def _clean_cache(): + ext_data._load_all_cache.clear() + yield + ext_data._load_all_cache.clear() + + +def _patched_loads(monkeypatch, counter: dict): + real_loads = json.loads + + def _counting(text): + counter["loads"] += 1 + return real_loads(text) + + monkeypatch.setattr(ext_data.json, "loads", _counting) + + +def _config(cid: str, label: str) -> ExtConfig: + return ExtConfig(id=cid, label=label, mode="snapshot", + fields=[ExtField(name="score", dtype="float")]) + + +def test_second_load_all_hits_cache_without_disk_parse(tmp_path, monkeypatch): + store = ExtConfigStore(tmp_path) + store.upsert(_config("cfg_a", "A")) + store.upsert(_config("cfg_b", "B")) + + counter = {"loads": 0} + _patched_loads(monkeypatch, counter) + first = store.load_all() + assert counter["loads"] == 2, "缓存为空时逐 config.json parse" + again = store.load_all() + assert counter["loads"] == 2, "签名未变时不得重复读盘+parse" + assert [c.id for c in first] == [c.id for c in again] == ["cfg_a", "cfg_b"] + + +def test_upsert_edit_invalidates_cache(tmp_path): + store = ExtConfigStore(tmp_path) + store.upsert(_config("cfg_a", "old")) + assert store.load_all()[0].label == "old" + + store.upsert(_config("cfg_a", "new")) + assert store.load_all()[0].label == "new" + + +def test_delete_invalidates_cache(tmp_path): + store = ExtConfigStore(tmp_path) + store.upsert(_config("cfg_a", "A")) + store.upsert(_config("cfg_b", "B")) + assert {c.id for c in store.load_all()} == {"cfg_a", "cfg_b"} + + assert store.delete("cfg_a") is True + assert {c.id for c in store.load_all()} == {"cfg_b"} + + +def test_external_config_change_visible(tmp_path): + import os + + store = ExtConfigStore(tmp_path) + store.upsert(_config("cfg_a", "A")) + assert store.load_all()[0].label == "A" + + cp = tmp_path / "ext_data" / "cfg_a" / "config.json" + raw = json.loads(cp.read_text(encoding="utf-8")) + raw["label"] = "externally-edited" + cp.write_text(json.dumps(raw), encoding="utf-8") + st = cp.stat() + os.utime(cp, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000)) + + assert store.load_all()[0].label == "externally-edited" + + +def test_load_all_returns_copy_not_cached_object(tmp_path): + store = ExtConfigStore(tmp_path) + store.upsert(_config("cfg_a", "A")) + first = store.load_all() + first[0].label = "mutated" + first[0].fields.append(ExtField(name="junk", dtype="string")) + + again = store.load_all() + assert again[0].label == "A" + assert [f.name for f in again[0].fields] == ["score"] + + +def test_load_all_without_config_dir_returns_empty(tmp_path): + store = ExtConfigStore(tmp_path) + assert store.load_all() == [] + assert store.load_all() == [] diff --git a/backend/tests/test_ext_config_store_safety.py b/backend/tests/test_ext_config_store_safety.py new file mode 100644 index 0000000..1374b6b --- /dev/null +++ b/backend/tests/test_ext_config_store_safety.py @@ -0,0 +1,43 @@ +"""ExtConfigStore 的 config_id 安全校验 — 拒绝路径穿越等非法 id (fail-closed)。 + +删除端点的 config_id 来自 URL path 参数, 不经创建端点的 pattern 校验; +若直接拼接路径, `../victim` 可让 rmtree 删除 ext_data 之外的目录。 +""" +from __future__ import annotations + +import json + +from app.services.ext_data import ExtConfigStore + + +def _write_config(data_dir, config_id: str) -> None: + d = data_dir / "ext_data" / config_id + d.mkdir(parents=True, exist_ok=True) + (d / "config.json").write_text( + json.dumps({ + "id": config_id, + "label": "测试", + "mode": "snapshot", + "fields": [{"name": "score"}], + }), + encoding="utf-8", + ) + + +def test_delete_rejects_path_traversal_ids(tmp_path): + store = ExtConfigStore(tmp_path) + _write_config(tmp_path, "ok_config") + + # 穿越目标: ext_data 之外、含 config.json 的目录 (满足旧实现 rmtree 的前置条件) + victim = tmp_path / "victim" + victim.mkdir() + (victim / "config.json").write_text("{}", encoding="utf-8") + + for bad in ("../victim", "..\\victim", "a/b", "", "."): + assert store.delete(bad) is False, bad + assert store.get(bad) is None, bad + assert (victim / "config.json").exists(), "穿越删除必须被拒绝" + + # 合法 id 不受影响 + assert store.get("ok_config") is not None + assert store.delete("ok_config") is True diff --git a/backend/tests/test_factor_library_v2.py b/backend/tests/test_factor_library_v2.py new file mode 100644 index 0000000..b24462b --- /dev/null +++ b/backend/tests/test_factor_library_v2.py @@ -0,0 +1,173 @@ +"""新增因子维度的解析正确性测试 (收益形态/流动性/涨停基因/量价扩展)。 + +金标准路径一致性由 test_matrix_strategy.py::test_research_factor_catalog_matches_matrix_features +覆盖; 这里用可手算的小样本验证公式本身。 +""" +from __future__ import annotations + +import math +from datetime import date, timedelta + +import numpy as np +import polars as pl + +from app.backtest.matrix import build_market_data_matrix, matrix_feature +from app.strategy.scoring import materialize_scoring_columns + + +def _panel(rows: list[dict]) -> pl.DataFrame: + return pl.DataFrame(rows).sort(["symbol", "date"]) + + +def _single_symbol_panel(n_days: int = 70) -> pl.DataFrame: + start = date(2025, 1, 1) + rows = [] + for offset in range(n_days): + change = [0.0, 0.01, -0.02, 0.03, 0.05, -0.01, 0.02, -0.03, 0.04, 0.01][offset % 10] + close = 10.0 if offset == 0 else rows[-1]["close"] * (1 + change) + volume = 1000.0 + (offset % 5) * 200 + consecutive = 0 + if offset % 15 == 0: + consecutive = 1 + (offset // 15) % 3 if offset % 30 == 0 else 1 + rows.append({ + "symbol": "000001.SZ", + "date": start + timedelta(days=offset), + "open": close * 0.99, + "high": close * 1.02, + "low": close * 0.97, + "close": close, + "volume": volume, + "amount": volume * 100.0 * close, + "turnover_rate": 1.0 + (offset % 7) * 0.3, + "consecutive_limit_ups": consecutive, + }) + return _panel(rows) + + +def _tail_value(frame: pl.DataFrame, name: str) -> float: + return frame.tail(1).to_dicts()[0][name] + + +def test_max_ret_up_days_and_limit_up_counts(): + panel = _single_symbol_panel() + frame = materialize_scoring_columns(panel, { + "max_ret_20d", "up_days_20d", + "limit_up_count_20d", "limit_up_count_60d", + }) + + changes = [ + panel["close"][i] / panel["close"][i - 1] - 1 + for i in range(1, panel.height) + ] + window = changes[-20:] + assert math.isclose(_tail_value(frame, "max_ret_20d"), max(window), rel_tol=1e-9) + assert math.isclose( + _tail_value(frame, "up_days_20d"), + float(sum(1 for value in window if value > 0)), + rel_tol=1e-9, + ) + + hits = [ + 1 if (row["consecutive_limit_ups"] or 0) > 0 else 0 + for row in panel.iter_rows(named=True) + ] + assert math.isclose(_tail_value(frame, "limit_up_count_20d"), float(sum(hits[-20:])), rel_tol=1e-9) + assert math.isclose(_tail_value(frame, "limit_up_count_60d"), float(sum(hits[-60:])), rel_tol=1e-9) + + +def test_amihud_and_turnover_z(): + panel = _single_symbol_panel() + frame = materialize_scoring_columns(panel, {"amihud_20d", "turnover_z_60d"}) + + rows = panel.to_dicts() + illiq = [ + abs(rows[i]["close"] / rows[i - 1]["close"] - 1) / (rows[i]["amount"] / 1e8) + for i in range(1, panel.height) + ] + assert math.isclose(_tail_value(frame, "amihud_20d"), sum(illiq[-20:]) / 20, rel_tol=1e-6) + + baseline = [rows[i]["turnover_rate"] for i in range(panel.height - 61, panel.height - 1)] + mean = sum(baseline) / 60 + variance = sum((value - mean) ** 2 for value in baseline) / 59 + std = variance ** 0.5 + expected_z = (rows[-1]["turnover_rate"] - mean) / std + assert math.isclose(_tail_value(frame, "turnover_z_60d"), expected_z, rel_tol=1e-6) + + +def test_vwap_bias_and_vol_trend(): + panel = _single_symbol_panel() + frame = materialize_scoring_columns(panel, {"vwap_bias", "vol_trend_5_60"}) + + last = panel.tail(1).to_dicts()[0] + vwap = last["amount"] / (last["volume"] * 100.0) + assert math.isclose(_tail_value(frame, "vwap_bias"), last["close"] / vwap - 1, rel_tol=1e-9) + + volumes = panel["volume"].to_list() + fast = sum(volumes[-5:]) / 5 + slow = sum(volumes[-60:]) / 60 + assert math.isclose(_tail_value(frame, "vol_trend_5_60"), fast / slow - 1, rel_tol=1e-9) + + +def test_ret_skew_matches_population_skew(): + # 周期夹具的 20 日窗口恰含两个完整周期, 偏度恒为 0; 用不对称收益验证公式。 + start = date(2025, 3, 1) + changes = [0.01, -0.005, -0.004, 0.09, -0.006, 0.002, -0.003, -0.002, -0.005, 0.003] * 7 + rows = [] + close = 10.0 + for offset, change in enumerate(changes): + close = close * (1 + change) + rows.append({ + "symbol": "000001.SZ", + "date": start + timedelta(days=offset), + "open": close * 0.99, + "high": close * 1.02, + "low": close * 0.97, + "close": close, + "volume": 1000.0 + offset, + "amount": (1000.0 + offset) * close, + "turnover_rate": 1.0, + "consecutive_limit_ups": 0, + }) + panel = _panel(rows) + frame = materialize_scoring_columns(panel, {"ret_skew_20d"}) + + window = changes[-20:] + mean = sum(window) / 20 + central_second = sum((value - mean) ** 2 for value in window) / 20 + central_third = sum((value - mean) ** 3 for value in window) / 20 + expected = central_third / central_second ** 1.5 + assert abs(expected) > 0.1 + assert math.isclose(_tail_value(frame, "ret_skew_20d"), expected, rel_tol=1e-6) + + +def test_vol_price_corr_matches_pearson(): + panel = _single_symbol_panel() + frame = materialize_scoring_columns(panel, {"vol_price_corr_20d"}) + + closes = panel["close"].to_list() + changes = [closes[i] / closes[i - 1] - 1 for i in range(panel.height - 20, panel.height)] + volumes = panel["volume"].to_list()[-20:] + n = 20 + mean_x = sum(changes) / n + mean_y = sum(volumes) / n + cov = sum((x - mean_x) * (y - mean_y) for x, y in zip(changes, volumes, strict=True)) / n + var_x = sum((x - mean_x) ** 2 for x in changes) / n + var_y = sum((y - mean_y) ** 2 for y in volumes) / n + expected = cov / (var_x * var_y) ** 0.5 + assert math.isclose(_tail_value(frame, "vol_price_corr_20d"), expected, rel_tol=1e-6) + + +def test_matrix_limit_up_counts_use_consecutive_field(): + panel = _single_symbol_panel() + frame = materialize_scoring_columns(panel, {"limit_up_count_20d"}) + market = build_market_data_matrix( + panel, + field_columns={"amount", "turnover_rate", "consecutive_limit_ups"}, + ) + np.testing.assert_allclose( + matrix_feature(market, "limit_up_count_20d")[:, 0], + frame["limit_up_count_20d"].to_numpy(), + rtol=1e-6, + atol=1e-6, + equal_nan=True, + ) diff --git a/backend/tests/test_fundamental_factors.py b/backend/tests/test_fundamental_factors.py new file mode 100644 index 0000000..861a4c9 --- /dev/null +++ b/backend/tests/test_fundamental_factors.py @@ -0,0 +1,193 @@ +"""财务因子点时接入测试: 公告日门控 / 无数据 null 安全 / 双路径一致 / 历史累积同步。""" +from __future__ import annotations + +from datetime import date, timedelta +from pathlib import Path + +import numpy as np +import polars as pl + +from app.backtest.fundamentals import ( + FUNDAMENTAL_FACTOR_NAMES, + attach_fundamental_factors, + build_fundamental_matrices, + load_fundamental_snapshot, +) +from app.backtest.matrix import build_market_data_matrix + + +def _snapshot_frame(rows: list[dict]) -> pl.DataFrame: + frame = pl.DataFrame({ + "period_end": ["2026-03-31"] * len(rows), + "symbol": [r["symbol"] for r in rows], + "announce_date": [r["announce"] for r in rows], + "roe": [r.get("roe", 10.0) for r in rows], + "bps": [r.get("bps", 5.0) for r in rows], + "revenue_yoy": [r.get("revenue_yoy", 8.0) for r in rows], + }) + return ( + frame.with_columns( + pl.col("announce_date").str.slice(0, 10).str.to_date().alias("_announce") + ) + .sort(["symbol", "_announce"]) + ) + + +def _daily_panel(start: date, days: int, symbols: tuple[str, ...]) -> pl.DataFrame: + rows = [] + for offset in range(days): + for symbol in symbols: + rows.append({ + "symbol": symbol, + "date": start + timedelta(days=offset), + "open": 10.0, + "high": 10.5, + "low": 9.5, + "close": 10.0 + offset * 0.1, + "volume": 1000.0, + }) + return pl.DataFrame(rows).sort(["symbol", "date"]) + + +def test_attach_gates_on_announce_date_strictly(): + panel = _daily_panel(date(2026, 4, 1), 10, ("600000.SH", "000001.SZ")) + snapshot = _snapshot_frame([ + # 公告日 4-5: 4-5 当天不可用, 4-6 起 roe=20 + {"symbol": "600000.SH", "announce": "2026-04-05", "roe": 20.0, "bps": 4.0}, + # 无财务数据的标的 + ]) + attached = attach_fundamental_factors(panel, snapshot, ["roe_latest", "pb_latest"]) + + values = attached.filter(pl.col("symbol") == "600000.SH").sort("date") + roe = values["roe_latest"].to_list() + assert roe[:5] == [None] * 5 # 4-1 ~ 4-5 均不可用 (含公告当日) + assert roe[5:] == [20.0] * 5 # 4-6 起生效 + + # 无财务数据标的全 null, 绝不填 0 + other = attached.filter(pl.col("symbol") == "000001.SZ")["roe_latest"] + assert other.null_count() == other.len() + + # pb = close / bps, 公告前 null + pb = values["pb_latest"].to_list() + assert pb[:5] == [None] * 5 + close_6 = values.filter(pl.col("date") == date(2026, 4, 6))["close"].item() + assert abs(pb[5] - close_6 / 4.0) < 1e-12 + + +def test_attach_replaces_with_newer_announcement(): + panel = _daily_panel(date(2026, 4, 1), 20, ("600000.SH",)) + snapshot = _snapshot_frame([ + {"symbol": "600000.SH", "announce": "2026-04-05", "roe": 20.0}, + {"symbol": "600000.SH", "announce": "2026-04-15", "roe": 33.0}, + ]) + attached = attach_fundamental_factors(panel, snapshot, ["roe_latest"]).sort("date") + roe = attached["roe_latest"].to_list() + assert roe[4] is None # 4-5 公告日 + assert roe[5] == 20.0 # 4-6 起 20.0 + assert roe[13] == 20.0 # 4-14 + assert roe[14] is None # 4-15 二次公告日, 当天仍不可用 (严格大于) + assert roe[15] == 33.0 # 4-16 起新公告生效 + + +def test_attach_without_snapshot_keeps_null_columns(): + panel = _daily_panel(date(2026, 4, 1), 5, ("600000.SH",)) + attached = attach_fundamental_factors(panel, None, ["roe_latest", "pb_latest"]) + for name in ("roe_latest", "pb_latest"): + assert name in attached.columns + assert attached[name].null_count() == attached.height + + +def test_matrix_field_matches_polars_attach(): + panel = _daily_panel(date(2026, 4, 1), 12, ("600000.SH", "000001.SZ")) + snapshot = _snapshot_frame([ + {"symbol": "600000.SH", "announce": "2026-04-05", "roe": 20.0, "bps": 4.0}, + {"symbol": "000001.SZ", "announce": "2026-04-08", "roe": -5.0, "bps": -1.0}, + ]) + attached = attach_fundamental_factors(panel, snapshot, ["roe_latest", "pb_latest"]) + market = build_market_data_matrix(panel) + matrices = build_fundamental_matrices(market, snapshot, ["roe_latest", "pb_latest"]) + symbols = {s: i for i, s in enumerate(market.symbols)} + dates = {str(d): t for t, d in enumerate( + sorted({row["date"] for row in panel.iter_rows(named=True)}) + )} + for name in ("roe_latest", "pb_latest"): + matrix = matrices[name] + for row in attached.iter_rows(named=True): + expected = row[name] + actual = matrix[dates[str(row["date"])], symbols[row["symbol"]]] + if expected is None: + assert np.isnan(actual), (name, row["date"], row["symbol"], actual) + else: + np.testing.assert_allclose(actual, expected, rtol=1e-6) + + +def test_bps_nonpositive_gives_null_pb(): + panel = _daily_panel(date(2026, 4, 1), 8, ("000001.SZ",)) + snapshot = _snapshot_frame([ + {"symbol": "000001.SZ", "announce": "2026-04-02", "bps": -1.0}, + ]) + attached = attach_fundamental_factors(panel, snapshot, ["pb_latest"]) + assert attached["pb_latest"].null_count() == attached.height + + +def test_load_snapshot_from_missing_dir_returns_none(tmp_path: Path): + assert load_fundamental_snapshot(tmp_path) is None + assert load_fundamental_snapshot(None) is None + + +def test_snapshot_requires_announce_date(tmp_path: Path): + out = tmp_path / "financials" / "metrics" + out.mkdir(parents=True) + pl.DataFrame({ + "symbol": ["600000.SH"], + "announce_date": [None], + "roe": [10.0], + "gross_margin": [30.0], + "net_margin": [5.0], + "revenue_yoy": [8.0], + "net_income_yoy": [6.0], + "debt_to_asset_ratio": [40.0], + "bps": [5.0], + }).write_parquet(out / "part.parquet") + # 公告日缺失的行无法做点时门控, 视为无有效快照 + assert load_fundamental_snapshot(tmp_path) is None + + +def test_fundamental_factor_names_are_catalogued(): + from app.backtest.factor import FACTOR_COLUMNS + + catalog_ids = {item["id"] for item in FACTOR_COLUMNS} + assert catalog_ids >= FUNDAMENTAL_FACTOR_NAMES + + +def test_financial_sync_merges_history(tmp_path: Path): + from app.services import financial_sync as fs + + old = pl.DataFrame({ + "symbol": ["600000.SH", "600000.SH"], + "period_end": ["2025-09-30", "2025-12-31"], + "announce_date": ["2025-10-28", "2026-01-20"], + "roe": [8.0, 9.0], + }) + latest = pl.DataFrame({ + "symbol": ["600000.SH", "000001.SZ"], + "period_end": ["2026-03-31", "2026-03-31"], + "announce_date": ["2026-04-25", "2026-04-24"], + "roe": [10.0, 5.0], + }) + merged = fs._merge_report_history(old, latest) + assert merged.height == 4 # 旧各期保留 + 新一期并入 + # 同期修正: 旧 2025-12-31 公告 2026-01-20 vs 更晚的修正公告 + revised = pl.DataFrame({ + "symbol": ["600000.SH"], + "period_end": ["2025-12-31"], + "announce_date": ["2026-02-01"], + "roe": [9.5], + }) + merged2 = fs._merge_report_history(old, revised) + row = merged2.filter( + (pl.col("symbol") == "600000.SH") & (pl.col("period_end") == "2025-12-31") + ) + assert row.height == 1 + assert row["roe"].item() == 9.5 + assert merged2.height == 2 # 修正不增加行数 diff --git a/backend/tests/test_get_daily_cache_reuse.py b/backend/tests/test_get_daily_cache_reuse.py new file mode 100644 index 0000000..7c4d1df --- /dev/null +++ b/backend/tests/test_get_daily_cache_reuse.py @@ -0,0 +1,106 @@ +"""get_daily 复用 enriched 历史缓存的等价性测试。 + +个股对话框打开时 /api/kline/daily 每个行情 tick 调用一次; 旧路径每次 +150 天扫描 + 全套指标重算, 新路径优先从预计算历史缓存裁剪。 +本测试证明: 同一份数据下两条路径输出逐列一致, 且缓存命中时不触发扫描。 +""" +from __future__ import annotations + +from datetime import date, timedelta + +import polars as pl +from polars.testing import assert_frame_equal + +from app.tickflow.repository import KlineRepository + +SYM = "600001.SH" + + +def _raw_frame(days: int = 80) -> pl.DataFrame: + """构造 ~57 个交易日的 14 列形态数据 (复权价与原始价一致, 无除权)。""" + base = date(2026, 4, 1) + rows = [] + price = 10.0 + d = base + while len(rows) < days: + if d.weekday() < 5: + open_ = price * (1 + ((len(rows) % 7) - 3) * 0.004) + close = price * (1 + ((len(rows) % 5) - 2) * 0.006) + high = max(open_, close) * 1.01 + low = min(open_, close) * 0.99 + price = close + rows.append({ + "symbol": SYM, "date": d, + "open": round(open_, 4), "high": round(high, 4), + "low": round(low, 4), "close": round(close, 4), + "volume": 10000.0 + (len(rows) % 10) * 500.0, + "amount": 1.0e7, + "raw_close": round(close, 4), "raw_high": round(high, 4), + "raw_low": round(low, 4), + }) + d += timedelta(days=1) + return pl.DataFrame(rows).sort(["symbol", "date"]) + + +def _bare_repo(raw: pl.DataFrame) -> tuple[KlineRepository, dict]: + repo = KlineRepository.__new__(KlineRepository) + repo._enriched_history_cache = None + repo._enriched_history_start = None + repo._enriched_cache = None + repo._enriched_cache_date = None + repo.get_instruments = lambda: pl.DataFrame() # type: ignore[method-assign] + repo.get_historical_shares = lambda: pl.DataFrame() # type: ignore[method-assign] + repo.get_enriched_latest = lambda: (pl.DataFrame(), None) # type: ignore[method-assign] + calls = {"scan": 0} + + def _scan(symbol, start, end, columns): + calls["scan"] += 1 + return raw.filter((pl.col("date") >= start) & (pl.col("date") <= end)) + + repo._scan_daily_symbol = _scan # type: ignore[method-assign] + return repo, calls + + +def test_get_daily_cache_path_matches_scan_path(): + raw = _raw_frame() + dates = raw["date"].to_list() + start, end = dates[30], dates[-1] + + # 旧路径: 无历史缓存 → 扫描 + 即时计算 + repo_old, calls_old = _bare_repo(raw) + result_old = repo_old.get_daily(SYM, start, end) + assert calls_old["scan"] == 1 + + # 新路径: 预计算历史缓存 (同一真实计算栈 _compute_enriched_range 构建) + repo_new, calls_new = _bare_repo(raw) + hist = repo_new._compute_enriched_range(raw) + repo_new._enriched_history_cache = hist + repo_new._enriched_history_start = hist["date"].min() + result_new = repo_new.get_daily(SYM, start, end) + + assert calls_new["scan"] == 0, "缓存命中时不得回退到扫描路径" + assert result_new.height == result_old.height + + common = sorted(set(result_old.columns) & set(result_new.columns)) + assert {"open", "close", "ma5", "ma20", "ma60", "rsi_14"} <= set(common) + assert_frame_equal( + result_old.sort("date").select(common), + result_new.sort("date").select(common), + check_exact=False, + rel_tol=1e-9, + ) + + +def test_get_daily_falls_back_to_scan_when_cache_does_not_cover_start(): + raw = _raw_frame() + dates = raw["date"].to_list() + + repo, calls = _bare_repo(raw) + hist = repo._compute_enriched_range(raw) + repo._enriched_history_cache = hist + repo._enriched_history_start = hist["date"].min() + + # 请求起点早于缓存覆盖 → 必须回退扫描路径 + result = repo.get_daily(SYM, dates[0] - timedelta(days=5), dates[-1]) + assert calls["scan"] == 1 + assert not result.is_empty() diff --git a/backend/tests/test_heavy_job_limiter.py b/backend/tests/test_heavy_job_limiter.py new file mode 100644 index 0000000..5ebc56d --- /dev/null +++ b/backend/tests/test_heavy_job_limiter.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import threading +import time + +import pytest + +from app.services.heavy_job_limiter import ( + HeavyJobCancelledError, + HeavyJobLimiter, + HeavyJobLimitTimeoutError, + heavy_job_limiter, + shared_heavy_job_limiter, +) + + +def test_weighted_capacity_and_timeout() -> None: + limiter = HeavyJobLimiter(capacity=2) + + assert limiter.acquire("normal", timeout=0) + assert limiter.acquire("normal", timeout=0) + assert limiter.available == 0 + assert not limiter.acquire("normal", timeout=0.01) + limiter.release("normal") + assert not limiter.acquire("mining", timeout=0) + limiter.release("normal") + assert limiter.acquire("mining", timeout=0) + assert limiter.in_use == 2 + limiter.release("mining") + + +def test_waiting_acquire_can_be_cancelled() -> None: + limiter = HeavyJobLimiter(capacity=2, cancel_poll_interval=0.01) + cancel_event = threading.Event() + assert limiter.acquire("mining", timeout=0) + + result: list[bool] = [] + waiter = threading.Thread( + target=lambda: result.append( + limiter.acquire("normal", timeout=1, cancel_event=cancel_event) + ) + ) + waiter.start() + time.sleep(0.03) + cancel_event.set() + waiter.join(timeout=1) + + assert not waiter.is_alive() + assert result == [False] + assert limiter.in_use == 2 + limiter.release("mining") + + +def test_context_manager_releases_after_body_error() -> None: + limiter = HeavyJobLimiter(capacity=2) + + with pytest.raises(ValueError, match="body failed"), limiter.slot("mining", timeout=0): + raise ValueError("body failed") + + assert limiter.in_use == 0 + assert limiter.acquire("mining", timeout=0) + limiter.release("mining") + + +def test_context_manager_distinguishes_timeout_and_cancellation() -> None: + limiter = HeavyJobLimiter(capacity=2) + assert limiter.acquire("mining", timeout=0) + + with pytest.raises(HeavyJobLimitTimeoutError), limiter.slot("normal", timeout=0.01): + pytest.fail("unreachable") + + cancelled = threading.Event() + cancelled.set() + with pytest.raises(HeavyJobCancelledError), limiter.slot("normal", cancel_event=cancelled): + pytest.fail("unreachable") + + limiter.release("mining") + + +def test_invalid_release_does_not_overfill_capacity() -> None: + limiter = HeavyJobLimiter(capacity=2) + + with pytest.raises(RuntimeError): + limiter.release("normal") + assert limiter.available == 2 + + assert limiter.acquire("normal", timeout=0) + with pytest.raises(RuntimeError): + limiter.release("mining") + assert limiter.in_use == 1 + limiter.release("normal") + + +def test_module_aliases_share_the_capacity_two_singleton() -> None: + assert heavy_job_limiter is shared_heavy_job_limiter + assert shared_heavy_job_limiter.capacity == 2 diff --git a/backend/tests/test_indicator_needed.py b/backend/tests/test_indicator_needed.py index fe4d7ba..1e38b87 100644 --- a/backend/tests/test_indicator_needed.py +++ b/backend/tests/test_indicator_needed.py @@ -28,6 +28,19 @@ def _bars(n: int = 90) -> pl.DataFrame: return pl.DataFrame(rows) +def test_compute_indicators_assume_sorted_matches_default_values(): + bars = _bars().sort(["symbol", "date"]) + + default = compute_indicators(bars, needed={"macd_hist", "rsi_14", "momentum_20d"}) + fast = compute_indicators( + bars, + needed={"macd_hist", "rsi_14", "momentum_20d"}, + assume_sorted=True, + ) + + assert fast.equals(default) + + def test_compute_signals_subset_matches_full_values(): indicators = compute_indicators(_bars()) full = compute_signals(indicators) diff --git a/backend/tests/test_kline_hot_path_memory.py b/backend/tests/test_kline_hot_path_memory.py new file mode 100644 index 0000000..3f9707c --- /dev/null +++ b/backend/tests/test_kline_hot_path_memory.py @@ -0,0 +1,155 @@ +"""kline 每秒热路径测试 — _get_stock_info 走内存缓存, _attach_ext 复用签名缓存 value_map。 + +契约等值: 输出与旧的 DuckDB 扫描 / 逐配置读 parquet 路径一致。 +""" +from __future__ import annotations + +from types import SimpleNamespace + +import polars as pl + +from app.api import screener +from app.api.kline import _attach_ext, _get_stock_info +from app.services.ext_data import ExtConfig, ExtConfigStore, ExtField + + +class _FakeRepo: + """最小 repo 替身: _get_stock_info/_load_ext_value_maps 只用到这些成员。""" + + def __init__(self, instruments: pl.DataFrame, data_dir) -> None: + self._instruments = instruments + self.store = SimpleNamespace(data_dir=data_dir, db=None) + self.execute_one_calls = 0 + + def get_instruments(self) -> pl.DataFrame: + return self._instruments + + def execute_one(self, *args, **kwargs): + self.execute_one_calls += 1 + raise AssertionError("热路径不得再走 DuckDB 扫 instruments parquet") + + +def _instruments_df() -> pl.DataFrame: + return pl.DataFrame({ + "symbol": ["600000.SH", "000001.SZ", "510300.SH"], + "name": ["浦发银行", "平安银行", "沪深300ETF"], + "total_shares": [1.0e9, 2.0e9, None], + "float_shares": [5.0e8, 1.0e9, None], + }) + + +# ---------------------------------------------------------------- _get_stock_info + +def test_get_stock_info_reads_memory_cache(): + repo = _FakeRepo(_instruments_df(), None) + assert _get_stock_info(repo, "600000.SH") == { + "name": "浦发银行", "total_shares": 1.0e9, "float_shares": 5.0e8, + } + assert repo.execute_one_calls == 0 + + +def test_get_stock_info_null_shares_become_none(): + repo = _FakeRepo(_instruments_df(), None) + info = _get_stock_info(repo, "510300.SH") + assert info["name"] == "沪深300ETF" + assert info["total_shares"] is None + assert info["float_shares"] is None + + +def test_get_stock_info_missing_symbol_returns_empty(): + repo = _FakeRepo(_instruments_df(), None) + assert _get_stock_info(repo, "999999.SH") == {} + + +def test_get_stock_info_missing_columns_returns_empty_like_sql_error(): + df = pl.DataFrame({"symbol": ["600000.SH"], "name": ["浦发银行"]}) + repo = _FakeRepo(df, None) + assert _get_stock_info(repo, "600000.SH") == {} + + +def test_get_stock_info_repo_failure_returns_empty(): + repo = SimpleNamespace() + repo.get_instruments = lambda: (_ for _ in ()).throw(RuntimeError("boom")) + assert _get_stock_info(repo, "600000.SH") == {} + + +# ---------------------------------------------------------------- _attach_ext + +def test_attach_ext_empty_columns_untouched(): + resp = {"symbol": "600000.SH", "stock_info": {"name": "x"}, "rows": []} + repo = _FakeRepo(_instruments_df(), None) + out = _attach_ext(resp, repo, "600000.SH", None) + assert out is resp + assert "ext" not in out["stock_info"] + out = _attach_ext(resp, repo, "600000.SH", " ") + assert "ext" not in out["stock_info"] + + +def test_attach_ext_invalid_spec_untouched(): + resp = {"symbol": "600000.SH", "stock_info": {"name": "x"}, "rows": []} + repo = _FakeRepo(_instruments_df(), None) + out = _attach_ext(resp, repo, "600000.SH", "foo_bar") # 无 '.' 分隔 + assert out is resp + assert "ext" not in out["stock_info"] + + +def test_attach_ext_maps_values_from_loader(monkeypatch): + resp = {"symbol": "600000.SH", "stock_info": {"name": "x"}, "rows": []} + original_info = resp["stock_info"] + repo = _FakeRepo(_instruments_df(), None) + monkeypatch.setattr( + screener, "_load_ext_value_maps", + lambda r, cols: {"cfg1__score": {"600000.SH": 88.5}, "cfg1__note": {}}, + ) + out = _attach_ext(resp, repo, "600000.SH", "cfg1.score,cfg1.note") + assert out["stock_info"]["ext"] == {"cfg1__score": 88.5, "cfg1__note": None} + # 原地更新 resp 是该函数契约; 但原 stock_info 字典对象不被修改 (与旧实现一致) + assert out is resp + assert "ext" not in original_info + assert out["stock_info"] is not original_info + + +def test_attach_ext_loader_failure_yields_none_values(monkeypatch): + resp = {"symbol": "600000.SH", "stock_info": {}, "rows": []} + repo = _FakeRepo(_instruments_df(), None) + + def _boom(r, cols): + raise RuntimeError("boom") + + monkeypatch.setattr(screener, "_load_ext_value_maps", _boom) + out = _attach_ext(resp, repo, "600000.SH", "cfg1.score") + assert out["stock_info"]["ext"] == {"cfg1__score": None} + + +# ------------------------------------------- _attach_ext 与真实 _load_ext_value_maps 集成 + +def _setup_ext_snapshot(tmp_path): + store = ExtConfigStore(tmp_path) + store.upsert(ExtConfig( + id="benchx", label="基准扩展", mode="snapshot", + fields=[ExtField(name="score", dtype="float")], + )) + cfg_dir = tmp_path / "ext_data" / "benchx" + cfg_dir.mkdir(parents=True, exist_ok=True) + pl.DataFrame({ + "symbol": [f"{600000 + i}.SH" for i in range(50)], + "score": [float(i) for i in range(50)], + }).write_parquet(cfg_dir / "part.parquet") + return store + + +def test_attach_ext_real_loader_returns_parquet_values(tmp_path): + _setup_ext_snapshot(tmp_path) + repo = _FakeRepo(_instruments_df(), tmp_path) + resp = {"symbol": "600005.SH", "stock_info": {"name": "x"}, "rows": []} + out = _attach_ext(resp, repo, "600005.SH", "benchx.score") + assert out["stock_info"]["ext"] == {"benchx__score": 5.0} + assert isinstance(out["stock_info"]["ext"]["benchx__score"], float) + + +def test_attach_ext_real_loader_unknown_symbol_is_none(tmp_path): + _setup_ext_snapshot(tmp_path) + repo = _FakeRepo(_instruments_df(), tmp_path) + resp = {"symbol": "999999.SH", "stock_info": {}, "rows": []} + out = _attach_ext(resp, repo, "999999.SH", "benchx.score") + assert out["stock_info"]["ext"] == {"benchx__score": None} diff --git a/backend/tests/test_kline_sync_timezone.py b/backend/tests/test_kline_sync_timezone.py new file mode 100644 index 0000000..44d3c78 --- /dev/null +++ b/backend/tests/test_kline_sync_timezone.py @@ -0,0 +1,38 @@ +"""时区契约测试 — 分时拉取窗口必须按北京时间解释, 与服务器本地时区无关。 + +fetch_minute_single 构造的 naive datetime 会被 _datetime_to_ms 的 .timestamp() +按服务器本地时区解释: UTC 容器 (Docker 默认) 上窗口偏移 8 小时, 补拉必为空。 +""" +from __future__ import annotations + +from datetime import date, datetime + +from app.market_time import CN_TZ +from app.services import kline_sync + + +def test_fetch_minute_single_window_is_beijing_wall_clock(monkeypatch): + captured: dict[str, int] = {} + + def _fake_try_custom_minute(*args, **kwargs): + return (None, True) # 未配自定义源 → 走 TickFlow 分支 + + class _FakeKlines: + @staticmethod + def batch(symbols, period, start_time, end_time, **kwargs): + captured["start_ms"] = start_time + captured["end_ms"] = end_time + return [] + + class _FakeClient: + klines = _FakeKlines + + monkeypatch.setattr(kline_sync, "_try_custom_minute", _fake_try_custom_minute) + monkeypatch.setattr(kline_sync, "get_client", lambda: _FakeClient()) + + kline_sync.fetch_minute_single("600000.SH", date(2026, 8, 14)) + + start = datetime.fromtimestamp(captured["start_ms"] / 1000, tz=CN_TZ) + end = datetime.fromtimestamp(captured["end_ms"] / 1000, tz=CN_TZ) + assert (start.date(), start.hour, start.minute) == (date(2026, 8, 14), 9, 25) + assert (end.date(), end.hour, end.minute) == (date(2026, 8, 14), 15, 5) diff --git a/backend/tests/test_last_fetch_throttle.py b/backend/tests/test_last_fetch_throttle.py new file mode 100644 index 0000000..05d4073 --- /dev/null +++ b/backend/tests/test_last_fetch_throttle.py @@ -0,0 +1,64 @@ +"""last_fetch 落盘节流测试 — 30s 内只写一次盘, 内存值路径不受影响。""" +from __future__ import annotations + +import pytest + +from app.services import preferences, quote_service + + +@pytest.fixture(autouse=True) +def _reset_throttle_state(monkeypatch): + monkeypatch.setattr(quote_service, "_last_fetch_written_at_ms", 0.0) + yield + quote_service._last_fetch_written_at_ms = 0.0 + + +@pytest.fixture() +def save_counter(monkeypatch): + counter = {"saves": 0, "values": []} + + def _counting(updates: dict) -> dict: + counter["saves"] += 1 + counter["values"].append(updates) + return {} + + monkeypatch.setattr(preferences, "save", _counting) + return counter + + +def test_first_call_after_start_writes(save_counter): + quote_service._persist_last_fetch(1_700_000_000_000.0) + assert save_counter["saves"] == 1 + assert save_counter["values"][0] == {"last_fetch_ms": 1_700_000_000_000.0} + + +def test_writes_within_window_are_skipped(save_counter): + t0 = 1_700_000_000_000.0 + quote_service._persist_last_fetch(t0) + quote_service._persist_last_fetch(t0 + 10_000.0) # +10s: 跳过 + quote_service._persist_last_fetch(t0 + 29_999.0) # 仍不足 30s: 跳过 + assert save_counter["saves"] == 1 + + +def test_write_resumes_after_window(save_counter): + t0 = 1_700_000_000_000.0 + quote_service._persist_last_fetch(t0) + quote_service._persist_last_fetch(t0 + 30_000.0) # 恰好 30s: 恢复写盘 + assert save_counter["saves"] == 2 + assert save_counter["values"][1] == {"last_fetch_ms": 1_700_000_030_000.0} + + +def test_write_failure_does_not_propagate_and_retries(monkeypatch): + calls = {"n": 0} + + def _flaky(updates: dict) -> dict: + calls["n"] += 1 + if calls["n"] == 1: + raise OSError("disk full") + return {} + + monkeypatch.setattr(preferences, "save", _flaky) + t0 = 1_700_000_000_000.0 + quote_service._persist_last_fetch(t0) # 失败但不抛出 + quote_service._persist_last_fetch(t0 + 1_000.0) # 未成功过 -> 立即重试 + assert calls["n"] == 2 diff --git a/backend/tests/test_main_mining_lifespan.py b/backend/tests/test_main_mining_lifespan.py new file mode 100644 index 0000000..8b6e9e0 --- /dev/null +++ b/backend/tests/test_main_mining_lifespan.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace + +import pytest + +import app.main as main_module + + +def test_lifespan_holds_mining_process_lock_around_application(monkeypatch) -> None: + events: list[str] = [] + + class LockStub: + def __init__(self, data_dir) -> None: + del data_dir + events.append("lock_created") + + def acquire(self) -> None: + events.append("lock_acquired") + + def release(self) -> None: + events.append("lock_released") + + @asynccontextmanager + async def application_lifespan(_app): + events.append("application_started") + try: + yield + finally: + events.append("application_stopped") + + monkeypatch.setattr(main_module, "MiningProcessLock", LockStub) + monkeypatch.setattr(main_module, "_application_lifespan", application_lifespan) + + async def exercise() -> None: + async with main_module.lifespan(SimpleNamespace()): + events.append("request_serving") + + asyncio.run(exercise()) + + assert events == [ + "lock_created", + "lock_acquired", + "application_started", + "request_serving", + "application_stopped", + "lock_released", + ] + + +def test_lifespan_releases_lock_when_application_shutdown_raises(monkeypatch) -> None: + events: list[str] = [] + + class LockStub: + def __init__(self, data_dir) -> None: + del data_dir + + def acquire(self) -> None: + events.append("acquired") + + def release(self) -> None: + events.append("released") + + @asynccontextmanager + async def application_lifespan(_app): + try: + yield + finally: + raise RuntimeError("shutdown failed") + + monkeypatch.setattr(main_module, "MiningProcessLock", LockStub) + monkeypatch.setattr(main_module, "_application_lifespan", application_lifespan) + + async def exercise() -> None: + async with main_module.lifespan(SimpleNamespace()): + pass + + with pytest.raises(RuntimeError, match="shutdown failed"): + asyncio.run(exercise()) + + assert events == ["acquired", "released"] diff --git a/backend/tests/test_market_mainline.py b/backend/tests/test_market_mainline.py new file mode 100644 index 0000000..2581d5c --- /dev/null +++ b/backend/tests/test_market_mainline.py @@ -0,0 +1,162 @@ +"""市场主线(market_mainline)与过滤配置单元测试。""" +from __future__ import annotations + +from datetime import date + +import polars as pl + +from app.services import market_mainline, preferences + + +def _write_enriched(root, rows: list[dict]) -> None: + enriched = root / "kline_daily_enriched" + by_date: dict[date, list[dict]] = {} + for r in rows: + by_date.setdefault(r["date"], []).append(r) + for d, day_rows in by_date.items(): + part = enriched / f"date={d.isoformat()}" / "part.parquet" + part.parent.mkdir(parents=True, exist_ok=True) + pl.DataFrame(day_rows).write_parquet(part) + + +def _fake_repo(tmp_path): + import types + + return types.SimpleNamespace(store=types.SimpleNamespace(data_dir=tmp_path)) + + +def _patch_map(monkeypatch, mapping: dict[str, list[str]], kind: str = "concept") -> None: + map_df = pl.DataFrame( + {"_sym_up": [s for s, ms in mapping.items() for _ in ms], + kind: [m for _, ms in mapping.items() for m in ms]}, + schema={"_sym_up": pl.Utf8, kind: pl.Utf8}, + ).unique() + + def fake_load(repo, k="concept"): + return (map_df, map_df[kind].n_unique()) if k == kind else (pl.DataFrame(), 0) + + monkeypatch.setattr(market_mainline, "_load_concept_map_df", fake_load) + + +def _mk_rows(d: date, spec: list[tuple[str, int, float]]) -> list[dict]: + return [ + {"symbol": sym, "date": d, "consecutive_limit_ups": consec, "amount": amt} + for sym, consec, amt in spec + ] + + +class TestComputeMainline: + def _setup(self, tmp_path, monkeypatch): + d1, d2 = date(2024, 1, 2), date(2024, 1, 3) + # 概念 X: d1 三个涨停(2,1,1), d2 三个涨停(3,2,1); 概念 Y: 单股 2 板 + # S5 无概念映射; 大概念 BIG 成员 700 家但只有 5 家涨停(数据里只写 5 行) + rows = _mk_rows(d1, [("S1.SH", 2, 5e8), ("S2.SH", 1, 1e8), ("S3.SH", 1, 2e8), + ("S4.SH", 2, 3e8), ("S5.SH", 1, 1e8), + ("B1.SH", 1, 1e8), ("B2.SH", 1, 1e8)]) + rows += _mk_rows(d2, [("S1.SH", 3, 6e8), ("S2.SH", 2, 2e8), ("S3.SH", 0, 1e8), + ("S4.SH", 3, 4e8), ("S5.SH", 1, 1e8), + ("B1.SH", 2, 1e8), ("B2.SH", 0, 1e8)]) + _write_enriched(tmp_path, rows) + mapping = { + "S1.SH": ["X"], "S2.SH": ["X"], "S3.SH": ["X"], + "S4.SH": ["X", "Y"], "S5.SH": [], + "B1.SH": ["BIG"], "B2.SH": ["BIG"], + **{f"F{i}.SH": ["BIG"] for i in range(700)}, # BIG 成员 702 → 超 600 上限 + } + _patch_map(monkeypatch, mapping) + return _fake_repo(tmp_path), d1, d2 + + def test_aggregation_and_big_concept_filter(self, tmp_path, monkeypatch): + repo, d1, d2 = self._setup(tmp_path, monkeypatch) + out = market_mainline.compute_mainline_range( + repo, tmp_path, d1, d2, kind="concept", + filter_cfg={"min_members": 4, "max_members": 600, "blacklist": []}, + ) + members = set(out["member"].to_list()) + assert "BIG" not in members # 成员数超上限被过滤 + assert "X" in members + x_d2 = out.filter((pl.col("date") == d2) & (pl.col("member") == "X")).to_dicts()[0] + assert x_d2["limit_up_count"] == 3 # S1,S2,S4 + assert x_d2["ge2_count"] == 3 + assert x_d2["max_boards"] == 3 + assert x_d2["rungs_filled"] == 2 # 档位 {2,3} + assert x_d2["leader_symbol"] == "S1.SH" # 最高板且成交额大 + assert x_d2["rank"] == 1 + + def test_blacklist_and_min_limit_up(self, tmp_path, monkeypatch): + repo, d1, d2 = self._setup(tmp_path, monkeypatch) + out = market_mainline.compute_mainline_range( + repo, tmp_path, d1, d2, kind="concept", + filter_cfg={"min_members": 1, "max_members": 5000, "blacklist": ["X"]}, + ) + # X 被黑名单; BIG 只有 2-3 家涨停 < _MIN_LIMIT_UP=3 也不参与 → 只剩空/无 X + assert "X" not in set(out["member"].to_list()) + + def test_upsert_replaces_same_day_kind(self, tmp_path, monkeypatch): + repo, d1, d2 = self._setup(tmp_path, monkeypatch) + cfg = {"min_members": 4, "max_members": 600, "blacklist": []} + first = market_mainline.compute_mainline_range(repo, tmp_path, d1, d1, kind="concept", filter_cfg=cfg) + market_mainline.upsert_mainline_history(tmp_path, first) + both = market_mainline.compute_mainline_range(repo, tmp_path, d1, d2, kind="concept", filter_cfg=cfg) + market_mainline.upsert_mainline_history(tmp_path, both) + stored = pl.read_parquet(market_mainline.mainline_path(tmp_path)) + assert set(stored["date"].to_list()) == {d1, d2} + # 同日重算不产生重复行 + assert stored.filter(pl.col("date") == d1).height == first.height + + def test_incremental_fills_missing_days(self, tmp_path, monkeypatch): + repo, d1, d2 = self._setup(tmp_path, monkeypatch) + cfg = {"min_members": 4, "max_members": 600, "blacklist": []} + first = market_mainline.compute_mainline_range(repo, tmp_path, d1, d1, kind="concept", filter_cfg=cfg) + market_mainline.upsert_mainline_history(tmp_path, first) + new = market_mainline.compute_mainline_incremental(repo, tmp_path, kind="concept") + assert not new.is_empty() + assert set(new["date"].to_list()) == {d2} + + def test_industry_level_truncation(self, tmp_path, monkeypatch): + d1 = date(2024, 1, 2) + rows = _mk_rows(d1, [("S1.SH", 2, 5e8), ("S2.SH", 1, 1e8), + ("S3.SH", 1, 2e8), ("S4.SH", 3, 4e8)]) + _write_enriched(tmp_path, rows) + _patch_map( + monkeypatch, + {"S1.SH": ["计算机-软件开发-垂直应用软件"], + "S2.SH": ["计算机-软件开发-垂直应用软件"], + "S3.SH": ["计算机-IT服务-IT服务Ⅲ"], + "S4.SH": ["计算机-软件开发-垂直应用软件"]}, + kind="industry", + ) + out = market_mainline.compute_mainline_range( + _fake_repo(tmp_path), tmp_path, d1, d1, kind="industry", + filter_cfg={"min_members": 1, "max_members": 5000, "blacklist": []}, + ) + members = set(out["member"].to_list()) + assert "计算机-软件开发" in members + assert all(m.count("-") <= 1 for m in members) + sw = out.filter(pl.col("member") == "计算机-软件开发").to_dicts()[0] + assert sw["limit_up_count"] == 3 + assert sw["max_boards"] == 3 + + +class TestMainlineFilterPreferences: + def test_blacklist_string_parsing_and_clamp(self, tmp_path, monkeypatch): + path = tmp_path / "preferences.json" + monkeypatch.setattr(preferences, "_path", lambda: path) + got = preferences.set_mainline_filter_config({ + "max_members": 99999, # 超上限被夹到 5000 + "min_members": 0, # 低于下限被夹到 1 + "blacklist": "融资融券, 沪股通；深股通", # noqa: RUF001 + }) + assert got["max_members"] == 5000 + assert got["min_members"] == 1 + assert set(got["blacklist"]) == {"融资融券", "沪股通", "深股通"} + # 部分更新: 只改黑名单, 其他保持 + got2 = preferences.set_mainline_filter_config({"blacklist": ["ST板块"]}) + assert got2["blacklist"] == ["ST板块"] + assert got2["max_members"] == 5000 + + def test_defaults(self, tmp_path, monkeypatch): + path = tmp_path / "preferences.json" + monkeypatch.setattr(preferences, "_path", lambda: path) + cfg = preferences.get_mainline_filter_config() + assert cfg == {"min_members": 4, "max_members": 600, "blacklist": []} diff --git a/backend/tests/test_market_phase.py b/backend/tests/test_market_phase.py new file mode 100644 index 0000000..0468b4b --- /dev/null +++ b/backend/tests/test_market_phase.py @@ -0,0 +1,200 @@ +"""市场情绪周期阶段(market_phase)单元测试 — 梯队指标与阶段规则引擎。""" +from __future__ import annotations + +from datetime import date, timedelta +from itertools import pairwise + +import polars as pl + +from app.services.market_phase import ( + CLIMAX_GE2, + EBB_PROMO, + ICE_FIRST_BOARD, + ICE_GE2, + ICE_HEIGHT, + PHASE_CLIMAX, + PHASE_EBB, + PHASE_ICE, + PHASE_IGNITE, + PHASE_RALLY, + PHASE_REPAIR, + classify_phase_series, + finalize_ladder_row, + with_prev_consecutive, +) +from app.services.regime_builder import _aggregate_daily, refresh_phase_labels, regime_path + + +def _days(n: int, start: str = "2024-01-01") -> list[date]: + d0 = date.fromisoformat(start) + out, cur = [], d0 + while len(out) < n: + if cur.weekday() < 5: + out.append(cur) + cur += timedelta(days=1) + return out + + +def _frame(rows: list[dict]) -> pl.DataFrame: + base = { + "change_pct": 0.01, "amount": 1e8, "close": 10.0, "ma20": 9.5, + "signal_limit_up": True, "signal_limit_down": False, + "signal_broken_limit_up": False, + } + return pl.DataFrame([{**base, **r} for r in rows]) + + +class TestLadderMetrics: + def test_prev_consecutive_and_counts(self): + d1, d2, d3 = _days(3) + df = _frame([ + {"symbol": "A", "date": d1, "consecutive_limit_ups": 1}, + {"symbol": "A", "date": d2, "consecutive_limit_ups": 2}, + {"symbol": "A", "date": d3, "consecutive_limit_ups": 0}, + {"symbol": "B", "date": d1, "consecutive_limit_ups": 1}, + {"symbol": "B", "date": d2, "consecutive_limit_ups": 0}, + {"symbol": "C", "date": d2, "consecutive_limit_ups": 3}, + ]) + out = with_prev_consecutive(df) + prev = {r["symbol"] + str(r["date"]): r["_prev_consec"] for r in out.iter_rows(named=True)} + assert prev["A" + str(d1)] is None + assert prev["A" + str(d2)] == 1 + assert prev["B" + str(d2)] == 1 + + agg = _aggregate_daily(df) + by_date = {r["date"]: r for r in agg.iter_rows(named=True)} + assert by_date[d1]["first_board"] == 2 + assert by_date[d1]["ge2_count"] == 0 + # d2: A=2板(晋级), B=断板(昨1今0), C=3板(新面孔); pool=2(A,B) <10 → promo null + assert by_date[d2]["ge2_count"] == 2 + assert by_date[d2]["promo_pool"] == 2 + assert by_date[d2]["promo_rate"] is None + assert by_date[d2]["ladder_completeness"] == 1.0 # 档位 {2,3}, height=3 → 2/2 + assert by_date[d3]["first_board"] == 0 + + def test_promo_rate_with_sufficient_pool(self): + d1, d2 = _days(2) + rows = [] + for i in range(12): + rows.append({"symbol": f"S{i}", "date": d1, "consecutive_limit_ups": 1}) + rows.append({ + "symbol": f"S{i}", "date": d2, + "consecutive_limit_ups": 2 if i < 4 else 0, # 4 晋级, 8 断板 + }) + agg = _aggregate_daily(_frame(rows)) + d2_row = {r["date"]: r for r in agg.iter_rows(named=True)}[d2] + assert d2_row["promo_pool"] == 12 + assert d2_row["promo_rate"] == round(4 / 12, 4) + + def test_promo_small_pool_is_null(self): + row = {"promo_pool": 5, "promo_ok": 5, "max_consecutive": 3, "rungs_filled": 2} + assert finalize_ladder_row(row)["promo_rate"] is None + row2 = {"promo_pool": 20, "promo_ok": 8, "max_consecutive": 3, "rungs_filled": 2} + assert finalize_ladder_row(row2)["promo_rate"] == 0.4 + + def test_completeness_gap(self): + # height=5, 只有 2板和5板 → rungs {2,5} → 2/4 + row = {"max_consecutive": 5, "rungs_filled": 2, "promo_pool": 0, "promo_ok": 0} + assert finalize_ladder_row(row)["ladder_completeness"] == 0.5 + + +def _series(specs: list[dict]) -> pl.DataFrame: + """specs: [{days, height, first, ge2, promo, seal, state}] 逐段展开成日序。""" + rows = [] + for sp in specs: + for _ in range(sp["days"]): + rows.append({ + "date": None, # 由调用方生成后填充 + "max_consecutive": sp["height"], + "first_board": sp["first"], + "ge2_count": sp["ge2"], + "promo_rate": sp["promo"], + "seal_rate": sp["seal"], + **({"state": sp.get("state", "range")} if sp.get("state") else {}), + }) + dates = _days(len(rows)) + for r, d in zip(rows, dates, strict=True): + r["date"] = d + return pl.DataFrame(rows) + + +class TestClassifyPhaseSeries: + def _labels(self, specs): + df = _series(specs) + return classify_phase_series(df)["phase"].to_list() + + def test_climax_and_persistence(self): + labels = self._labels([ + {"days": 6, "height": 5, "first": 40, "ge2": 12, "promo": 0.2, "seal": 0.6, "state": "strong"}, + {"days": 5, "height": 12, "first": 300, "ge2": CLIMAX_GE2 + 40, "promo": 0.5, "seal": 0.7, "state": "strong"}, + {"days": 8, "height": 5, "first": 40, "ge2": 12, "promo": 0.2, "seal": 0.6, "state": "range"}, + ]) + assert PHASE_CLIMAX in labels + assert labels[-1] == PHASE_REPAIR + # EMA 平滑 + 2 日确认: 阶段切换总数有限, 不出现 1 日翻转噪声 + switches = sum(1 for a, b in pairwise(labels) if a != b) + assert switches <= 4 + + def test_rally_positive_state(self): + labels = self._labels([ + {"days": 8, "height": 8, "first": 60, "ge2": 18, "promo": 0.3, "seal": 0.7, "state": "strong"}, + ]) + assert PHASE_RALLY in labels + + def test_rally_vetoed_in_weak_state(self): + labels = self._labels([ + {"days": 8, "height": 8, "first": 60, "ge2": 18, "promo": 0.3, "seal": 0.7, "state": "weak"}, + ]) + assert PHASE_RALLY not in labels + assert all(p == PHASE_REPAIR for p in labels) + + def test_ice(self): + labels = self._labels([ + {"days": 6, "height": ICE_HEIGHT - 1, "first": ICE_FIRST_BOARD - 1, + "ge2": ICE_GE2 - 1, "promo": 0.1, "seal": 0.5, "state": "range"}, + ]) + assert PHASE_ICE in labels + + def test_ebb_from_high(self): + labels = self._labels([ + {"days": 8, "height": 9, "first": 60, "ge2": 20, "promo": 0.3, "seal": 0.7, "state": "strong"}, + {"days": 6, "height": 6, "first": 30, "ge2": 6, "promo": EBB_PROMO - 0.03, "seal": 0.5, "state": "range"}, + ]) + assert PHASE_EBB in labels + + def test_ignite_expansion(self): + labels = self._labels([ + {"days": 6, "height": 4, "first": 25, "ge2": 5, "promo": 0.15, "seal": 0.6, "state": "range"}, + {"days": 8, "height": 6, "first": 50, "ge2": 14, "promo": 0.24, "seal": 0.68, "state": "lean_strong"}, + ]) + assert PHASE_IGNITE in labels + + def test_promo_null_leading_days_ffilled(self): + specs = [ + {"days": 4, "height": 5, "first": 40, "ge2": 10, "promo": None, "seal": 0.6, "state": "range"}, + {"days": 4, "height": 6, "first": 50, "ge2": 14, "promo": 0.25, "seal": 0.68, "state": "strong"}, + ] + df = _series(specs) + out = classify_phase_series(df) + assert out["phase"].null_count() == 0 + + +class TestRefreshPhaseLabels: + def test_roundtrip_writes_phase_keeps_state(self, tmp_path): + specs = [ + {"days": 10, "height": 8, "first": 60, "ge2": 18, "promo": 0.3, "seal": 0.7, "state": "strong"}, + ] + df = _series(specs) + regime_path(tmp_path).parent.mkdir(parents=True) + df.write_parquet(regime_path(tmp_path)) + n = refresh_phase_labels(tmp_path) + assert n == 10 + out = pl.read_parquet(regime_path(tmp_path)) + assert "phase" in out.columns + assert PHASE_RALLY in set(out["phase"].to_list()) + assert set(out["state"].to_list()) == {"strong"} + + def test_missing_columns_returns_zero(self, tmp_path): + regime_path(tmp_path).parent.mkdir(parents=True) + pl.DataFrame({"date": _days(3), "state": ["range"] * 3}).write_parquet(regime_path(tmp_path)) + assert refresh_phase_labels(tmp_path) == 0 diff --git a/backend/tests/test_matrix_field_columns_expansion.py b/backend/tests/test_matrix_field_columns_expansion.py new file mode 100644 index 0000000..ae41ebe --- /dev/null +++ b/backend/tests/test_matrix_field_columns_expansion.py @@ -0,0 +1,65 @@ +"""每日信号路径的字段依赖展开回归测试。 + +挖掘发布的 FactorRankResearchMatrixStrategy 把因子权重放在类级 SCORING, +META["scoring"] 为空。每日信号/实时矩阵路径通过 _matrix_field_columns 决定 +矩阵字段, 必须展开 required_fields_for_params 的虚拟因子依赖 +(limit_up_count_* -> consecutive_limit_ups), 否则 compute_signals 抛 +"MarketDataMatrix missing field: consecutive_limit_ups"。 +""" +from __future__ import annotations + +import types +from datetime import date, timedelta + +import polars as pl + +from app.backtest.matrix import build_market_data_matrix +from app.strategy.builtin.factor_rank_research import FactorRankResearchMatrixStrategy +from app.strategy.engine import StrategyEngine + + +def _mined_limit_up_strategy() -> types.SimpleNamespace: + strategy = FactorRankResearchMatrixStrategy( + {"amplitude": 2.0, "limit_up_count_60d": 1.0}, + {"amplitude": "low", "limit_up_count_60d": "low"}, + ) + return types.SimpleNamespace( + matrix_strategy=strategy, + basic_filter=None, + meta={"scoring": {}}, + ) + + +def test_matrix_field_columns_expand_parameter_scoring_dependencies(): + strategy = _mined_limit_up_strategy() + fields = StrategyEngine._matrix_field_columns(strategy, None, {}) + assert "consecutive_limit_ups" in fields + assert "amplitude" in fields + + +def test_mined_limit_up_strategy_signals_build_from_panel_fields(): + rows = [] + start = date(2024, 1, 1) + for offset in range(80): + close = 10.0 + offset * 0.04 + rows.append({ + "symbol": "000001.SZ", + "date": start + timedelta(days=offset), + "open": close - 0.05, + "high": close + 0.15, + "low": close - 0.15, + "close": close, + "volume": 1000.0 + offset * 5.0, + "amount": 100000.0, + "amplitude": 1.5, + "turnover_rate": 5.0, + "consecutive_limit_ups": (offset % 17) + 1 if offset % 17 == 0 else 0, + }) + panel = pl.DataFrame(rows) + strategy = _mined_limit_up_strategy() + market = build_market_data_matrix( + panel, + field_columns=StrategyEngine._matrix_field_columns(strategy, None, {}), + ) + signals = strategy.matrix_strategy.compute_signals(market, {}) + assert signals.shape == market.shape diff --git a/backend/tests/test_matrix_prewarm_owner.py b/backend/tests/test_matrix_prewarm_owner.py new file mode 100644 index 0000000..256f73c --- /dev/null +++ b/backend/tests/test_matrix_prewarm_owner.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import threading +import time + +from app.services.heavy_job_limiter import HeavyJobCancelledError, HeavyJobLimiter +from app.services.matrix_prewarm_owner import MatrixCachePrewarmOwner + + +def test_owner_deduplicates_running_work_and_reuses_after_completion() -> None: + owner = MatrixCachePrewarmOwner() + started = threading.Event() + release = threading.Event() + calls: list[str] = [] + + def target() -> None: + calls.append("run") + started.set() + assert release.wait(2) + + assert owner.schedule(target) + assert started.wait(1) + assert not owner.schedule(target) + + release.set() + deadline = time.monotonic() + 1 + while time.monotonic() < deadline and len(calls) < 2: + if owner.schedule(lambda: calls.append("run")): + break + time.sleep(0.005) + + assert calls == ["run", "run"] + assert owner.shutdown(timeout=1) + + +def test_shutdown_cancels_limiter_wait_and_rejects_new_work() -> None: + limiter = HeavyJobLimiter(capacity=2, cancel_poll_interval=0.005) + owner = MatrixCachePrewarmOwner() + waiting = threading.Event() + cancelled = threading.Event() + assert limiter.acquire("mining", timeout=0) + + def target() -> None: + waiting.set() + try: + with limiter.slot("normal", cancel_event=owner.cancel_event): + raise AssertionError("cancelled prewarm must not acquire capacity") + except HeavyJobCancelledError: + cancelled.set() + + assert owner.schedule(target) + assert waiting.wait(1) + assert owner.shutdown(timeout=1) + assert cancelled.is_set() + assert not owner.schedule(target) + limiter.release("mining") + + +def test_shutdown_join_is_bounded_for_uncooperative_target() -> None: + owner = MatrixCachePrewarmOwner() + release = threading.Event() + started = threading.Event() + + def target() -> None: + started.set() + release.wait(2) + + assert owner.schedule(target) + assert started.wait(1) + before = time.monotonic() + assert not owner.shutdown(timeout=0.02) + assert time.monotonic() - before < 0.2 + + release.set() + assert owner.shutdown(timeout=1) diff --git a/backend/tests/test_mining_api.py b/backend/tests/test_mining_api.py new file mode 100644 index 0000000..bd49847 --- /dev/null +++ b/backend/tests/test_mining_api.py @@ -0,0 +1,606 @@ +from __future__ import annotations + +import json +from datetime import date, timedelta +from pathlib import Path +from types import SimpleNamespace +from urllib.parse import quote + +import polars as pl +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.api.mining import router +from app.backtest.mining import compute_candidate_signature +from app.services.mining_jobs import MiningRunStore +from app.strategy.engine import StrategyEngine + +_FACTOR_DEFINITION = { + "kind": "factor_rank", + "factor_names": ["turnover_rate"], + "scoring": {"turnover_rate": 1.0}, + "directions": {"turnover_rate": "high"}, +} +_FACTOR_SIGNATURE = compute_candidate_signature(_FACTOR_DEFINITION) + + +class _Repo: + def __init__(self, data_dir) -> None: + self.store = SimpleNamespace(data_dir=data_dir) + + @staticmethod + def get_matrix_data_generation(asset_type="stock"): + return f"generation-{asset_type}" + + @staticmethod + def latest_enriched_date(asset_type="stock"): + del asset_type + return date(2026, 1, 9) + + @staticmethod + def get_instruments_asset(asset_type): + del asset_type + return pl.DataFrame({"symbol": ["000001.SZ"], "name": ["测试"]}) + + +class _Manager: + def __init__(self, data_dir) -> None: + self.store = MiningRunStore(data_dir) + + def start(self, request, fingerprint, force=False, source="manual", run_id=None): + del force + manifest = self.store.create(request, fingerprint, run_id=run_id) + self.store.append_event( + manifest["run_id"], + "queued", + {"status": "queued", "source": source}, + ) + return manifest + + def cancel(self, run_id): + manifest = self.store.get(run_id) + if manifest is None: + raise KeyError(run_id) + if manifest["status"] == "queued": + manifest = self.store.transition_status(run_id, "cancelled") + self.store.append_event(run_id, "cancelled", {"status": "cancelled"}) + return manifest + + +def _write_enriched_dates( + data_dir: Path, + count: int, + *, + asset_type: str = "stock", + first: date = date(2020, 1, 1), +) -> list[date]: + dirname = "kline_etf_enriched" if asset_type == "etf" else "kline_daily_enriched" + values = [first + timedelta(days=index) for index in range(count)] + for value in values: + partition = data_dir / dirname / f"date={value.isoformat()}" + partition.mkdir(parents=True, exist_ok=True) + (partition / "part.parquet").touch() + return values + + +def _client(tmp_path): + _write_enriched_dates(tmp_path, 219, first=date(2022, 8, 15)) + app = FastAPI() + app.include_router(router) + app.state.repo = _Repo(tmp_path) + app.state.mining_manager = _Manager(tmp_path) + app.state.strategy_engine = SimpleNamespace() + return TestClient(app), app.state.mining_manager.store + + +def _successful_run(store: MiningRunStore, run_id: str = "result-run"): + manifest = store.create( + { + "factor_names": ["turnover_rate"], + "strategy_ids": [], + "asset_type": "stock", + "budget_profile": "exploratory", + "correlation_threshold": 0.75, + }, + {"generation": "test"}, + run_id=run_id, + ) + store.append_event(run_id, "queued", {"status": "queued", "source": "manual"}) + store.transition_status(run_id, "running") + store.append_event(run_id, "running", {"status": "running"}) + + frames = { + "factors": pl.DataFrame({ + "factor_name": ["turnover_rate"], + "label": ["换手率"], + "direction": [1], + "score": [1.2], + "ic_mean": [0.1], + "ir": [0.8], + "coverage": [1.0], + "turnover": [0.2], + "spread_return": [None], + "spread_sharpe": [None], + "selected": [True], + "excluded_reason": [None], + }), + "correlation": pl.DataFrame({ + "factor_x": ["turnover_rate"], + "factor_y": ["turnover_rate"], + "rho": [1.0], + "pair_count": [3], + }), + "candidates": pl.DataFrame({ + "signature": [_FACTOR_SIGNATURE], + "name": ["因子组合 · 换手率"], + "kind": ["factor_combination"], + "factor_names_json": ['["turnover_rate"]'], + "strategy_id": [None], + "definition_json": [json.dumps(_FACTOR_DEFINITION, sort_keys=True)], + "regime_state": ["overall"], + "score": [0.9], + "oos_return": [0.08], + "oos_sharpe": [0.9], + "oos_max_drawdown": [-0.12], + "oos_positive_fold_ratio": [1.0], + "oos_n_trades": [70], + "confidence": ["standard"], + "valid_folds": [3], + "skipped_folds": [0], + "promoted_candidate_id": [None], + "published_strategy_id": [None], + }, schema_overrides={ + "strategy_id": pl.String, + "promoted_candidate_id": pl.String, + "published_strategy_id": pl.String, + }), + "folds": pl.DataFrame({ + "candidate_signature": [_FACTOR_SIGNATURE, _FACTOR_SIGNATURE], + "fold": [0, 0], + "label": ["OOS 1", "OOS 1"], + "regime_state": ["overall", "strong"], + "n_dates": [3, 2], + "train_start": ["2025-01-01", "2025-01-01"], + "train_end": ["2025-12-31", "2025-12-31"], + "test_start": ["2026-01-05", "2026-01-05"], + "test_end": ["2026-01-09", "2026-01-09"], + "selected_factors_json": ['["turnover_rate"]'] * 2, + "total_return": [0.08, 0.1], + "sharpe": [0.9, 1.1], + "max_drawdown": [-0.12, -0.08], + "n_trades": [70, 40], + "skipped": [False, False], + "reason": [None, None], + }), + } + for name, frame in frames.items(): + frame.write_parquet(store.artifact_path(run_id, name)) + store.register_artifact(run_id, name) + store.write_summary(run_id, { + "status": "succeeded", + "factor_count": 1, + "selected_factor_count": 1, + "candidate_count": 1, + "valid_fold_count": 1, + "skipped_fold_count": 0, + "confidence": "low", + "budget_exhausted": False, + "elapsed_ms": 123.4, + "data_as_of": "2026-01-09", + "methodology_version": "factor_v2", + "algorithm_version": "mining-v1", + "panel_scans": 1, + "matrix_bytes": 4096, + "phase_ms": {"panel": 1.0, "total": 123.4}, + "worker": {"peak_rss_bytes": 1000, "serialized_result_bytes": 2000}, + }) + store.transition_status(run_id, "succeeded") + store.append_event(run_id, "succeeded", {"status": "succeeded"}) + return manifest + + +def test_start_is_strict_and_projects_server_signature(tmp_path): + client, store = _client(tmp_path) + payload = { + "factor_names": ["turnover_rate"], + "budget_profile": "exploratory", + "force": False, + } + + response = client.post("/api/backtest/mining/runs", json=payload) + + assert response.status_code == 200 + body = response.json() + assert body["status"] == "queued" + assert body["signature"] == store.get(body["run_id"])["run_signature"] + assert body["request"]["factor_names"] == ["turnover_rate"] + assert "force" not in body["request"] + assert body["source"] == "manual" + assert client.post( + "/api/backtest/mining/runs", + json={**payload, "matching": "close_t"}, + ).status_code == 422 + assert client.post( + "/api/backtest/mining/runs", + json={"factor_names": ["unknown_factor"]}, + ).status_code == 422 + + +def test_start_accepts_iso_dates_and_persists_json_safe_request(tmp_path): + client, store = _client(tmp_path) + + response = client.post( + "/api/backtest/mining/runs", + json={ + "factor_names": ["turnover_rate"], + "budget_profile": "exploratory", + "start": "2022-08-15", + "end": "2026-08-15", + }, + ) + + assert response.status_code == 200 + run_id = response.json()["run_id"] + manifest = store.get(run_id) + assert manifest is not None + assert manifest["request"]["start"] == "2022-08-15" + assert manifest["request"]["end"] == "2026-08-15" + assert client.post( + "/api/backtest/mining/runs", + json={ + "factor_names": ["turnover_rate"], + "start": "2022/08/15", + }, + ).status_code == 422 + + +@pytest.mark.parametrize( + ("profile", "required_bars"), + [("exploratory", 219), ("balanced", 786), ("strict", 1164)], +) +def test_availability_enforces_exact_profile_boundaries( + tmp_path, + profile, + required_bars, +): + client, _store = _client(tmp_path) + dates = _write_enriched_dates( + tmp_path, + required_bars, + first=date(2015, 1, 1), + ) + + eligible = client.get( + "/api/backtest/mining/availability", + params={ + "asset_type": "stock", + "budget_profile": profile, + "start": dates[0].isoformat(), + "end": dates[-1].isoformat(), + }, + ) + insufficient = client.get( + "/api/backtest/mining/availability", + params={ + "asset_type": "stock", + "budget_profile": profile, + "start": dates[1].isoformat(), + "end": dates[-1].isoformat(), + }, + ) + + assert eligible.status_code == 200 + assert eligible.json() == { + "asset_type": "stock", + "budget_profile": profile, + "trading_bars": required_bars, + "required_bars": required_bars, + "outer_folds": 1 if profile == "exploratory" else 3, + "required_outer_folds": 1 if profile == "exploratory" else 3, + "eligible": True, + "available_start": dates[0].isoformat(), + "available_end": "2023-03-21", + "effective_start": dates[0].isoformat(), + "effective_end": dates[-1].isoformat(), + "suggested_start": dates[0].isoformat(), + } + assert insufficient.status_code == 200 + assert insufficient.json()["trading_bars"] == required_bars - 1 + assert insufficient.json()["eligible"] is False + assert insufficient.json()["suggested_start"] == dates[0].isoformat() + + +def test_start_rejects_balanced_625_bar_range_before_creating_run(tmp_path): + client, store = _client(tmp_path) + dates = _write_enriched_dates(tmp_path, 625, first=date(2024, 1, 15)) + + response = client.post( + "/api/backtest/mining/runs", + json={ + "factor_names": ["turnover_rate"], + "budget_profile": "balanced", + "start": dates[0].isoformat(), + "end": dates[-1].isoformat(), + "force": True, + }, + ) + + assert response.status_code == 400 + assert response.json()["detail"] == ( + "balanced mining requires at least 786 enriched trading bars for 3 outer " + "folds; effective range 2024-01-15 to 2025-09-30 has 625" + ) + assert store.list_runs() == [] + + +def test_availability_uses_asset_specific_valid_partitions(tmp_path): + client, _store = _client(tmp_path) + etf_dates = _write_enriched_dates( + tmp_path, + 219, + asset_type="etf", + first=date(2024, 1, 1), + ) + malformed = tmp_path / "kline_etf_enriched" / "date=not-a-date" + malformed.mkdir(parents=True) + (malformed / "part.parquet").touch() + (tmp_path / "kline_etf_enriched" / "date=2023-12-31").mkdir(parents=True) + + response = client.get( + "/api/backtest/mining/availability", + params={ + "asset_type": "etf", + "budget_profile": "exploratory", + "start": etf_dates[0].isoformat(), + "end": etf_dates[-1].isoformat(), + }, + ) + + assert response.status_code == 200 + assert response.json()["trading_bars"] == 219 + assert response.json()["eligible"] is True + + +def test_result_reconstructs_artifacts_without_exposing_definition(tmp_path): + client, store = _client(tmp_path) + _successful_run(store) + + response = client.get("/api/backtest/mining/runs/result-run/result") + + assert response.status_code == 200 + body = response.json() + assert body["summary"]["peak_rss_bytes"] == 1000 + assert body["correlation"]["matrix"] == [[1.0]] + assert body["candidates"][0]["factor_names"] == ["turnover_rate"] + assert "definition_json" not in body["candidates"][0] + assert body["candidates"][0]["folds"][0]["selected_factors"] == ["turnover_rate"] + assert body["candidates"][0]["gate"] == {"qualified": True, "reasons": []} + assert body["request_summary"] == { + "asset_type": "stock", + "budget_profile": "exploratory", + "start": None, + "end": None, + "factor_count": 1, + "strategy_count": 0, + "commission_pct": None, + "stamp_tax_pct": None, + "slippage_bps": None, + "correlation_threshold": 0.75, + } + regimes = {row["state"]: row for row in body["regimes"]} + assert regimes["overall"]["n_dates"] == 3 + assert regimes["strong"]["n_dates"] == 2 + assert regimes["range"]["total_return"] is None + assert body["telemetry"]["panel_scans"] == 1 + + +def test_result_marks_legacy_fold_rows_without_evaluation_kind(tmp_path): + client, store = _client(tmp_path) + _successful_run(store) + + response = client.get("/api/backtest/mining/runs/result-run/result") + + assert response.status_code == 200 + fold = response.json()["candidates"][0]["folds"][0] + assert fold["evaluation_kind"] is None + + +def test_publish_endpoint_rejects_candidate_below_evidence_gate(tmp_path): + client, store = _client(tmp_path) + _successful_run(store) + path = store.artifact_path("result-run", "candidates") + pl.read_parquet(path).with_columns( + pl.lit("low").alias("confidence"), + ).write_parquet(path) + + response = client.post( + f"/api/backtest/mining/runs/result-run/candidates/" + f"{quote(_FACTOR_SIGNATURE, safe='')}/publish", + ) + + assert response.status_code == 400 + assert "exploratory results can only be saved" in response.json()["detail"] + + +def test_promote_endpoint_uses_persisted_definition_only(tmp_path): + client, store = _client(tmp_path) + _successful_run(store) + + response = client.post( + f"/api/backtest/mining/runs/result-run/candidates/" + f"{quote(_FACTOR_SIGNATURE, safe='')}/promote" + ) + + assert response.status_code == 200 + body = response.json() + assert body["status"] == "pending" + assert body["config"]["origin_run_id"] == "result-run" + assert body["config"]["candidate_signature"] == _FACTOR_SIGNATURE + assert body["config"]["factor_names"] == ["turnover_rate"] + assert "definition_json" not in body + persisted = pl.read_parquet(store.artifact_path("result-run", "candidates")) + assert persisted["promoted_candidate_id"][0] == body["id"] + + +def test_publish_endpoint_uses_persisted_definition_and_invalidates_runtime( + tmp_path, +) -> None: + client, store = _client(tmp_path) + _successful_run(store) + builtin_dir = Path(__file__).resolve().parents[1] / "app" / "strategy" / "builtin" + custom_dir = tmp_path / "strategies" / "custom" + engine = StrategyEngine(strategy_dirs=[builtin_dir, custom_dir]) + invalidations: list[str] = [] + client.app.state.strategy_engine = engine + client.app.state.monitor_engine = SimpleNamespace( + invalidate_strategy_state=lambda: invalidations.append("monitor") + ) + + response = client.post( + f"/api/backtest/mining/runs/result-run/candidates/" + f"{quote(_FACTOR_SIGNATURE, safe='')}/publish", + json={ + "strategy_id": "caller-selected", + "scoring": {"rsi_14": 999.0}, + }, + ) + + assert response.status_code == 200 + strategy_id = response.json()["strategy_id"] + strategy = engine.get(strategy_id) + assert strategy.meta["origin_run_id"] == "result-run" + assert strategy.meta["candidate_signature"] == _FACTOR_SIGNATURE + assert strategy.matrix_strategy._scoring == {"turnover_rate": 1.0} + assert strategy.matrix_strategy._directions == {"turnover_rate": "high"} + assert invalidations == ["monitor"] + persisted = pl.read_parquet(store.artifact_path("result-run", "candidates")) + assert persisted["published_strategy_id"][0] == strategy_id + + +def test_result_fails_closed_when_artifact_is_missing(tmp_path): + client, store = _client(tmp_path) + _successful_run(store) + store.artifact_path("result-run", "folds").unlink() + + response = client.get("/api/backtest/mining/runs/result-run/result") + + assert response.status_code == 500 + assert response.json()["detail"] == "mining result artifacts are unavailable" + + +def test_sse_maps_failed_event_and_honors_last_event_id(tmp_path): + client, store = _client(tmp_path) + store.create( + {"factor_names": ["turnover_rate"]}, + {"generation": "test"}, + run_id="failed-run", + ) + queued = store.append_event( + "failed-run", "queued", {"status": "queued", "source": "manual"} + ) + store.transition_status("failed-run", "failed", error="worker failed") + failed = store.append_event( + "failed-run", "error", {"status": "failed", "message": "worker failed"} + ) + + response = client.get( + "/api/backtest/mining/runs/failed-run/events", + headers={"Last-Event-ID": str(queued["id"])}, + ) + + assert response.status_code == 200 + assert f"id: {failed['id']}" in response.text + assert "event: failed" in response.text + assert "event: error" not in response.text + assert "worker failed" in response.text + + +def test_sse_recovers_progress_snapshot_when_history_is_truncated(tmp_path): + client, store = _client(tmp_path) + store.create( + {"factor_names": ["turnover_rate"]}, + {"generation": "test"}, + run_id="truncated-run", + ) + store.write_summary( + "truncated-run", + {"progress": {"phase": "search", "done": 7, "total": 10}}, + ) + for index in range(260): + store.append_event( + "truncated-run", + "progress", + {"phase": "search", "done": index, "total": 260}, + ) + store.transition_status("truncated-run", "failed", error="worker failed") + store.append_event( + "truncated-run", + "error", + {"status": "failed", "message": "worker failed"}, + ) + + response = client.get("/api/backtest/mining/runs/truncated-run/events") + + assert response.status_code == 200 + assert '"done": 7' in response.text + assert "event: failed" in response.text + + +def test_start_rejects_incompatible_strategy_before_creating_run(tmp_path): + client, store = _client(tmp_path) + client.app.state.strategy_engine = SimpleNamespace( + get=lambda _strategy_id: SimpleNamespace( + meta={ + "research_only": True, + "asset_types": ["stock"], + "timeframes": ["1d"], + }, + execution_backend="matrix_native", + ) + ) + + response = client.post( + "/api/backtest/mining/runs", + json={ + "factor_names": ["turnover_rate"], + "strategy_ids": ["factor_rank_research"], + "budget_profile": "exploratory", + }, + ) + + assert response.status_code == 400 + assert store.list_runs() == [] + + +def test_config_patch_merges_current_values(tmp_path, monkeypatch): + client, _store = _client(tmp_path) + current = { + "mining_schedule_enabled": False, + "mining_schedule_weekday": 4, + "mining_budget_profile": "balanced", + } + saved = [] + monkeypatch.setattr( + "app.api.mining.preferences.get_mining_schedule", + lambda: dict(current), + ) + + def set_schedule(enabled, weekday, profile): + saved.append((enabled, weekday, profile)) + return { + "mining_schedule_enabled": enabled, + "mining_schedule_weekday": weekday, + "mining_budget_profile": profile, + } + + monkeypatch.setattr("app.api.mining.preferences.set_mining_schedule", set_schedule) + + response = client.patch( + "/api/backtest/mining/config", + json={"mining_schedule_enabled": True}, + ) + + assert response.status_code == 200 + assert saved == [(True, 4, "balanced")] + assert client.patch("/api/backtest/mining/config", json={}).status_code == 400 diff --git a/backend/tests/test_mining_candidates.py b/backend/tests/test_mining_candidates.py new file mode 100644 index 0000000..78c2d28 --- /dev/null +++ b/backend/tests/test_mining_candidates.py @@ -0,0 +1,679 @@ +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace + +import polars as pl +import pytest + +from app.backtest.candidates import CandidateStore, CandidateValidationError +from app.backtest.mining import compute_candidate_signature +from app.services.mining_candidates import ( + MiningCandidateService, + _published_strategy_id, +) +from app.services.mining_jobs import MiningRunStore +from app.strategy import config as strategy_config +from app.strategy.engine import StrategyEngine + + +class _StrategyEngine: + def __init__(self) -> None: + self.strategies = { + "existing_daily": SimpleNamespace( + id="existing_daily", + execution_backend="matrix_native", + meta={ + "id": "existing_daily", + "research_only": False, + "timeframes": ["1d"], + "asset_types": ["stock"], + }, + ) + } + + def get(self, strategy_id: str): + if strategy_id not in self.strategies: + raise KeyError(strategy_id) + return self.strategies[strategy_id] + + def list_strategies(self): + return [strategy.meta for strategy in self.strategies.values()] + + +def _factor_definition() -> dict: + return { + "kind": "factor_rank", + "factor_names": ["turnover_rate", "rsi_14"], + "scoring": {"turnover_rate": 1.0, "rsi_14": 2.0}, + "directions": {"turnover_rate": "high", "rsi_14": "low"}, + } + + +def _create_run( + tmp_path, + *, + definition: dict | None = None, + status: str = "succeeded", + run_id: str = "mining-run", +) -> tuple[MiningRunStore, str, str]: + definition = definition or _factor_definition() + signature = compute_candidate_signature(definition) + store = MiningRunStore(tmp_path) + store.create( + { + "factor_names": ["turnover_rate", "rsi_14"], + "strategy_ids": ["existing_daily", "ma_golden_cross"], + "asset_type": "stock", + "start": "2025-01-01", + "end": "2026-01-09", + "budget_profile": "exploratory", + "commission_pct": 0.0002, + "stamp_tax_pct": 0.0005, + "slippage_bps": 5.0, + }, + {"generation": "test"}, + run_id=run_id, + ) + frame = pl.DataFrame({ + "signature": [signature], + "name": ["因子组合候选"], + "kind": [ + "existing_strategy" + if definition["kind"] == "existing_strategy" + else "factor_combination" + ], + "factor_names_json": [json.dumps(definition.get("factor_names", []))], + "strategy_id": [definition.get("strategy_id")], + "definition_json": [json.dumps(definition, sort_keys=True)], + "regime_state": ["overall"], + "score": [0.9], + "oos_return": [0.08], + "oos_sharpe": [0.9], + "oos_max_drawdown": [-0.12], + "oos_positive_fold_ratio": [1.0], + "oos_n_trades": [70], + "confidence": ["standard"], + "valid_folds": [3], + "skipped_folds": [0], + "promoted_candidate_id": [None], + "published_strategy_id": [None], + }, schema_overrides={ + "strategy_id": pl.String, + "promoted_candidate_id": pl.String, + "published_strategy_id": pl.String, + }) + frame.write_parquet(store.artifact_path(run_id, "candidates")) + store.register_artifact(run_id, "candidates") + store.write_summary(run_id, { + "data_as_of": "2026-01-09", + "algorithm_version": "mining-v1", + "methodology_version": "factor_v2", + }) + if status != "queued": + store.transition_status(run_id, "running") + store.transition_status(run_id, status) + return store, run_id, signature + + +def _service(tmp_path, store: MiningRunStore) -> MiningCandidateService: + return MiningCandidateService( + tmp_path, + store, + CandidateStore(tmp_path), + _StrategyEngine(), + strategy_cache_invalidator=lambda _data_dir: None, + ) + + +def _real_service( + tmp_path, + store: MiningRunStore, + *, + cache_invalidator=None, + monitor_invalidator=None, +) -> tuple[MiningCandidateService, StrategyEngine]: + builtin_dir = Path(__file__).resolve().parents[1] / "app" / "strategy" / "builtin" + custom_dir = tmp_path / "strategies" / "custom" + engine = StrategyEngine(strategy_dirs=[builtin_dir, custom_dir]) + service = MiningCandidateService( + tmp_path, + store, + CandidateStore(tmp_path), + engine, + strategy_cache_invalidator=cache_invalidator or (lambda _data_dir: None), + monitor_state_invalidator=monitor_invalidator, + ) + return service, engine + + +def test_promote_rereads_artifact_and_repairs_backlink_idempotently(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + service = _service(tmp_path, store) + + first = service.promote(run_id, signature) + second = service.promote(run_id, signature) + + assert second == first + assert first["kind"] == "strategy" + assert first["source_id"].startswith("mined_factor_") + assert first["status"] == "pending" + assert first["config"]["origin_run_id"] == run_id + assert first["config"]["candidate_signature"] == signature + assert first["config"]["factor_names"] == ["turnover_rate", "rsi_14"] + assert first["config"]["directions"] == ["high", "low"] + assert first["config"]["weights"] == [1.0, 2.0] + assert first["metrics"]["oos_sharpe"] == pytest.approx(0.9) + persisted = pl.read_parquet(store.artifact_path(run_id, "candidates")).row(0, named=True) + assert persisted["promoted_candidate_id"] == first["id"] + assert len(CandidateStore(tmp_path).list()) == 1 + + +def test_promote_repairs_backlink_after_partial_failure(tmp_path, monkeypatch) -> None: + store, run_id, signature = _create_run(tmp_path) + service = _service(tmp_path, store) + original = service._write_backlink + calls = 0 + + def fail_once(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError("injected backlink failure") + return original(*args, **kwargs) + + monkeypatch.setattr(service, "_write_backlink", fail_once) + with pytest.raises(RuntimeError, match="injected"): + service.promote(run_id, signature) + + created = CandidateStore(tmp_path).list() + assert len(created) == 1 + recovered = service.promote(run_id, signature) + assert recovered["id"] == created[0]["id"] + persisted = pl.read_parquet(store.artifact_path(run_id, "candidates")).row(0, named=True) + assert persisted["promoted_candidate_id"] == created[0]["id"] + assert len(CandidateStore(tmp_path).list()) == 1 + + +def test_promote_rejects_non_successful_run(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path, status="queued") + + with pytest.raises(ValueError, match="successful"): + _service(tmp_path, store).promote(run_id, signature) + + assert CandidateStore(tmp_path).list() == [] + + +def test_promote_rejects_tampered_definition_before_candidate_write(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + path = store.artifact_path(run_id, "candidates") + frame = pl.read_parquet(path).with_columns( + pl.lit(json.dumps({ + "kind": "factor_rank", + "factor_names": ["turnover_rate"], + "scoring": {"turnover_rate": 1.0}, + "directions": {"turnover_rate": "high"}, + })).alias("definition_json") + ) + frame.write_parquet(path) + + with pytest.raises(ValueError, match=r"signature|definition"): + _service(tmp_path, store).promote(run_id, signature) + + assert CandidateStore(tmp_path).list() == [] + + +def test_promote_rejects_factor_not_selected_in_origin_request(tmp_path) -> None: + definition = { + "kind": "factor_rank", + "factor_names": ["momentum_20d"], + "scoring": {"momentum_20d": 1.0}, + "directions": {"momentum_20d": "high"}, + } + store, run_id, signature = _create_run(tmp_path, definition=definition) + + with pytest.raises(ValueError, match="origin request"): + _service(tmp_path, store).promote(run_id, signature) + + +def test_promote_existing_strategy_revalidates_current_engine_contract(tmp_path) -> None: + definition = {"kind": "existing_strategy", "strategy_id": "existing_daily"} + store, run_id, signature = _create_run(tmp_path, definition=definition) + + promoted = _service(tmp_path, store).promote(run_id, signature) + + assert promoted["source_id"] == "existing_daily" + assert promoted["config"]["strategy_id"] == "existing_daily" + + +def test_publish_existing_strategy_returns_verified_id_and_repairs_backlink(tmp_path) -> None: + definition = {"kind": "existing_strategy", "strategy_id": "existing_daily"} + store, run_id, signature = _create_run(tmp_path, definition=definition) + service = _service(tmp_path, store) + + first = service.publish(run_id, signature) + second = service.publish(run_id, signature) + with pytest.raises(TypeError): + service.publish(run_id, signature, "existing_daily") + + assert first == second == {"ok": True, "strategy_id": "existing_daily"} + persisted = pl.read_parquet(store.artifact_path(run_id, "candidates")).row(0, named=True) + assert persisted["published_strategy_id"] == "existing_daily" + assert CandidateStore(tmp_path).list() == [] + + +def test_promote_requires_unique_artifact_signature(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + path = store.artifact_path(run_id, "candidates") + frame = pl.read_parquet(path) + pl.concat([frame, frame]).write_parquet(path) + + with pytest.raises(ValueError, match="duplicate"): + _service(tmp_path, store).promote(run_id, signature) + + +def test_promote_concurrent_calls_create_one_authoritative_candidate(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + service = _service(tmp_path, store) + + with ThreadPoolExecutor(max_workers=8) as executor: + items = list(executor.map( + lambda _index: service.promote(run_id, signature), + range(16), + )) + + assert len({item["id"] for item in items}) == 1 + assert len(CandidateStore(tmp_path).list()) == 1 + + +def test_promote_rejects_artifact_conflicting_with_existing_store_record( + tmp_path, +) -> None: + store, run_id, signature = _create_run(tmp_path) + service = _service(tmp_path, store) + original = service.promote(run_id, signature) + path = store.artifact_path(run_id, "candidates") + pl.read_parquet(path).with_columns( + pl.lit(1.1).alias("oos_sharpe"), + pl.lit(None, dtype=pl.String).alias("promoted_candidate_id"), + ).write_parquet(path) + + with pytest.raises(CandidateValidationError, match="冲突"): + service.promote(run_id, signature) + + persisted = pl.read_parquet(path).row(0, named=True) + assert persisted["promoted_candidate_id"] is None + stored = CandidateStore(tmp_path).list()[0] + assert stored == original + assert stored["metrics"]["oos_sharpe"] == pytest.approx(0.9) + + +@pytest.mark.parametrize( + ("definition", "message"), + [ + ( + { + "kind": "factor_rank", + "factor_names": ["turnover_rate"], + "scoring": {"turnover_rate": 0.0}, + "directions": {"turnover_rate": "high"}, + }, + "positive", + ), + ( + { + "kind": "factor_rank", + "factor_names": ["turnover_rate"], + "scoring": {"turnover_rate": float("inf")}, + "directions": {"turnover_rate": "high"}, + }, + "finite", + ), + ( + { + "kind": "factor_rank", + "factor_names": ["turnover_rate"], + "scoring": {"turnover_rate": 1.0}, + "directions": {"turnover_rate": "sideways"}, + }, + "directions", + ), + ( + { + "kind": "factor_rank", + "factor_names": ["not_a_factor"], + "scoring": {"not_a_factor": 1.0}, + "directions": {"not_a_factor": "high"}, + }, + "unknown factors", + ), + ( + { + "kind": "factor_rank", + "factor_names": [ + "momentum_5d", + "momentum_10d", + "momentum_20d", + "momentum_30d", + "momentum_60d", + ], + "scoring": { + "momentum_5d": 1.0, + "momentum_10d": 1.0, + "momentum_20d": 1.0, + "momentum_30d": 1.0, + "momentum_60d": 1.0, + }, + "directions": { + "momentum_5d": "high", + "momentum_10d": "high", + "momentum_20d": "high", + "momentum_30d": "high", + "momentum_60d": "high", + }, + }, + "1 to 4", + ), + ], +) +def test_promote_rejects_invalid_factor_contract(tmp_path, definition, message) -> None: + store, run_id, signature = _create_run(tmp_path) + path = store.artifact_path(run_id, "candidates") + frame = pl.read_parquet(path).with_columns( + pl.lit(signature).alias("signature"), + pl.lit(json.dumps(definition)).alias("definition_json"), + pl.lit(json.dumps(definition["factor_names"])).alias("factor_names_json"), + ) + frame.write_parquet(path) + + with pytest.raises(ValueError, match=message): + _service(tmp_path, store).promote(run_id, signature) + + +@pytest.mark.parametrize("column", ["oos_sharpe", "oos_return", "score"]) +def test_promote_rejects_nonfinite_artifact_values(tmp_path, column) -> None: + store, run_id, signature = _create_run(tmp_path) + path = store.artifact_path(run_id, "candidates") + pl.read_parquet(path).with_columns(pl.lit(float("nan")).alias(column)).write_parquet(path) + + with pytest.raises(ValueError, match="finite"): + _service(tmp_path, store).promote(run_id, signature) + + +def test_promote_rejects_corrupt_or_missing_schema_artifact(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + path = store.artifact_path(run_id, "candidates") + path.write_bytes(b"not parquet") + with pytest.raises(RuntimeError, match="failed to read"): + _service(tmp_path, store).promote(run_id, signature) + + path.unlink() + pl.DataFrame({"signature": [signature]}).write_parquet(path) + with pytest.raises(ValueError, match="schema"): + _service(tmp_path, store).promote(run_id, signature) + + +def test_publish_factor_discovers_public_strategy_and_repairs_runtime_state(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + invalidations: list[object] = [] + service, engine = _real_service( + tmp_path, + store, + cache_invalidator=lambda data_dir: invalidations.append(data_dir), + monitor_invalidator=lambda: invalidations.append("monitor"), + ) + strategy_config.save_override( + tmp_path, + "factor_rank_research", + {"params": {"entry_score": 99.0}}, + ) + + result = service.publish(run_id, signature) + repeated = service.publish(run_id, signature) + + assert repeated == result + strategy = engine.get(result["strategy_id"]) + assert strategy.source == "custom" + assert strategy.execution_backend == "matrix_native" + assert strategy.meta["origin_run_id"] == run_id + assert strategy.meta["candidate_signature"] == signature + assert strategy.meta["research_only"] is False + assert strategy.meta["asset_types"] == ["stock"] + assert strategy.matrix_strategy._scoring == {"turnover_rate": 1.0, "rsi_14": 2.0} + assert strategy.matrix_strategy._directions == { + "turnover_rate": "high", + "rsi_14": "low", + } + assert result["strategy_id"] in { + meta["id"] for meta in engine.list_strategies() + } + assert strategy_config.load_override( + tmp_path, "factor_rank_research" + ) == {"params": {"entry_score": 99.0}} + assert not (tmp_path / "user_data" / "strategy_overrides" / f"{result['strategy_id']}.json").exists() + persisted = pl.read_parquet(store.artifact_path(run_id, "candidates")).row(0, named=True) + assert persisted["published_strategy_id"] == result["strategy_id"] + assert invalidations == [tmp_path, "monitor", tmp_path, "monitor"] + + +def test_publish_factor_repairs_backlink_after_source_was_verified( + tmp_path, + monkeypatch, +) -> None: + store, run_id, signature = _create_run(tmp_path) + service, _engine = _real_service(tmp_path, store) + original = service._write_backlink + calls = 0 + + def fail_once(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError("injected publication backlink failure") + return original(*args, **kwargs) + + monkeypatch.setattr(service, "_write_backlink", fail_once) + with pytest.raises(RuntimeError, match="injected publication"): + service.publish(run_id, signature) + + result = service.publish(run_id, signature) + strategy_path = tmp_path / "strategies" / "custom" / f"{result['strategy_id']}.py" + assert strategy_path.is_file() + persisted = pl.read_parquet(store.artifact_path(run_id, "candidates")).row(0, named=True) + assert persisted["published_strategy_id"] == result["strategy_id"] + + +def test_publish_factor_uses_server_run_scoped_id_and_refuses_collision(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + service, _engine = _real_service(tmp_path, store) + expected_id = _published_strategy_id(run_id, signature) + + with pytest.raises(TypeError): + service.publish(run_id, signature, "mined_factor_caller_selected") + + target = tmp_path / "strategies" / "custom" / f"{expected_id}.py" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("user owned", encoding="utf-8") + with pytest.raises(ValueError, match="collision"): + service.publish(run_id, signature) + assert target.read_text(encoding="utf-8") == "user owned" + + +def test_publish_factor_same_signature_in_different_runs_has_independent_ids( + tmp_path, +) -> None: + store, first_run_id, signature = _create_run(tmp_path, run_id="mining-run-one") + _other_store, second_run_id, second_signature = _create_run( + tmp_path, + run_id="mining-run-two", + ) + service, engine = _real_service(tmp_path, store) + + first = service.publish(first_run_id, signature) + second = service.publish(second_run_id, second_signature) + + assert signature == second_signature + assert first["strategy_id"] == _published_strategy_id(first_run_id, signature) + assert second["strategy_id"] == _published_strategy_id( + second_run_id, + second_signature, + ) + assert first["strategy_id"] != second["strategy_id"] + assert engine.has(first["strategy_id"]) + assert engine.has(second["strategy_id"]) + + +def test_publish_factor_rejects_inconsistent_backlink(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + path = store.artifact_path(run_id, "candidates") + pl.read_parquet(path).with_columns( + pl.lit("mined_factor_other").alias("published_strategy_id") + ).write_parquet(path) + service, engine = _real_service(tmp_path, store) + + with pytest.raises(ValueError, match="backlink is inconsistent"): + service.publish(run_id, signature) + + expected_id = _published_strategy_id(run_id, signature) + assert not engine.has(expected_id) + assert not (tmp_path / "strategies" / "custom" / f"{expected_id}.py").exists() + + +def test_publish_factor_rolls_back_file_and_skips_invalidations_on_reload_failure( + tmp_path, + monkeypatch, +) -> None: + store, run_id, signature = _create_run(tmp_path) + invalidations: list[str] = [] + service, engine = _real_service( + tmp_path, + store, + cache_invalidator=lambda _data_dir: invalidations.append("cache"), + monitor_invalidator=lambda: invalidations.append("monitor"), + ) + real_reload = engine.reload + calls = 0 + + def fail_once() -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise ValueError("injected reload failure") + real_reload() + + monkeypatch.setattr(engine, "reload", fail_once) + expected_id = _published_strategy_id(run_id, signature) + with pytest.raises(RuntimeError, match="injected reload failure"): + service.publish(run_id, signature) + + assert not (tmp_path / "strategies" / "custom" / f"{expected_id}.py").exists() + assert not engine.has(expected_id) + assert invalidations == [] + persisted = pl.read_parquet(store.artifact_path(run_id, "candidates")).row(0, named=True) + assert persisted["published_strategy_id"] is None + + +def test_publish_factor_rolls_back_on_runtime_invalidation_failure(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + invalidations: list[str] = [] + + def fail_monitor() -> None: + invalidations.append("monitor") + raise ValueError("injected monitor invalidation failure") + + service, engine = _real_service( + tmp_path, + store, + cache_invalidator=lambda _data_dir: invalidations.append("cache"), + monitor_invalidator=fail_monitor, + ) + expected_id = _published_strategy_id(run_id, signature) + + with pytest.raises(RuntimeError, match="injected monitor invalidation failure"): + service.publish(run_id, signature) + + assert invalidations == ["cache", "monitor"] + assert not (tmp_path / "strategies" / "custom" / f"{expected_id}.py").exists() + assert not engine.has(expected_id) + persisted = pl.read_parquet(store.artifact_path(run_id, "candidates")).row(0, named=True) + assert persisted["published_strategy_id"] is None + + +def test_publish_existing_does_not_reload_or_invalidate(tmp_path) -> None: + definition = {"kind": "existing_strategy", "strategy_id": "ma_golden_cross"} + store, run_id, signature = _create_run(tmp_path, definition=definition) + invalidations: list[str] = [] + service, engine = _real_service( + tmp_path, + store, + cache_invalidator=lambda _data_dir: invalidations.append("cache"), + monitor_invalidator=lambda: invalidations.append("monitor"), + ) + original = engine.get("ma_golden_cross") + + result = service.publish(run_id, signature) + + assert result == {"ok": True, "strategy_id": "ma_golden_cross"} + assert engine.get("ma_golden_cross") is original + assert invalidations == [] + +def _rewrite_candidate_metric( + store: MiningRunStore, + run_id: str, + column: str, + value, +) -> None: + path = store.artifact_path(run_id, "candidates") + frame = pl.read_parquet(path) + frame.with_columns( + pl.lit(value, dtype=frame.schema[column]).alias(column), + ).write_parquet(path) + + +def test_publish_rejects_low_confidence_exploratory_candidate(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + _rewrite_candidate_metric(store, run_id, "confidence", "low") + service = _service(tmp_path, store) + + with pytest.raises(ValueError, match="exploratory results can only be saved"): + service.publish(run_id, signature) + + +def test_publish_rejects_candidate_below_evidence_gate(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + _rewrite_candidate_metric(store, run_id, "oos_sharpe", 0.2) + service = _service(tmp_path, store) + + with pytest.raises(ValueError, match=r"OOS Sharpe of at least 0.5"): + service.publish(run_id, signature) + + +def test_publish_rejects_candidate_with_insufficient_folds(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + _rewrite_candidate_metric(store, run_id, "valid_folds", 1) + service = _service(tmp_path, store) + + with pytest.raises(ValueError, match="at least 2 valid outer folds"): + service.publish(run_id, signature) + + +def test_publish_rejects_candidate_missing_required_metrics(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + _rewrite_candidate_metric(store, run_id, "oos_n_trades", None) + service = _service(tmp_path, store) + + with pytest.raises(ValueError, match="does not meet the promotion gate"): + service.publish(run_id, signature) + + +def test_promote_still_allowed_for_gated_candidates(tmp_path) -> None: + store, run_id, signature = _create_run(tmp_path) + _rewrite_candidate_metric(store, run_id, "confidence", "low") + _rewrite_candidate_metric(store, run_id, "oos_sharpe", 0.1) + service = _service(tmp_path, store) + + result = service.promote(run_id, signature) + + assert result["status"] == "pending" diff --git a/backend/tests/test_mining_jobs.py b/backend/tests/test_mining_jobs.py new file mode 100644 index 0000000..9263bff --- /dev/null +++ b/backend/tests/test_mining_jobs.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import json +import threading +from datetime import date +from pathlib import Path + +import pytest + +from app.services.mining_jobs import ( + ACTIVE_RUN_STATUSES, + MAX_EVENT_PAYLOAD_BYTES, + SUCCESS_RUN_STATUSES, + InvalidMiningStatusTransitionError, + MiningRunStore, + MiningRunValidationError, + canonicalize_request, + compute_run_signature, +) + + +def _store(tmp_path: Path) -> MiningRunStore: + return MiningRunStore(tmp_path) + + +def test_create_and_atomic_summary_reads(tmp_path: Path) -> None: + store = _store(tmp_path) + manifest = store.create( + {"symbols": ["000001.SZ"], "start": date(2025, 1, 1)}, + {"daily_generation": 7}, + run_id="run_atomic", + ) + run_dir = tmp_path / "research" / "mining" / "runs" / "run_atomic" + + assert manifest["status"] == "queued" + assert manifest["request"]["start"] == "2025-01-01" + assert json.loads((run_dir / "manifest.json").read_text(encoding="utf-8")) == manifest + assert store.read_summary("run_atomic") == {} + + failures: list[Exception] = [] + stop = threading.Event() + + def reader() -> None: + while not stop.is_set(): + try: + value = store.read_summary("run_atomic") + assert isinstance(value.get("iteration"), int) + except Exception as exc: + failures.append(exc) + return + + store.write_summary("run_atomic", {"iteration": -1}) + thread = threading.Thread(target=reader) + thread.start() + for iteration in range(50): + store.write_summary("run_atomic", {"iteration": iteration}) + stop.set() + thread.join(timeout=1) + + assert failures == [] + assert store.read_summary("run_atomic") == {"iteration": 49} + assert not list(run_dir.glob("*.tmp")) + assert not list(run_dir.glob(".*.tmp")) + + +def test_signature_is_canonical_and_covers_request_dimensions() -> None: + first = { + "symbols": ["000001.SZ", "600000.SH"], + "window": {"start": "2025-01-01", "end": "2025-06-30"}, + "budget": 100, + } + reordered = { + "budget": 100, + "window": {"end": "2025-06-30", "start": "2025-01-01"}, + "symbols": ["000001.SZ", "600000.SH"], + } + + assert canonicalize_request(first) == canonicalize_request(reordered) + signature = compute_run_signature(first, {"daily": "v7", "enriched": "v2"}) + assert signature == compute_run_signature(reordered, {"enriched": "v2", "daily": "v7"}) + assert len(signature) == 64 + assert signature != compute_run_signature( + {**first, "budget": 101}, {"daily": "v7", "enriched": "v2"} + ) + assert signature != compute_run_signature(first, {"daily": "v8", "enriched": "v2"}) + assert signature != compute_run_signature( + {**first, "symbols": list(reversed(first["symbols"]))}, + {"daily": "v7", "enriched": "v2"}, + ) + + +def test_events_are_bounded_monotonic_and_support_after_id(tmp_path: Path) -> None: + store = _store(tmp_path) + store.create({}, "data-v1", run_id="event_run") + + for index in range(300): + event = store.append_event("event_run", "progress", {"step": index}) + assert event["id"] == index + 1 + + retained = store.read_events("event_run") + assert len(retained) == 256 + assert [event["id"] for event in retained] == list(range(45, 301)) + assert [event["id"] for event in store.read_events("event_run", after_id=295)] == list( + range(296, 301) + ) + + with pytest.raises(MiningRunValidationError, match="payload exceeds"): + store.append_event( + "event_run", + "result", + {"large_result": "x" * (MAX_EVENT_PAYLOAD_BYTES + 1)}, + ) + + +def test_status_transitions_summary_and_artifact_registration(tmp_path: Path) -> None: + store = _store(tmp_path) + store.create({"budget": 10}, "data-v1", run_id="lifecycle") + + running = store.transition_status("lifecycle", "running") + assert running["started_at"] is not None + cancelling = store.transition_status("lifecycle", "cancelling") + assert cancelling["cancellation_requested_at"] is not None + cancelled = store.transition_status("lifecycle", "cancelled") + assert cancelled["finished_at"] is not None + + with pytest.raises(InvalidMiningStatusTransitionError): + store.transition_status("lifecycle", "running") + + store.create({}, "data-v1", run_id="artifacts") + artifact = store.artifact_path("artifacts", "factors") + manifest = store.register_artifact("artifacts", "factors", artifact) + assert artifact.name == "factors.parquet" + assert manifest["artifacts"] == {"factors": "factors.parquet"} + assert store.write_summary("artifacts", {"candidate_count": 3}) == {"candidate_count": 3} + + with pytest.raises(MiningRunValidationError, match="escapes"): + store.register_artifact("artifacts", "folds", "../other/folds.parquet") + + +def test_historical_manifest_defaults_and_startup_recovery(tmp_path: Path) -> None: + store = _store(tmp_path) + runs_root = store.runs_root + old_dir = runs_root / "old_running" + old_dir.mkdir() + (old_dir / "manifest.json").write_text( + json.dumps( + { + "run_id": "old_running", + "status": "running", + "request": {"budget": 10}, + "data_fingerprint": "v1", + "created_at": "2025-01-01T00:00:00+00:00", + "artifacts": { + "factors": "factors.parquet", + "folds": "../escaped/folds.parquet", + "unknown": "unknown.parquet", + }, + } + ), + encoding="utf-8", + ) + store.create({}, "v1", run_id="was_cancelling") + store.transition_status("was_cancelling", "cancelling") + store.create({}, "v1", run_id="still_queued") + + historical = store.get("old_running") + assert historical is not None + assert historical["artifacts"] == {"factors": "factors.parquet"} + assert historical["finished_at"] is None + assert historical["run_signature"] == compute_run_signature({"budget": 10}, "v1") + + assert store.recover_interrupted() == 3 + assert store.get("old_running")["status"] == "interrupted" # type: ignore[index] + assert store.get("was_cancelling")["status"] == "interrupted" # type: ignore[index] + assert store.get("still_queued")["status"] == "interrupted" # type: ignore[index] + assert store.recover_interrupted() == 0 + + +def test_run_ids_and_paths_are_restricted_to_runs_root(tmp_path: Path) -> None: + store = _store(tmp_path) + + for run_id in ["", ".", "..", "../escape", "a/b", "a\\b", "with space"]: + with pytest.raises(MiningRunValidationError): + store.get(run_id) + + with pytest.raises(MiningRunValidationError): + store.create({}, "v1", run_id="../escape") + with pytest.raises(MiningRunValidationError): + store.create({}, "v1", run_id="") + assert not (tmp_path / "research" / "mining" / "escape").exists() + + +def test_find_by_signature_can_filter_active_and_success_runs(tmp_path: Path) -> None: + store = _store(tmp_path) + queued = store.create({"budget": 1}, "v1", run_id="queued_match") + store.create({"budget": 1}, "v1", run_id="failed_match") + store.transition_status("failed_match", "failed", error="failed") + store.create({"budget": 2}, "v1", run_id="success_match") + store.transition_status("success_match", "running") + store.transition_status("success_match", "succeeded_with_budget_exhausted") + + assert ( + store.find_by_signature(queued["run_signature"], statuses=ACTIVE_RUN_STATUSES)["run_id"] + == "queued_match" + ) # type: ignore[index] + assert store.find_by_signature(queued["run_signature"], statuses=SUCCESS_RUN_STATUSES) is None + success = store.get("success_match") + assert success is not None + assert ( + store.find_by_signature(success["run_signature"], statuses=SUCCESS_RUN_STATUSES)["run_id"] + == "success_match" + ) # type: ignore[index] + + +def test_list_runs_is_bounded_sorted_and_filters_status(tmp_path: Path) -> None: + store = _store(tmp_path) + store.create({"budget": 1}, "v1", run_id="first") + store.create({"budget": 2}, "v1", run_id="second") + store.transition_status("second", "running") + store.transition_status("second", "succeeded") + + runs = store.list_runs(limit=1) + assert [run["run_id"] for run in runs] == ["second"] + assert [run["run_id"] for run in store.list_runs(statuses={"queued"})] == ["first"] + + with pytest.raises(MiningRunValidationError, match="limit"): + store.list_runs(limit=0) + with pytest.raises(MiningRunValidationError, match="statuses"): + store.list_runs(statuses={"unknown"}) # type: ignore[arg-type] + + +def test_concurrent_event_appends_have_unique_contiguous_ids(tmp_path: Path) -> None: + first_store = _store(tmp_path) + second_store = _store(tmp_path) + first_store.create({}, "v1", run_id="concurrent") + barrier = threading.Barrier(5) + failures: list[Exception] = [] + + def append_batch(store: MiningRunStore, worker: int) -> None: + try: + barrier.wait() + for index in range(30): + store.append_event("concurrent", "progress", {"worker": worker, "index": index}) + except Exception as exc: + failures.append(exc) + + threads = [ + threading.Thread( + target=append_batch, + args=(first_store if worker % 2 == 0 else second_store, worker), + ) + for worker in range(4) + ] + for thread in threads: + thread.start() + barrier.wait() + for thread in threads: + thread.join(timeout=3) + + assert all(not thread.is_alive() for thread in threads) + assert failures == [] + ids = [event["id"] for event in first_store.read_events("concurrent")] + assert ids == list(range(1, 121)) diff --git a/backend/tests/test_mining_manager.py b/backend/tests/test_mining_manager.py new file mode 100644 index 0000000..dbaa6f0 --- /dev/null +++ b/backend/tests/test_mining_manager.py @@ -0,0 +1,352 @@ +from __future__ import annotations + +import threading +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +import app.services.mining_manager as mining_manager_module +from app.services.heavy_job_limiter import HeavyJobLimiter +from app.services.mining_manager import MiningJobManager + + +def _task_factory(kind: str, data_dir: Path, payload: dict[str, Any]) -> dict[str, Any]: + return { + "kind": kind, + "data_dir": str(data_dir), + "payload": payload, + } + + +def _wait_for_status( + manager: MiningJobManager, + run_id: str, + status: str, + *, + timeout: float = 2.0, +) -> dict[str, Any]: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + manifest = manager.store.get(run_id) + assert manifest is not None + if manifest["status"] == status: + return manifest + time.sleep(0.005) + pytest.fail(f"run {run_id} did not reach {status}") + + +@pytest.fixture +def isolated_limiter(monkeypatch: pytest.MonkeyPatch) -> HeavyJobLimiter: + limiter = HeavyJobLimiter(capacity=2, cancel_poll_interval=0.005) + monkeypatch.setattr(mining_manager_module, "shared_heavy_job_limiter", limiter) + return limiter + + +@pytest.fixture +def make_manager( + tmp_path: Path, + isolated_limiter: HeavyJobLimiter, +): + managers: list[MiningJobManager] = [] + + def factory( + runner: Callable[ + [dict[str, Any], Callable[[dict[str, Any]], None], threading.Event], + dict[str, Any], + ], + ) -> MiningJobManager: + manager = MiningJobManager( + tmp_path, + worker_runner=runner, + task_factory=_task_factory, + ) + managers.append(manager) + return manager + + yield factory + + for manager in managers: + manager.shutdown() + assert isolated_limiter.in_use == 0 + + +def test_start_records_states_events_progress_and_worker_payload( + make_manager, tmp_path: Path +) -> None: + progress_recorded = threading.Event() + finish = threading.Event() + tasks: list[dict[str, Any]] = [] + progress = {"phase": "screen", "done": 1, "total": 2} + result = {"status": "succeeded", "candidate_count": 3, "elapsed_ms": 12.5} + + def runner(task, progress_cb, cancel_event): + tasks.append(task) + progress_cb(progress) + progress_recorded.set() + assert finish.wait(2) + assert not cancel_event.is_set() + return result + + manager = make_manager(runner) + request = {"factor_names": ["momentum"], "budget_profile": "balanced"} + created = manager.start(request, {"daily": "v1"}, source="scheduled") + run_id = created["run_id"] + + assert created["status"] == "queued" + assert progress_recorded.wait(1) + assert manager.store.read_summary(run_id) == {"progress": progress} + assert [event["type"] for event in manager.store.read_events(run_id)] == [ + "queued", + "running", + "progress", + ] + assert manager.store.read_events(run_id)[0]["payload"]["source"] == "scheduled" + assert tasks == [ + { + "kind": "mining", + "data_dir": str(tmp_path.resolve()), + "payload": { + "run_id": run_id, + "request": request, + "data_fingerprint": {"daily": "v1"}, + "source": "scheduled", + }, + } + ] + + finish.set() + terminal = _wait_for_status(manager, run_id, "succeeded") + assert terminal["started_at"] is not None + assert terminal["finished_at"] is not None + assert manager.store.read_summary(run_id) == result + assert [event["type"] for event in manager.store.read_events(run_id)] == [ + "queued", + "running", + "progress", + "succeeded", + ] + + +def test_start_accepts_valid_persistent_run_id(make_manager) -> None: + def runner(task, progress_cb, cancel_event): + return {"status": "succeeded"} + + manager = make_manager(runner) + created = manager.start( + {"factor_names": ["value"]}, + "data-v1", + run_id="weekly_claim_2026_33", + ) + + assert created["run_id"] == "weekly_claim_2026_33" + terminal = _wait_for_status(manager, created["run_id"], "succeeded") + assert terminal["run_id"] == "weekly_claim_2026_33" + + +def test_duplicate_persistent_run_id_reuses_existing_without_starting_runner( + make_manager, +) -> None: + runner_called = threading.Event() + + def runner(task, progress_cb, cancel_event): + runner_called.set() + return {"status": "succeeded"} + + manager = make_manager(runner) + existing = manager.store.create( + {"factor_names": ["value"]}, + "data-v1", + run_id="weekly_claim_2026_33", + ) + + reused = manager.start( + {"factor_names": ["value"]}, + "data-v1", + run_id="weekly_claim_2026_33", + ) + + assert reused == existing + assert not runner_called.wait(0.05) + + +def test_start_reuses_active_and_success_but_force_creates_new_run(make_manager) -> None: + started = threading.Event() + release = threading.Event() + tasks: list[dict[str, Any]] = [] + + def runner(task, progress_cb, cancel_event): + tasks.append(task) + started.set() + assert release.wait(2) + return {"status": "succeeded", "candidate_count": 1} + + manager = make_manager(runner) + request = {"factor_names": ["value"]} + first = manager.start(request, "data-v1") + assert started.wait(1) + + active_reuse = manager.start(request, "data-v1") + assert active_reuse["run_id"] == first["run_id"] + assert len(tasks) == 1 + + release.set() + _wait_for_status(manager, first["run_id"], "succeeded") + success_reuse = manager.start(request, "data-v1") + assert success_reuse["run_id"] == first["run_id"] + assert len(tasks) == 1 + + forced = manager.start(request, "data-v1", force=True) + assert forced["run_id"] != first["run_id"] + _wait_for_status(manager, forced["run_id"], "succeeded") + assert len(tasks) == 2 + assert len(manager.store.list_runs()) == 2 + + +def test_cancel_while_waiting_for_capacity_never_calls_runner( + make_manager, + isolated_limiter: HeavyJobLimiter, +) -> None: + runner_called = threading.Event() + + def runner(task, progress_cb, cancel_event): + runner_called.set() + return {"status": "succeeded"} + + assert isolated_limiter.acquire("mining", timeout=0) + try: + manager = make_manager(runner) + created = manager.start({"factor_names": ["quality"]}, "data-v1") + run_id = created["run_id"] + assert manager.store.get(run_id)["status"] == "queued" # type: ignore[index] + + cancelling = manager.cancel(run_id) + assert cancelling["status"] == "cancelling" + _wait_for_status(manager, run_id, "cancelled") + assert not runner_called.is_set() + assert [event["type"] for event in manager.store.read_events(run_id)] == [ + "queued", + "cancelling", + "cancelled", + ] + finally: + isolated_limiter.release("mining") + + +def test_cancel_running_job_wins_over_worker_success(make_manager) -> None: + runner_started = threading.Event() + + def runner(task, progress_cb, cancel_event): + runner_started.set() + assert cancel_event.wait(2) + return {"status": "succeeded", "candidate_count": 9} + + manager = make_manager(runner) + created = manager.start({"factor_names": ["growth"]}, "data-v1") + run_id = created["run_id"] + assert runner_started.wait(1) + + cancelling = manager.cancel(run_id) + assert cancelling["status"] == "cancelling" + _wait_for_status(manager, run_id, "cancelled") + event_types = [event["type"] for event in manager.store.read_events(run_id)] + assert event_types == ["queued", "running", "cancelling", "cancelled"] + assert manager.store.read_summary(run_id) == {} + + +def test_runner_exception_marks_failed_and_appends_error_event(make_manager) -> None: + def runner(task, progress_cb, cancel_event): + raise RuntimeError("mining exploded") + + manager = make_manager(runner) + created = manager.start({"factor_names": ["size"]}, "data-v1") + run_id = created["run_id"] + + failed = _wait_for_status(manager, run_id, "failed") + assert failed["error"] == "mining exploded" + events = manager.store.read_events(run_id) + assert [event["type"] for event in events] == ["queued", "running", "error"] + assert events[-1]["payload"] == { + "status": "failed", + "message": "mining exploded", + } + + +def test_non_dict_worker_result_is_rejected(make_manager) -> None: + def runner(task, progress_cb, cancel_event): + return ["full", "result"] + + manager = make_manager(runner) + created = manager.start({"factor_names": ["liquidity"]}, "data-v1") + + failed = _wait_for_status(manager, created["run_id"], "failed") + assert failed["error"] == "mining worker result must be a compact dict" + + +def test_budget_exhausted_result_uses_distinct_success_status(make_manager) -> None: + result = { + "status": "succeeded_with_budget_exhausted", + "candidate_count": 2, + "budget_exhausted": True, + } + + def runner(task, progress_cb, cancel_event): + return result + + manager = make_manager(runner) + created = manager.start({"factor_names": ["volatility"]}, "data-v1") + run_id = created["run_id"] + + _wait_for_status(manager, run_id, "succeeded_with_budget_exhausted") + assert manager.store.read_summary(run_id) == result + assert manager.store.read_events(run_id)[-1]["type"] == ("succeeded_with_budget_exhausted") + + +def test_recover_interrupted_delegates_to_store(make_manager) -> None: + def runner(task, progress_cb, cancel_event): + return {"status": "succeeded"} + + manager = make_manager(runner) + manager.store.create({}, "v1", run_id="running_before_restart") + manager.store.transition_status("running_before_restart", "running") + manager.store.create({}, "v1", run_id="cancelling_before_restart") + manager.store.transition_status("cancelling_before_restart", "cancelling") + manager.store.create({}, "v1", run_id="queued_before_restart") + + assert manager.recover_interrupted() == 3 + assert manager.store.get("running_before_restart")["status"] == "interrupted" # type: ignore[index] + assert manager.store.get("cancelling_before_restart")["status"] == "interrupted" # type: ignore[index] + assert manager.store.get("queued_before_restart")["status"] == "interrupted" # type: ignore[index] + + +def test_shutdown_sets_cancel_uses_bounded_join_and_keeps_history( + make_manager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + runner_started = threading.Event() + release_runner = threading.Event() + + def runner(task, progress_cb, cancel_event): + runner_started.set() + assert release_runner.wait(2) + return {"status": "succeeded"} + + monkeypatch.setattr(mining_manager_module, "_SHUTDOWN_JOIN_SECONDS", 0.02) + manager = make_manager(runner) + created = manager.start({"factor_names": ["reversal"]}, "data-v1") + run_id = created["run_id"] + assert runner_started.wait(1) + + started = time.monotonic() + manager.shutdown() + elapsed = time.monotonic() - started + assert elapsed < 0.2 + assert manager.store.get(run_id)["status"] == "cancelling" # type: ignore[index] + + release_runner.set() + _wait_for_status(manager, run_id, "cancelled") + assert manager.store.get(run_id) is not None + with pytest.raises(RuntimeError, match="shut down"): + manager.start({"factor_names": ["new"]}, "data-v1") diff --git a/backend/tests/test_mining_process_lock.py b/backend/tests/test_mining_process_lock.py new file mode 100644 index 0000000..4d15cee --- /dev/null +++ b/backend/tests/test_mining_process_lock.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import multiprocessing +import os +from pathlib import Path +from queue import Empty + +from app.services.mining_process_lock import ( + MiningProcessLock, + MiningProcessLockError, +) + + +def _acquire_in_spawned_process(data_dir: str, result_queue) -> None: + lock = MiningProcessLock(Path(data_dir)) + try: + lock.acquire() + except MiningProcessLockError as exc: + result_queue.put(("blocked", str(exc))) + return + try: + result_queue.put(("acquired", None)) + finally: + lock.release() + + +def _spawn_lock_attempt(data_dir: Path) -> tuple[str, str | None]: + context = multiprocessing.get_context("spawn") + result_queue = context.Queue() + process = context.Process( + target=_acquire_in_spawned_process, + args=(str(data_dir), result_queue), + ) + process.start() + process.join(timeout=10) + if process.is_alive(): + process.terminate() + process.join(timeout=5) + raise AssertionError("spawned lock process did not exit") + assert process.exitcode == 0 + try: + return result_queue.get(timeout=2) + except Empty as exc: + raise AssertionError("spawned lock process returned no result") from exc + finally: + result_queue.close() + result_queue.join_thread() + + +def test_process_lock_rejects_contention_and_allows_acquire_after_release( + tmp_path, +) -> None: + owner = MiningProcessLock(tmp_path) + owner.acquire() + + blocked, message = _spawn_lock_attempt(tmp_path) + assert blocked == "blocked" + assert message is not None and "already owns mining" in message + + owner.release() + acquired, message = _spawn_lock_attempt(tmp_path) + assert acquired == "acquired" + assert message is None + + +def test_process_lock_handle_is_not_inheritable_and_release_is_idempotent( + tmp_path, +) -> None: + lock = MiningProcessLock(tmp_path) + lock.acquire() + + stream = vars(lock)["_stream"] + assert stream is not None + assert not os.get_inheritable(stream.fileno()) + + lock.release() + lock.release() diff --git a/backend/tests/test_mining_schedule.py b/backend/tests/test_mining_schedule.py new file mode 100644 index 0000000..a661d01 --- /dev/null +++ b/backend/tests/test_mining_schedule.py @@ -0,0 +1,427 @@ +from __future__ import annotations + +from datetime import date, datetime, timedelta +from pathlib import Path +from types import SimpleNamespace +from zoneinfo import ZoneInfo + +import polars as pl +import pytest + +from app.jobs import daily_pipeline +from app.services import mining_schedule, preferences +from app.services.mining_jobs import MiningRunStore + + +class FakeRepo: + def __init__(self, data_dir: Path, *, latest: date = date(2026, 8, 14)) -> None: + self.store = SimpleNamespace(data_dir=data_dir) + self.latest = latest + self.generation = "generation-1" + + def latest_enriched_date(self, asset_type: str = "stock") -> date | None: + assert asset_type == "stock" + return self.latest + + def get_matrix_data_generation(self, asset_type: str = "stock") -> str: + assert asset_type == "stock" + return self.generation + + def get_instruments_asset(self, asset_type: str = "stock") -> pl.DataFrame: + assert asset_type == "stock" + return pl.DataFrame({ + "symbol": ["000001.SZ"], + "name": ["示例"], + "total_shares": [1_000_000.0], + "float_shares": [800_000.0], + }) + + +class FakeManager: + def __init__(self, data_dir: Path) -> None: + self.store = MiningRunStore(data_dir) + self.calls: list[dict] = [] + + def start(self, request, fingerprint, *, force: bool, source: str, run_id: str): + call = { + "request": request, + "fingerprint": fingerprint, + "force": force, + "source": source, + "run_id": run_id, + } + self.calls.append(call) + manifest = self.store.create(request, fingerprint, run_id=run_id) + return {"run_id": manifest["run_id"]} + + +@pytest.fixture +def scheduled_state(tmp_path: Path, monkeypatch): + repo = FakeRepo(tmp_path) + manager = FakeManager(tmp_path) + state = SimpleNamespace(repo=repo, mining_manager=manager, strategy_engine=None) + monkeypatch.setattr( + preferences, + "get_mining_schedule", + lambda: { + "mining_schedule_enabled": True, + "mining_schedule_weekday": 4, + "mining_budget_profile": "balanced", + }, + ) + _write_prerequisites(tmp_path, repo.latest, days=1200) + return state + + +def _friday(week_offset: int = 0) -> datetime: + return datetime(2026, 8, 14, 16, tzinfo=ZoneInfo("Asia/Shanghai")) + timedelta( + weeks=week_offset + ) + + +def _write_prerequisites(data_dir: Path, latest: date, *, days: int) -> None: + enriched = data_dir / "kline_daily_enriched" + trading_dates: list[date] = [] + for offset in range(days): + day = latest - timedelta(days=offset) + if day.weekday() >= 5: + continue + trading_dates.append(day) + partition = enriched / f"date={day.isoformat()}" + partition.mkdir(parents=True, exist_ok=True) + (partition / "part.parquet").write_bytes(b"enriched") + regime = data_dir / "regime_history" / "part.parquet" + regime.parent.mkdir(parents=True, exist_ok=True) + pl.DataFrame({ + "date": sorted(trading_dates), + "state": ["range"] * len(trading_dates), + }).write_parquet(regime) + + +def test_beijing_date_and_iso_week_use_china_timezone(): + utc = ZoneInfo("UTC") + instant = datetime(2026, 8, 13, 16, 30, tzinfo=utc) + + assert mining_schedule.beijing_date(instant) == date(2026, 8, 14) + assert mining_schedule.iso_week(date(2027, 1, 1)) == (2026, 53) + + +def test_fingerprint_retries_generation_change_and_returns_stable_token( + tmp_path, + monkeypatch, +) -> None: + repo = FakeRepo(tmp_path) + state = SimpleNamespace(strategy_engine=None) + generations = iter([ + "generation-1", + "generation-2", + "generation-2", + "generation-2", + ]) + monkeypatch.setattr(repo, "get_matrix_data_generation", lambda _asset: next(generations)) + + fingerprint = mining_schedule.build_data_fingerprint( + repo, + state, + {"asset_type": "stock", "strategy_ids": []}, + ) + + assert fingerprint["generation"] == "generation-2" + + +def test_implementation_metadata_is_recursive_content_based_and_root_independent( + tmp_path, +) -> None: + roots = [tmp_path / "first" / "app", tmp_path / "second" / "app"] + for root in roots: + nested = root / "backtest" + nested.mkdir(parents=True) + (root / "main.py").write_text("VALUE = 1\n", encoding="utf-8") + (nested / "runtime.py").write_text("RESULT = 1\n", encoding="utf-8") + (nested / "ignored.txt").write_text("ignored\n", encoding="utf-8") + + first = mining_schedule._implementation_metadata(roots[0]) + second = mining_schedule._implementation_metadata(roots[1]) + (roots[1] / "backtest" / "runtime.py").write_text("RESULT = 2\n", encoding="utf-8") + changed = mining_schedule._implementation_metadata(roots[1]) + + assert first == second + assert first["file_count"] == 2 + assert str(tmp_path) not in str(first) + assert first["digest"] != changed["digest"] + + +def test_fingerprint_covers_result_implementation_digest( + tmp_path, + monkeypatch, +) -> None: + repo = FakeRepo(tmp_path) + state = SimpleNamespace(strategy_engine=None) + first = mining_schedule.build_data_fingerprint( + repo, + state, + {"asset_type": "stock", "strategy_ids": []}, + ) + monkeypatch.setattr( + mining_schedule, + "_implementation_metadata", + lambda _root: {"file_count": 1, "digest": "changed-runtime"}, + ) + second = mining_schedule.build_data_fingerprint( + repo, + state, + {"asset_type": "stock", "strategy_ids": []}, + ) + + assert first["implementation"] != second["implementation"] + assert first["digest"] != second["digest"] + + +def test_selected_strategy_metadata_changes_with_same_size_source_edit(tmp_path) -> None: + source = tmp_path / "strategies" / "custom" / "demo.py" + source.parent.mkdir(parents=True) + source.write_text("VALUE = 1\n", encoding="utf-8") + strategy = SimpleNamespace(execution_backend="matrix_native", file_path=source) + state = SimpleNamespace( + strategy_engine=SimpleNamespace(get=lambda _strategy_id: strategy) + ) + + first = mining_schedule._selected_strategy_metadata( + state, + ["demo"], + tmp_path, + ) + source.write_text("VALUE = 2\n", encoding="utf-8") + second = mining_schedule._selected_strategy_metadata( + state, + ["demo"], + tmp_path, + ) + + assert first[0]["source"]["size"] == second[0]["source"]["size"] + assert first[0]["source"]["sha256"] != second[0]["source"]["sha256"] + assert first[0]["source_tree"]["digest"] != second[0]["source_tree"]["digest"] + + +def test_fingerprint_rejects_continuously_changing_generation( + tmp_path, + monkeypatch, +) -> None: + repo = FakeRepo(tmp_path) + state = SimpleNamespace(strategy_engine=None) + generations = iter(["a", "b", "c", "d"]) + monkeypatch.setattr(repo, "get_matrix_data_generation", lambda _asset: next(generations)) + + with pytest.raises(ValueError, match="changed"): + mining_schedule.build_data_fingerprint( + repo, + state, + {"asset_type": "stock", "strategy_ids": []}, + ) + + +def test_disabled_and_before_scheduled_weekday_do_not_enqueue( + scheduled_state, + monkeypatch, +): + state = scheduled_state + monkeypatch.setattr( + preferences, + "get_mining_schedule", + lambda: { + "mining_schedule_enabled": False, + "mining_schedule_weekday": 4, + "mining_budget_profile": "balanced", + }, + ) + assert mining_schedule.run_weekly_mining(state, now=_friday())["status"] == "disabled" + + monkeypatch.setattr( + preferences, + "get_mining_schedule", + lambda: { + "mining_schedule_enabled": True, + "mining_schedule_weekday": 4, + "mining_budget_profile": "balanced", + }, + ) + thursday = _friday() - timedelta(days=1) + assert mining_schedule.run_weekly_mining(state, now=thursday)["status"] == "weekday_mismatch" + assert state.mining_manager.calls == [] + + +def test_later_workday_catches_up_once_in_same_iso_week(scheduled_state, monkeypatch): + monkeypatch.setattr( + preferences, + "get_mining_schedule", + lambda: { + "mining_schedule_enabled": True, + "mining_schedule_weekday": 3, + "mining_budget_profile": "balanced", + }, + ) + + first = mining_schedule.run_weekly_mining(scheduled_state, now=_friday()) + second = mining_schedule.run_weekly_mining(scheduled_state, now=_friday()) + + assert first["status"] == "enqueued" + assert second == {"status": "already_claimed", "run_id": first["run_id"]} + assert len(scheduled_state.mining_manager.calls) == 1 + + +def test_same_week_and_fingerprint_enqueue_once(scheduled_state): + first = mining_schedule.run_weekly_mining(scheduled_state, now=_friday()) + second = mining_schedule.run_weekly_mining(scheduled_state, now=_friday()) + + assert first["status"] == "enqueued" + assert second == {"status": "already_claimed", "run_id": first["run_id"]} + assert len(scheduled_state.mining_manager.calls) == 1 + call = scheduled_state.mining_manager.calls[0] + assert call["force"] is False + assert call["source"] == "scheduled" + assert call["run_id"] == first["run_id"] + assert call["run_id"] == call["fingerprint"]["source_claim"] + assert call["request"]["asset_type"] == "stock" + assert call["request"]["symbols"] is None + assert call["request"]["strategy_ids"] == [] + assert call["request"]["require_regime"] is True + assert call["request"]["end"] == "2026-08-14" + assert len(call["request"]["factor_names"]) <= 48 + + +def test_profile_change_cannot_bypass_same_week_claim(scheduled_state, monkeypatch): + first = mining_schedule.run_weekly_mining(scheduled_state, now=_friday()) + scheduled_state.mining_manager.store.transition_status( + first["run_id"], "failed", error="worker failed" + ) + monkeypatch.setattr( + preferences, + "get_mining_schedule", + lambda: { + "mining_schedule_enabled": True, + "mining_schedule_weekday": 4, + "mining_budget_profile": "strict", + }, + ) + + second = mining_schedule.run_weekly_mining(scheduled_state, now=_friday()) + + assert second == {"status": "already_claimed", "run_id": first["run_id"]} + assert len(scheduled_state.mining_manager.calls) == 1 + + +def test_new_week_creates_new_claim_but_same_week_metadata_change_does_not( + scheduled_state, +): + manager = scheduled_state.mining_manager + first = mining_schedule.run_weekly_mining(scheduled_state, now=_friday()) + next_week = mining_schedule.run_weekly_mining(scheduled_state, now=_friday(1)) + + latest_file = ( + scheduled_state.repo.store.data_dir + / "kline_daily_enriched" + / "date=2026-08-14" + / "part.parquet" + ) + latest_file.write_bytes(b"changed-enriched-metadata") + changed = mining_schedule.run_weekly_mining(scheduled_state, now=_friday()) + + assert first["run_id"] != next_week["run_id"] + assert changed == {"status": "already_claimed", "run_id": first["run_id"]} + assert len(manager.calls) == 2 + + +def test_missing_regime_records_visible_skipped_prerequisite(scheduled_state): + regime = scheduled_state.repo.store.data_dir / "regime_history" / "part.parquet" + regime.unlink() + + result = mining_schedule.run_weekly_mining(scheduled_state, now=_friday()) + manifest = scheduled_state.mining_manager.store.get(result["run_id"]) + + assert result["status"] == "skipped_prerequisite" + assert manifest is not None + assert manifest["status"] == "skipped_prerequisite" + assert "regime" in manifest["error"] + assert scheduled_state.mining_manager.calls == [] + + +def test_incomplete_regime_coverage_records_visible_skip(scheduled_state): + regime_path = ( + scheduled_state.repo.store.data_dir + / "regime_history" + / "part.parquet" + ) + history = pl.read_parquet(regime_path).sort("date") + history.filter(pl.col("date") != history["date"][-2]).write_parquet(regime_path) + + result = mining_schedule.run_weekly_mining(scheduled_state, now=_friday()) + manifest = scheduled_state.mining_manager.store.get(result["run_id"]) + + assert result["status"] == "skipped_prerequisite" + assert manifest is not None + assert "T-1" in manifest["error"] + assert scheduled_state.mining_manager.calls == [] + + +def test_early_regime_gap_records_visible_skipped_prerequisite(scheduled_state): + data_dir = scheduled_state.repo.store.data_dir + regime_path = data_dir / "regime_history" / "part.parquet" + history = pl.read_parquet(regime_path).sort("date") + history.slice(1).write_parquet(regime_path) + + result = mining_schedule.run_weekly_mining(scheduled_state, now=_friday()) + manifest = scheduled_state.mining_manager.store.get(result["run_id"]) + + assert result["status"] == "skipped_prerequisite" + assert manifest is not None + assert "T-1" in manifest["error"] + assert scheduled_state.mining_manager.calls == [] + + +def test_insufficient_data_records_visible_skipped_prerequisite(tmp_path, monkeypatch): + repo = FakeRepo(tmp_path) + manager = FakeManager(tmp_path) + state = SimpleNamespace(repo=repo, mining_manager=manager, strategy_engine=None) + monkeypatch.setattr( + preferences, + "get_mining_schedule", + lambda: { + "mining_schedule_enabled": True, + "mining_schedule_weekday": 4, + "mining_budget_profile": "strict", + }, + ) + _write_prerequisites(tmp_path, repo.latest, days=30) + + result = mining_schedule.run_weekly_mining(state, now=_friday()) + manifest = manager.store.get(result["run_id"]) + + assert result["status"] == "skipped_prerequisite" + assert manifest is not None + assert manifest["status"] == "skipped_prerequisite" + assert "insufficient" in manifest["error"] + assert manager.calls == [] + + +def test_pipeline_failure_does_not_trigger_mining(monkeypatch): + mining_calls = [] + monkeypatch.setattr(daily_pipeline, "_run_tracked", lambda *_args: False) + monkeypatch.setattr( + "app.services.mining_schedule.run_weekly_mining", + lambda state: mining_calls.append(state), + ) + + daily_pipeline._scheduled_pipeline_task(lambda: None) + + assert mining_calls == [] + + +def test_enqueue_failure_does_not_escape_successful_pipeline(monkeypatch): + monkeypatch.setattr(daily_pipeline, "_run_tracked", lambda *_args: True) + + def fail_enqueue(_state): + raise RuntimeError("queue unavailable") + + monkeypatch.setattr("app.services.mining_schedule.run_weekly_mining", fail_enqueue) + + daily_pipeline._scheduled_pipeline_task(lambda: None) diff --git a/backend/tests/test_minute_range_api.py b/backend/tests/test_minute_range_api.py index 52a7712..c7c53d3 100644 --- a/backend/tests/test_minute_range_api.py +++ b/backend/tests/test_minute_range_api.py @@ -26,7 +26,13 @@ def _request(repo=None, capset=None): def test_minute_range_returns_latest_sessions_with_previous_closes(): repo = MagicMock() repo.resolve_asset_type.return_value = "stock" - repo.execute_one.return_value = ("浦发银行", 1.0, 1.0) + # _get_stock_info 走 instruments 内存缓存 (不再走 execute_one DuckDB 查询) + repo.get_instruments.return_value = pl.DataFrame({ + "symbol": ["600000.SH"], + "name": ["浦发银行"], + "total_shares": [1.0], + "float_shares": [1.0], + }) repo.get_minute_range.return_value = pl.DataFrame({ "symbol": ["600000.SH"] * 3, "datetime": [ diff --git a/backend/tests/test_override_signature_cache.py b/backend/tests/test_override_signature_cache.py new file mode 100644 index 0000000..a5e9bbc --- /dev/null +++ b/backend/tests/test_override_signature_cache.py @@ -0,0 +1,97 @@ +"""策略 override mtime 签名缓存测试 — 读盘去重, 写入/删除后立即可见, 返回深拷贝。""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from app.strategy import config as strat_config + + +@pytest.fixture(autouse=True) +def _clean_cache(): + strat_config._override_cache.clear() + strat_config._override_cache_sig.clear() + yield + strat_config._override_cache.clear() + strat_config._override_cache_sig.clear() + + +def _patched_loads(monkeypatch, counter: dict): + real_loads = json.loads + + def _counting(text): + counter["loads"] += 1 + return real_loads(text) + + monkeypatch.setattr(strat_config.json, "loads", _counting) + + +def test_second_load_hits_cache_without_disk_parse(tmp_path, monkeypatch): + strat_config.save_override(tmp_path, "s1", {"params": {"p": 1}}) + counter = {"loads": 0} + _patched_loads(monkeypatch, counter) + + assert strat_config.load_override(tmp_path, "s1")["params"] == {"p": 1} + assert strat_config.load_override(tmp_path, "s1")["params"] == {"p": 1} + assert counter["loads"] == 0 or counter["loads"] == 1, "save 后首次 load 允许一次 parse" + before = counter["loads"] + strat_config.load_override(tmp_path, "s1") + assert counter["loads"] == before, "签名未变时不得重复读盘+parse" + + +def test_external_file_change_visible(tmp_path): + strat_config.save_override(tmp_path, "s1", {"params": {"p": 1}}) + assert strat_config.load_override(tmp_path, "s1")["params"]["p"] == 1 + + p: Path = tmp_path / "user_data" / "strategy_overrides" / "s1.json" + p.write_text(json.dumps({"params": {"p": 2}}), encoding="utf-8") + st = p.stat() + os.utime(p, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000)) + + assert strat_config.load_override(tmp_path, "s1")["params"]["p"] == 2 + + +def test_save_override_invalidates_cache(tmp_path): + strat_config.save_override(tmp_path, "s1", {"params": {"p": 1}}) + assert strat_config.load_override(tmp_path, "s1")["params"]["p"] == 1 + + strat_config.save_override(tmp_path, "s1", {"params": {"p": 9}}) + assert strat_config.load_override(tmp_path, "s1")["params"]["p"] == 9 + + +def test_delete_override_invalidates_cache(tmp_path): + strat_config.save_override(tmp_path, "s1", {"params": {"p": 1}}) + assert strat_config.load_override(tmp_path, "s1") != {} + + strat_config.delete_override(tmp_path, "s1") + assert strat_config.load_override(tmp_path, "s1") == {} + assert strat_config.load_override(tmp_path, "s1") == {} + + +def test_load_returns_deep_copy_not_cached_object(tmp_path): + strat_config.save_override(tmp_path, "s1", {"params": {"p": 1}, "basic_filter": {"a": 1}}) + first = strat_config.load_override(tmp_path, "s1") + first["params"]["p"] = 999 + first["extra"] = True + + again = strat_config.load_override(tmp_path, "s1") + assert again["params"]["p"] == 1 + assert "extra" not in again + + +def test_basic_filter_cleaning_preserved(tmp_path): + strat_config.save_override( + tmp_path, "s1", {"basic_filter": {"keep": 1, "drop": None}}, + ) + data = strat_config.load_override(tmp_path, "s1") + assert data["basic_filter"] == {"keep": 1} + + strat_config.save_override(tmp_path, "s2", {"basic_filter": {"drop": None}}) + assert "basic_filter" not in strat_config.load_override(tmp_path, "s2") + + +def test_load_missing_override_returns_empty(tmp_path): + assert strat_config.load_override(tmp_path, "never_saved") == {} diff --git a/backend/tests/test_preferences_cache.py b/backend/tests/test_preferences_cache.py new file mode 100644 index 0000000..634a42d --- /dev/null +++ b/backend/tests/test_preferences_cache.py @@ -0,0 +1,123 @@ +"""preferences mtime 缓存测试 — 读盘去重, 且外部修改/自身写入后立即可见。""" +from __future__ import annotations + +import json +import os + +import pytest + +from app.services import preferences + + +@pytest.fixture(autouse=True) +def _isolated(tmp_path, monkeypatch): + path = tmp_path / "preferences.json" + monkeypatch.setattr(preferences, "_path", lambda: path) + preferences._invalidate_cache() + yield path + preferences._invalidate_cache() + + +def _patched_loads(monkeypatch, counter: dict): + real_loads = json.loads + + def _counting(text): + counter["loads"] += 1 + return real_loads(text) + + monkeypatch.setattr(preferences.json, "loads", _counting) + + +def test_second_load_hits_cache_without_disk_parse(_isolated, monkeypatch): + _isolated.write_text(json.dumps({"realtime_quotes_enabled": True}), encoding="utf-8") + counter = {"loads": 0} + _patched_loads(monkeypatch, counter) + + assert preferences.load()["realtime_quotes_enabled"] is True + assert preferences.load()["realtime_quotes_enabled"] is True + assert counter["loads"] == 1, "签名未变时第二次 load 不得重复读盘+parse" + + +def test_external_file_change_invalidates_cache(_isolated, monkeypatch): + _isolated.write_text(json.dumps({"realtime_quote_interval": 6.0}), encoding="utf-8") + assert preferences.load()["realtime_quote_interval"] == 6.0 + + _isolated.write_text(json.dumps({"realtime_quote_interval": 3.0}), encoding="utf-8") + # 同尺寸修改且 mtime 粒度可能不变时, 显式推进 mtime 模拟真实场景 + st = _isolated.stat() + os.utime(_isolated, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000)) + + assert preferences.load()["realtime_quote_interval"] == 3.0 + + +def test_save_then_load_sees_merged_values(_isolated): + _isolated.write_text(json.dumps({"a": 1}), encoding="utf-8") + out = preferences.save({"b": 2}) + assert out == {"a": 1, "b": 2} + assert preferences.load() == {"a": 1, "b": 2} + + +def test_interval_setter_invalidates_cache(_isolated): + preferences.set_realtime_quote_interval(2.0) + assert preferences.load()["realtime_quote_interval"] == 2.0 + + +def test_load_returns_copy_not_cached_object(_isolated): + _isolated.write_text(json.dumps({"k": [1, 2]}), encoding="utf-8") + first = preferences.load() + first["k"].append(3) + first["extra"] = True + again = preferences.load() + assert again == {"k": [1, 2]} + + +def test_mining_schedule_defaults_are_disabled(_isolated): + assert preferences.get_mining_schedule() == { + "mining_schedule_enabled": False, + "mining_schedule_weekday": 4, + "mining_budget_profile": "balanced", + } + + +def test_mining_schedule_invalid_stored_values_fail_closed(_isolated): + _isolated.write_text( + json.dumps( + { + "mining_schedule_enabled": "false", + "mining_schedule_weekday": True, + "mining_budget_profile": None, + } + ), + encoding="utf-8", + ) + + assert preferences.get_mining_schedule() == { + "mining_schedule_enabled": False, + "mining_schedule_weekday": 4, + "mining_budget_profile": "balanced", + } + + +def test_mining_schedule_setter_saves_group_once(monkeypatch): + calls = [] + monkeypatch.setattr(preferences, "save", lambda updates: calls.append(updates) or updates) + + result = preferences.set_mining_schedule(True, 2, "strict") + + assert result == { + "mining_schedule_enabled": True, + "mining_schedule_weekday": 2, + "mining_budget_profile": "strict", + } + assert calls == [result] + + +@pytest.mark.parametrize("weekday", [-1, 5, True]) +def test_mining_schedule_setter_rejects_invalid_weekday(weekday): + with pytest.raises(ValueError, match="weekday"): + preferences.set_mining_schedule(True, weekday, "balanced") + + +def test_mining_schedule_setter_rejects_invalid_profile(): + with pytest.raises(ValueError, match="profile"): + preferences.set_mining_schedule(True, 4, "exploratory") diff --git a/backend/tests/test_price_limits.py b/backend/tests/test_price_limits.py index 218bf55..8857621 100644 --- a/backend/tests/test_price_limits.py +++ b/backend/tests/test_price_limits.py @@ -200,6 +200,7 @@ def test_realtime_limit_prices_ignore_stale_instrument_date(): "close": [9.10], "raw_close": [9.10], "raw_high": [9.10], + "raw_low": [9.10], "_prev_close_raw": [10.0], "volume": [1000.0], }) @@ -215,3 +216,41 @@ def test_realtime_limit_prices_ignore_stale_instrument_date(): assert result["signal_limit_down"][0] is False assert "_instrument_as_of" not in result.columns + + +def test_limit_down_recovery_uses_raw_low_under_later_ex_div(): + """除权事件之后重算历史时, 跌停翘板"曾触及跌停"必须用原始价 low 判断。 + + day2 (历史日): 原始 low 9.30 未触及跌停价 9.00, 不应触发翘板; + 但 day3 除权 (ex_factor=2) 使 day2 前复权 low 变为 4.65, + 若误用复权 low 对比原始口径跌停价会误报翘板。 + day3 (除权日, 最新日不复权): 涨跌停基准切换为前复权昨收 4.825 → 跌停价 4.34, + 原始 low 4.34 触及且收阳未封死 → 真翘板。 + """ + raw = pl.DataFrame({ + "symbol": ["600001.SH"] * 3, + "date": [date(2024, 1, 2), date(2024, 1, 3), date(2024, 1, 4)], + "open": [10.00, 9.60, 4.30], + "high": [10.10, 9.70, 4.45], + "low": [9.90, 9.30, 4.34], + "close": [10.00, 9.65, 4.42], + "volume": [10000.0, 10000.0, 10000.0], + "amount": [1.0e7, 1.0e7, 1.0e7], + }) + factors = pl.DataFrame({ + "symbol": ["600001.SH"], + "trade_date": [date(2024, 1, 4)], + "ex_factor": [2.0], + }) + instruments = pl.DataFrame({ + "symbol": ["600001.SH"], + "name": ["普通股"], + "float_shares": [1.0e8], + }) + + df = pipeline.compute_enriched(raw, factors=factors, instruments=instruments) + + day2 = df.filter(pl.col("date") == date(2024, 1, 3)) + assert day2["signal_limit_down_recovery"][0] is False + day3 = df.filter(pl.col("date") == date(2024, 1, 4)) + assert day3["signal_limit_down_recovery"][0] is True diff --git a/backend/tests/test_quote_name_map_reuse.py b/backend/tests/test_quote_name_map_reuse.py new file mode 100644 index 0000000..37d0931 --- /dev/null +++ b/backend/tests/test_quote_name_map_reuse.py @@ -0,0 +1,57 @@ +"""监控 name_map 测试 — 走 repo.get_name_map() memo, 过滤空名称与旧行为一致。""" +from __future__ import annotations + +from types import SimpleNamespace + +from app.services.quote_service import _monitor_name_map + + +class _FakeRepo: + def __init__(self, mapping: dict[str, str]) -> None: + self._mapping = mapping + self.calls = 0 + + def get_name_map(self) -> dict[str, str]: + self.calls += 1 + return dict(self._mapping) + + +def test_filters_falsy_names_like_legacy_build(): + repo = _FakeRepo({ + "600000.SH": "浦发银行", + "000001.SZ": "", # 空名称: 旧 iter_rows 构建会跳过 + "399001.SZ": None, # None 名称: 同上 + "510300.SH": "沪深300ETF", + }) + assert _monitor_name_map(repo) == { + "600000.SH": "浦发银行", + "510300.SH": "沪深300ETF", + } + + +def test_merges_all_asset_types_from_repo_map(): + # get_name_map 已合并股票 + ETF + 指数 (股票优先), 监控回填无需再分表构建 + repo = _FakeRepo({"600000.SH": "股票", "510300.SH": "ETF", "000001.SH": "指数"}) + assert _monitor_name_map(repo) == repo._mapping + + +def test_delegates_to_repo_each_call(): + repo = _FakeRepo({"600000.SH": "浦发银行"}) + _monitor_name_map(repo) + _monitor_name_map(repo) + assert repo.calls == 2 + + +def test_empty_repo_map_returns_empty(): + assert _monitor_name_map(_FakeRepo({})) == {} + + +def test_survives_repo_failure(): + # 与旧行为一致: name_map 构建失败不影响监控主流程 (调用方捕获) + repo = SimpleNamespace() + try: + _monitor_name_map(repo) # type: ignore[arg-type] + raised = False + except AttributeError: + raised = True + assert raised, "repo 无 get_name_map 时应抛出, 由调用方 try 兜底" diff --git a/backend/tests/test_realtime_turnover_rate.py b/backend/tests/test_realtime_turnover_rate.py index c4ad75a..bb0a5aa 100644 --- a/backend/tests/test_realtime_turnover_rate.py +++ b/backend/tests/test_realtime_turnover_rate.py @@ -16,6 +16,7 @@ def _today_rows(turnover_rate: float | None = None) -> pl.DataFrame: "close": 10.0, "raw_close": 10.0, "raw_high": 10.0, + "raw_low": 10.0, "volume": 8000.0, } if turnover_rate is not None: diff --git a/backend/tests/test_regime_builder.py b/backend/tests/test_regime_builder.py index c031546..a33027a 100644 --- a/backend/tests/test_regime_builder.py +++ b/backend/tests/test_regime_builder.py @@ -341,17 +341,30 @@ def test_build_regime_mask_fails_when_required_t1_date_is_missing(tmp_path): ) -def test_build_regime_mask_first_day_allowed(tmp_path): - """首日无前一日环境数据 → 默认允许(不阻断)。""" +def test_build_regime_mask_first_formal_day_requires_warmup_predecessor(tmp_path): + """正式首日缺少前一交易标签时必须阻断; warmup 前缀可安全对齐。""" from app.backtest.strategy import StrategyBacktestService regime_builder.upsert_regime_history(tmp_path, pl.DataFrame({ - "date": [date(2026, 1, 1)], - "state": ["weak"], "score": [10], + "date": [date(2026, 1, 1), date(2026, 1, 2)], + "state": ["weak", "strong"], + "score": [10, 85], })) - labels = ("2026-01-01", "2026-01-02") + with pytest.raises(ValueError, match="正式首日"): + StrategyBacktestService._build_regime_mask( + ("2026-01-01", "2026-01-02"), + {"states": ["strong"]}, + tmp_path, + required_start=date(2026, 1, 1), + required_end=date(2026, 1, 2), + ) + mask = StrategyBacktestService._build_regime_mask( - labels, {"states": ["strong"]}, tmp_path, + ("2026-01-01", "2026-01-02", "2026-01-03"), + {"states": ["strong"]}, + tmp_path, + required_start=date(2026, 1, 2), + required_end=date(2026, 1, 3), ) - # 1/1 首日 → True; 1/2 由 1/1(weak) → False - assert mask.tolist() == [True, False] + assert mask is not None + assert mask.tolist() == [True, False, True] diff --git a/backend/tests/test_repository_index.py b/backend/tests/test_repository_index.py index 3e546d3..56c0450 100644 --- a/backend/tests/test_repository_index.py +++ b/backend/tests/test_repository_index.py @@ -46,6 +46,43 @@ def test_name_map_stock_beats_index(repo): assert repo.get_name_map(["600000.SH"]).get("600000.SH") == "浦发银行" +def _write_stock_instruments(repo, symbols, names): + pl.DataFrame({ + "symbol": symbols, "name": names, "code": [s[:6] for s in symbols], + "exchange": ["SH"] * len(symbols), "region": ["CN"] * len(symbols), + "type": ["stock"] * len(symbols), + "listing_date": [None] * len(symbols), "total_shares": [None] * len(symbols), + "float_shares": [None] * len(symbols), "tick_size": [None] * len(symbols), + "limit_up": [None] * len(symbols), "limit_down": [None] * len(symbols), + "as_of": ["2026-08-14"] * len(symbols), + }).write_parquet(repo.store.data_dir / "instruments" / "instruments.parquet") + repo._refresh_instruments() + + +def test_name_map_partial_query_does_not_poison_cache(repo): + """带 symbols 的部分查询不能把残缺映射写入缓存 (自选新加股票无名称的回归). + + 旧 bug: 首次 get_name_map(["600000.SH"]) 把只含 600000 的映射缓存住, + 之后自选加入 000001.SZ 再查名称命中残缺缓存 → name=None。 + """ + _write_stock_instruments(repo, ["600000.SH", "000001.SZ"], ["浦发银行", "平安银行"]) + first = repo.get_name_map(["600000.SH"]) + assert first == {"600000.SH": "浦发银行"} + # 缓存必须是全量: 后续其他 symbols 查询仍能命中 + second = repo.get_name_map(["000001.SZ"]) + assert second == {"000001.SZ": "平安银行"} + full = repo.get_name_map() + assert full == {"600000.SH": "浦发银行", "000001.SZ": "平安银行"} + + +def test_name_map_cache_invalidated_on_instruments_refresh(repo): + """维表刷新后缓存必须失效: 新收录的股票能立刻查到名称。""" + _write_stock_instruments(repo, ["600000.SH"], ["浦发银行"]) + assert repo.get_name_map(["600000.SH"]) == {"600000.SH": "浦发银行"} + _write_stock_instruments(repo, ["600000.SH", "301999.SZ"], ["浦发银行", "新股股份"]) + assert repo.get_name_map(["301999.SZ"]) == {"301999.SZ": "新股股份"} + + import datetime as _dt diff --git a/backend/tests/test_screener_builtin_params.py b/backend/tests/test_screener_builtin_params.py index 00e0d98..660d9d2 100644 --- a/backend/tests/test_screener_builtin_params.py +++ b/backend/tests/test_screener_builtin_params.py @@ -44,6 +44,11 @@ class _CapturingStrategyEngine: def has(self, strategy_id): return strategy_id == "builtin_strategy" + def get(self, strategy_id): + if not self.has(strategy_id): + raise ValueError(f"unknown strategy: {strategy_id}") + return types.SimpleNamespace(meta={"id": strategy_id}) + def run(self, strategy_id, context, *, pool=None, params=None, overrides=None): self.calls.append({ "kind": "run", diff --git a/backend/tests/test_settings_mining_schedule.py b/backend/tests/test_settings_mining_schedule.py new file mode 100644 index 0000000..3b7e78f --- /dev/null +++ b/backend/tests/test_settings_mining_schedule.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.api import settings + + +def test_mining_schedule_model_is_strict_and_forbids_extra_fields(): + valid = settings.MiningSchedulePrefs( + mining_schedule_enabled=True, + mining_schedule_weekday=4, + mining_budget_profile="strict", + ) + assert valid.model_dump() == { + "mining_schedule_enabled": True, + "mining_schedule_weekday": 4, + "mining_budget_profile": "strict", + } + + invalid_payloads = [ + { + "mining_schedule_enabled": "true", + "mining_schedule_weekday": 4, + "mining_budget_profile": "balanced", + }, + { + "mining_schedule_enabled": True, + "mining_schedule_weekday": 5, + "mining_budget_profile": "balanced", + }, + { + "mining_schedule_enabled": True, + "mining_schedule_weekday": 4, + "mining_budget_profile": "exploratory", + }, + { + "mining_schedule_enabled": True, + "mining_schedule_weekday": 4, + "mining_budget_profile": "balanced", + "unknown": True, + }, + ] + for payload in invalid_payloads: + with pytest.raises(ValidationError): + settings.MiningSchedulePrefs.model_validate(payload) + + +def test_update_mining_schedule_calls_group_setter_once(monkeypatch): + calls = [] + monkeypatch.setattr( + "app.services.preferences.set_mining_schedule", + lambda enabled, weekday, profile: ( + calls.append((enabled, weekday, profile)) + or { + "mining_schedule_enabled": enabled, + "mining_schedule_weekday": weekday, + "mining_budget_profile": profile, + } + ), + ) + request = settings.MiningSchedulePrefs( + mining_schedule_enabled=True, + mining_schedule_weekday=1, + mining_budget_profile="strict", + ) + + result = settings.update_mining_schedule(request) + + assert calls == [(True, 1, "strict")] + assert result["mining_budget_profile"] == "strict" diff --git a/backend/tests/test_st_limit_and_sharpe.py b/backend/tests/test_st_limit_and_sharpe.py index 75402b5..bb82bbd 100644 --- a/backend/tests/test_st_limit_and_sharpe.py +++ b/backend/tests/test_st_limit_and_sharpe.py @@ -55,6 +55,7 @@ def _two_day( "raw_close": [prev_close, today_close], "close": [prev_close, today_close], "raw_high": [prev_close, today_close], + "raw_low": [prev_close, today_close], "open": [prev_close, today_close], "high": [prev_close, today_close], "low": [prev_close, today_close], diff --git a/docs/features.md b/docs/features.md index 3fa5cce..8e9c09d 100644 --- a/docs/features.md +++ b/docs/features.md @@ -8,7 +8,7 @@ ## 🔍 选股引擎(Screener) -**20 个内置策略**,每个策略一个独立 Python 文件,基于 Polars 表达式向量化实现(`backend/app/strategy/builtin/`): +**18 个内置策略**,每个策略一个独立 Python 文件,基于 Polars 表达式向量化实现(`backend/app/strategy/builtin/`): | 类型 | 代表策略 | | :---------- | :------------------------------------------------------- | @@ -54,6 +54,10 @@ 输出净值曲线 · 夏普 · 最大回撤 · 胜率 · 交易明细。SSE 流式进度支持切页重连,不会丢失回测任务。 +**因子与策略挖掘**:复用已有日频因子和 matrix-native 策略，通过 T-1 市场环境、相关去重和嵌套样本外验证生成研究候选。任务在 spawn worker 中运行，支持持久 run ID、取消、刷新重连和显式发布；自动周度任务默认关闭且永不自动发布。完整口径见 [因子与策略挖掘](./mining.md)。 + +**市场阶段与主线**:市场环境页在原 5 档状态之外新增情绪周期阶段(冰点/启动/主升/高潮/退潮/修复,由连板梯队的高度、宽度、晋级率、梯队完整度判定,平均段长约 10 天)与主线识别(概念/行业维度的涨停梯队聚合排名,可配置宽基标签过滤)。完整口径见 [市场阶段与主线识别](./market-phase.md)。 + **ETF 支持**:三种模式的后端与 API 均支持 `asset_type=etf`,回测面板改从 `kline_etf_enriched` 读取(单次回测为单一资产类型,不混合股票与 ETF)。策略组合与因子回测页均有 `股票 / ETF` 切换,ETF 模式下策略列表与标的搜索跟随资产。需先开启 ETF 拉取并跑盘后管道。 --- diff --git a/docs/market-phase.md b/docs/market-phase.md new file mode 100644 index 0000000..90ba2af --- /dev/null +++ b/docs/market-phase.md @@ -0,0 +1,95 @@ +# 市场阶段（情绪周期）与主线识别 + +本文说明市场环境页的阶段体系与主线识别的口径、阈值来源、持久化设计、已知偏差与运行方式。它与原有的 5 档环境 state 并存，不替代任何既有消费方。 + +## 功能边界 + +阶段体系回答"市场处于情绪周期什么位置"，主线识别回答"当前/某阶段内什么板块概念在领涨"。两者都只用本地已存储数据（`consecutive_limit_ups`、`amount`、概念映射快照），不依赖扩展数据源，2020-08 起全历史可回算。 + +明确不做的： + +- 挖掘、回测环境过滤、因子 regime 统计仍用原 5 档 `state`，本期不切换到阶段体系。 +- 概念成分是当前快照，不改成 timeseries 积累模式（见「快照回填偏差」）。 +- 不生成任何交易信号；阶段与主线仅供研究分析。 + +## 与 5 档 state 的关系 + +| | `state`（原有） | `phase`（本期新增） | +|---|---|---| +| 驱动量 | 赚钱/投机/抗跌/趋势 4 维综合分 | 连板梯队：高度、宽度、晋级率、梯队完整度 | +| 档位 | 强势/偏强/震荡/偏弱/弱势（5 档） | 冰点/启动/主升/高潮/退潮/修复（6 阶段） | +| 消费方 | 回测环境过滤、挖掘、策略页 | 市场环境页分析与主线识别 | +| 持续性 | 日频打分，切换频繁（历史 76.6% 天数发生切换，平均段长 1.1-1.5 天） | EMA 平滑 + 2 日确认，平均段长 9.7 天 | + +`state`/`score` 列原样保留，新列（`phase`、`first_board`、`ge2_count`、`ge3_count`、`ge5_count`、`ladder_completeness`、`promo_rate`、`promo_pool`）对既有消费方透明。 + +## 每日梯队指标 + +全部由已存储的 `consecutive_limit_ups` 列向量化派生，实现在 `backend/app/services/market_phase.py`： + +- `first_board`：首板（1 连板）家数。 +- `ge2_count` / `ge3_count` / `ge5_count`：N 板以上家数（宽度）。 +- `promo_rate` 晋级率：昨日连板池今日继续封板的比例。昨日连板数用 `consec.shift(1).over("symbol")` 按个股前一日连接（跨批次边界也正确，`_compute_batch` 在 warmup 截断前计算）；池不足 10 家记 `null`（小样本噪声），不填 0。 +- `ladder_completeness` 梯队完整度：2..height 档位中非空档位占比；height < 3 时为 `null`。 +- 高度（`max_consecutive`）与封板率（`seal_rate`）沿用 regime 已有列。 + +## 阶段规则 + +词汇与优先级：`climax 高潮 > rally 主升 > ebb 退潮 > ignite 启动 > ice 冰点 > repair 修复（兜底）`。冰点排在退潮前判定——长期死市不应被标成"自高位退潮"。 + +阈值标定自 2020-08~2026-08 全市场 1454 个交易日的 p10/p60/p90 分位数，全部集中在模块顶部常量，调整只改那里： + +| 阶段 | 核心条件（EMA 平滑后） | +|---|---| +| 高潮 climax | ge2 ≥ 50（p90 的 2 倍）或首板 ≥ 220（p90 的 2.5 倍），历史占比 <2% | +| 主升 rally | 高度 ≥7 且 ge2 ≥15 且晋级率 ≥0.23（全中位以上）；或晋级率 ≥0.30（p85+）配合高度 ≥5、ge2 ≥12 | +| 退潮 ebb | 晋级率 <0.15（p20）且宽度自 5 日前高位（ge2>12 或高度>6）回落；或晋级率 <0.13 且封板率 <0.57 双弱 | +| 启动 ignite | 宽度/高度自低位扩张（ge2 较 5 日前 +3 且 ≥8，或高度抬升且 ≥5）且晋级率恢复到 ~0.19-0.20 | +| 冰点 ice | 高度 ≤4、ge2 ≤6、首板 ≤24 同时贴地（均 ~p10） | +| 修复 repair | 兜底 | + +**持续性设计**：驱动量先做 EMA 平滑（alpha=1/3，约 5 日），原始标签出来后再做 2 日确认（切换需连续 2 日出现新标签才生效）。 + +**弱档否决**：5 档 `state ∈ {weak, lean_weak}` 时，正向阶段（主升/高潮/启动）一律降为修复。这修复了一类错标：连板梯队强但大盘崩的交易日（如 2024-01 微盘流动性危机）会被梯队指标误判为主升/启动——涨停生态与大盘背离时，以大盘弱势为准。 + +**回填验收**（1454 天真实数据）：平均段长 9.7 天（对比原 5 档的 1.1-1.5 天）。抽查：2024-09-24→10-08 为启动→主升→高潮（九二四行情）；2024-01/02 微盘崩为退潮/冰点；高潮 6 年仅 2 段（2024-10-08、2024-10-31 ST 重组潮 17 板），均为真实极端期。 + +已知特性，如实说明：退潮/冰点天然是短段（平均 2.8/2.5 天）——恐慌释放本身快于趋势形成；修复是占比最高的兜底段（~74% 天数），A股大部分时间没有处于可辨认的周期位置。 + +## 主线识别 + +实现在 `backend/app/services/market_mainline.py`。复用 `rps_rotation` 的概念映射加载，窄扫描 `kline_daily_enriched` 的 symbol/date/consecutive_limit_ups/amount 四列，按 (date, 概念/行业) 聚合： + +- `limit_up_count` 涨停家数、`ge2_count` 二板以上家数、`max_boards` 最高板、`boards_sum`、`rungs_filled` 梯队档位数、龙头股（按连板数、成交额排序取第一）。 +- 主线分 = 当日截面 rank 归一后 `0.35*涨停数 + 0.25*最高板 + 0.25*梯队档位数 + 0.15*二板宽度`。这是"同板块涨停越多、梯队越完整越是主升主线"判据的直接量化。 +- 每概念当日涨停 <3 家不参与排名；每日持久化 top30 到 `data/mainline_history/part.parquet`（upsert 按 date+kind 整日替换）。 +- 行业维度取前两级（如 `计算机-软件开发`），作为概念维度的交叉验证。 + +### 宽基/风格标签过滤 + +融资融券（~7700 家）、沪深股通（~3300 家）这类超大概念会垄断 top1，需要过滤。配置存于用户偏好（`preferences.json`），市场环境页「过滤」面板可改： + +- 成员数上限：默认 600，超过视为宽基/风格标签不参与排名（夹取范围 50-5000）。保留华为概念（2006）、人工智能（2166）等真主题。 +- 成员数下限：默认 4（夹取 1-200）。 +- 名称黑名单：手动屏蔽任意概念，支持逗号/分号/空格分隔。 + +修改保存后自动触发全量主线重算（`POST /api/regime/mainline/recompute`，秒级）。API 层为 `PUT /api/preferences/mainline-filter`。 + +### 快照回填偏差 + +概念成分为**当前快照回看历史**（本地自 2026-07 起留存，无历史版本）。新纳入指数或改名的个股会出现在旧时段、成分调整会错归属，因此早年主线存在归属漂移，越近越准。此口径限制以 `membership_note` 字段随 API 返回并在页面展示。若后续把 ext 概念同步改为按月累积快照，主线历史精度会逐月自然提升。 + +## API 与运行 + +- `GET /api/regime/phases?start&end`：连续阶段段列表（阶段、区间、天数、高度/宽度/晋级率/封板率均值、段内 top 主线）。 +- `GET /api/regime/mainline?start&end&top&kind`：窗口内主线排行（top1 天数、日均分、最高板）与每日明细。 +- `POST /api/regime/mainline/recompute`：过滤配置变更后的全量重算（两类维度）。 +- `POST /api/regime/recompute`：扩展为重算 regime 后自动重标 phase 并回填主线。 +- daily pipeline 在 regime 步骤后追加主线增量（复用 `pipeline_regime_enabled` 开关，软失败不阻塞）。 + +阶段标签由 `regime_builder.refresh_phase_labels` 在每次 regime upsert 后对全量历史重标（1454 行，开销可忽略）——因为 EMA 与 2 日确认依赖完整序列。 + +## 测试 + +- `backend/tests/test_market_phase.py`：梯队聚合、晋级率（池不足记 null）、梯队完整度、阶段序列规则、弱档否决、持续性（噪声下不出现 1 日翻转）、refresh 往返。 +- `backend/tests/test_market_mainline.py`：概念/行业聚合、宽基过滤、黑名单、最少涨停家数、同日 upsert 替换、增量补齐、偏好读写与夹取。 diff --git a/docs/mining.md b/docs/mining.md new file mode 100644 index 0000000..79f666e --- /dev/null +++ b/docs/mining.md @@ -0,0 +1,138 @@ +# 因子与策略挖掘 + +本文说明 V1 因子与策略挖掘的金融口径、执行边界、结果解释和运行要求。 + +## 功能边界 + +V1 仅研究仓库已经提供的日频因子和用户明确选择的 matrix-native 日线策略。它完成以下闭环： + +1. 在训练区间重新计算因子方向与统计。 +2. 按日截面 Rank IC 计算因子相关性并去重。 +3. 搜索最多四个因子的受控排名组合；进入 beam 搜索的因子按训练折综合分排序并截断到 `beam_width` 个，保证单因子打分后组合搜索仍有代理预算可用。 +4. 用嵌套样本外验证比较新组合；用户选择的已有策略作为对照轨在每个 outer 测试窗独立评估，不参与因子竞争，也不会占用候选名额。 +5. 将运行、事件和结果持久化，允许刷新后重连。 +6. 将候选保存到研究候选库；只有用户显式确认且通过晋级门槛后才发布独立策略。 + +V1 不生成任意公式，不接受 AI 自由代码，不使用扩展数据生成新因子，也不使用分钟数据优化入场。分钟级成交和分钟因子需要独立的数据完整性、防未来函数和性能边界，属于后续版本范围。 + +因子数量以运行时 `FACTOR_COLUMNS` 目录为准。当前目录与后续版本可能不同；单次请求硬上限为 48，不应把历史设计稿中的固定数量当作稳定 API 契约。 + +当前目录除价量技术类（动量、均线偏离、趋势、波动率、量价、价格位置、超买超卖）外，还包含三个 A 股实证维度：收益形态（`max_ret_20d` 彩票效应、`ret_skew_20d`、`up_days_20d`）、流动性（`amihud_20d` 非流动性、`turnover_z_60d` 换手异动）、涨停基因（`limit_up_count_20d/60d`，基于存储列 `consecutive_limit_ups` 计数，涨停判定沿用环境信号口径）。这些因子仍只用本地日频存储数据计算，不依赖扩展数据源。 + +财务因子（`pb_latest`、`roe_latest`、`gross_margin_latest`、`net_margin_latest`、`revenue_yoy_latest`、`net_income_yoy_latest`、`debt_ratio_latest`）来自本地财务快照（数据页同步），采用严格点时口径：**因子值只在晚于公告日的交易日生效**（公告多在盘后， 保守取公告次一交易日起），同一报告期以最新公告为准。无财务数据的标的、公告前的日期一律为空值并从当日截面剔除，绝不填 0；本地完全没有财务数据时因子回测直接返回明确错误而不是全空结果。回测区间早于首次公告时同样报告"无有效数据"。挖掘中财务因子在训练折覆盖不足时综合分为 0 并被自然淘汰，不会阻塞其他因子。财务报表同步按 `(symbol, period_end)` 累积历史（同 shares 表模式），每次同步只拉最新期并为新标的补全量历史；随着季度累积，财务因子的可回测区间会逐步变长。 + +## 时间与收益口径 + +### 全局交易日标签 + +1、3、5 日 forward return 都按全局交易日轴精确连接。停牌或缺行不会把“下一条标的数据”误当成下一交易日。 + +周频使用每个 ISO 周的第一个实际交易日，月频使用每月第一个实际交易日。节假日不会导致整周漏掉调仓。 + +### T-1 市场环境 + +交易日 T 只能使用上一交易日已经得到的环境标签 T-1。**统计与挖掘口径**聚合为三档： + +```text +strong = strong + lean_strong +range = range +weak = lean_weak + weak +``` + +**策略回测的环境过滤**按原始五档匹配：勾选“强势”只允许 T-1 为 strong 的交易日入场，需要三档口径时必须同时勾选“强势+偏强”。挖掘内部的 strong/range/weak 环境拆分仍按上方三档聚合实现（显式枚举原始状态），两者互不影响。 + +正式区间缺少前驱交易日或前驱环境时会 fail-closed，不会用当日环境、最近自然日或默认环境补齐。 + +### 成交和成本 + +候选策略沿用现有回测撮合规则，包括 T+1、佣金、卖侧印花税、滑点以及涨跌停不可成交约束。因子统计中的 long-short spread 仅用于衡量因子区分能力，是理论价差，不表示 A 股可执行卖空。 + +## 防止未来数据 + +挖掘使用嵌套 walk-forward，而不是在完整样本上选择后再报告同一段表现： + +- 每个 inner 训练折独立学习方向、计算统计、相关去重和组合搜索。 +- inner test 只用于选择候选，不参与该折的训练计算。 +- 选择完成后在完整 outer train 上重新训练。 +- outer test 只进行一次最终样本外评估。 +- 训练和测试之间保留 purge 与 embargo；默认 purge 为 30 个交易日。 + +任何 fold 缺数据、缺 T-1 环境或不能可靠撮合时都记录为 skipped 并给出原因，缺失指标返回 `null`，不会显示为 0。 + +## 候选证据口径 + +run 级的“有效折”统计只描述因子赛道：它按每折被选中的候选聚合，不是任何单个候选的样本量。单个候选的逐折证据由三类行组成，`folds.parquet` 的 `evaluation_kind` 区分： + +- `selected`：该候选在该 outer 折经 inner 验证胜出后的 outer 测试结果。 +- `cross`：该候选的定义在其他折胜出后，在本折补评的跨折结果。胜出定义会在所有 outer 折上评估，使单个候选的证据不再依赖“它恰好在哪里获胜”。 +- `benchmark`：对照策略在该 outer 测试窗的独立评估，与因子赛道成败无关；即使因子赛道在某折没有任何候选完成内部验证，对照行仍会写入。 + +对照策略是固定定义，不需要逐折重拟合，因此每个 outer 折只评估一次；预算耗尽时写入带原因的 skipped 行，不会静默缺失。探索档（exploratory）结果固定为低置信度，只能保存为 pending 候选。 + +## 晋级与发布门槛 + +保存（promote）始终允许，结果进入研究候选库 pending 状态。发布（publish）在服务端强制校验以下证据门槛，不满足即拒绝并返回原因列表： + +- 非探索档置信度（exploratory 运行产生的候选不能发布）； +- 至少 2 个有效 outer 折； +- 正收益折比例不低于 2/3； +- 样本外 Sharpe 不低于 0.5； +- 最大回撤不劣于 -25%； +- 样本外交易数不低于 60。 + +门槛按 artifact 行指标在读取时计算，不改变历史 artifact 的 schema，因此旧运行的候选会得到相同的门槛判断。工作台会在候选上显示“达标/未达标”标记，未达标的候选“显式发布”按钮被禁用并展示原因。 + +## 置信度 + +三个预算档使用不同的训练窗口： + +| 档位 | 外层训练 | 外层测试 | 步长 | 用途 | +| --- | ---: | ---: | ---: | --- | +| exploratory | 126 | 63 | 63 | 数据较短时探索，只能作为低置信度 pending 候选 | +| balanced | 504 | 126 | 63 | 默认自动研究，至少需要 3 个 outer folds | +| strict | 756 | 126 | 126 | 更长训练窗，至少需要 3 个 outer folds | + +探索性结果不是已验证策略。候选晋级仍需同时检查正收益折比例、样本外 Sharpe、最大回撤、交易数和环境样本覆盖；发布动作会按上文“晋级与发布门槛”在服务端强制执行同一组阈值。 + +## 任务与资源隔离 + +挖掘通过 `spawn` 子进程执行，不在 FastAPI 请求线程、实时行情回调线程或浏览器连接生命周期内运行。浏览器断开不会取消任务；刷新后可通过持久 run ID 重连。取消请求会进入 `cancelling`，界面应等待后端进入终态。 + +共享重任务限流容量为 2：普通回测、优化、walk-forward 和矩阵预热占 1，挖掘独占 2。因此一个挖掘任务不会与另一项大矩阵计算并发。 + +当前 run store 和重任务 limiter 使用进程内锁。生产环境必须只启动一个应用进程负责挖掘；多 worker Uvicorn 部署不能保证跨进程单飞、事件追加和容量限制。要支持多应用进程，必须先增加操作系统级文件锁或外部任务协调器。 + +每个运行目录包含 manifest、compact summary、最近 256 条事件和四个 Parquet artifact。大面板、完整矩阵对象和逐折宽结果不会通过 worker IPC 返回。 + +## 自动运行 + +周度自动挖掘默认关闭，只允许 balanced 或 strict。启用后，它会在北京时间配置工作日及其后的同周工作日、日线 enriched 刷新和环境更新成功后尝试入队；如果配置日的数据流水线失败，后续工作日可以补跑。 + +同一 ISO 周使用确定性的 claim，只触发一次。已经生成运行记录的失败或 `skipped_prerequisite` 仍会占用该周 claim，不重复消耗资源。数据或 T-1 环境覆盖不足时会生成可见的 `skipped_prerequisite` 运行，不会静默降级到 exploratory。挖掘失败不改变已经成功的日度数据流水线状态。 + +自动任务只生成 pending 结果，永远不会自动发布策略。 + +## 保存与发布 + +“保存候选”只从服务端已注册的 `candidates.parquet` 读取定义，并重新校验 artifact schema、canonical signature、原始请求中的因子/策略范围和当前策略兼容性，再写入研究候选库。客户端只能提交 run ID 与 candidate signature，不能在保存时重交权重、方向、公式或代码。保存按 `(origin_run_id, candidate_signature)` 幂等；如果候选库已写入但 artifact backlink 回写失败，重试只修复 backlink，不会创建重复候选。 + +“发布”是独立的显式动作： + +- 已有策略候选返回原策略 ID，不复制或覆盖源文件。 +- 因子组合候选由服务端根据 run ID 与 candidate signature 派生独立策略 ID；客户端不能提交策略 ID、权重、方向、公式或代码。 +- 不同 run 即使得到相同组合，也会生成互相独立的策略文件；同一 run 与 signature 重试时只验证已有源码并修复 backlink。 +- 发布过程执行 AST/META 校验、create-only 原子写入、引擎 reload、策略缓存失效和监控状态失效。 +- reload 或运行时失效失败会回滚首次创建的新文件；backlink 回写失败不会删除已成功加载的策略，重试可修复 backlink。 +- 发布不会修改共享 `factor_rank_research` 源码或 override。 + +发布仅表示把固定候选变成可使用策略，不代表样本外表现会在未来持续。 + +## 性能口径 + +相关矩阵按交易日计算 pairwise-finite Spearman，只聚合因子平方级的逐日结果，不物化百万行永久 rank 宽表。缺失集合不同的因子会先取共同 finite 样本再排名；无法估计的 pair 保持空值，不按零处理。 + +生产 runtime 只生成本次搜索 horizon 的一列 forward label，并在阶段间释放中间列。内存表示优化对照 run `46ed81d537e9404baa06324a3ac3a45f` 在 132.0 万行、243 个交易日、44 个因子上峰值 RSS 为 1.291 GiB，总耗时 25.140 秒；相对 2.395 GiB、37.358 秒的原始对照基线，RSS 和耗时分别下降 46.09% 和 32.70%，四个 Parquet artifact 逐项完全一致。 + +最终 `mining-v2` 额外修复了缺失掩码下的 pairwise Spearman 口径，因此 correlation artifact 与 v1 不再要求数值相等。最终 run `10574abb5d8e4b32ade7cbdd23975c45` 峰值 RSS 为 1.350 GiB、总耗时 40.699 秒，相对原始对照基线内存下降 43.63%、耗时增加 8.94%；仍满足 1.5 GiB 内存门槛和统计增强耗时不超过 30% 的目标。v2 的 Float32 聚合与 Float64 慢参考具有相同 pair counts、空值位置和 0.75 剪枝判断，相关系数最大绝对误差为 `1.70e-7`。 + +这些结果只证明上述基准工作负载达到目标，不表示所有数据规模和搜索预算都具有固定内存上界。当前验证结果：后端全量测试 `876 passed, 24 warnings`，挖掘相关回归含基准轨、跨折证据与晋级门槛共 145 项通过，前端 TypeScript 检查和生产构建均通过；另以真实数据完成 exploratory 运行核验 `selected`/`cross`/`benchmark` 三类行与发布禁用。 diff --git a/docs/strategy.md b/docs/strategy.md index d2b3c1e..a63e454 100644 --- a/docs/strategy.md +++ b/docs/strategy.md @@ -16,6 +16,8 @@ | 量价 / 涨停 | 量价齐升 · 高换手强势 · 连板股 · 断板反包 · 涨停动量 · 接近涨停 | | 反转 / 波动 | 超跌反弹 · 超卖反转 · 新低反转 · 低波动龙头 · 回踩 MA20 · 回踩支撑 · 强势开盘 | +内置目录 `backend/app/strategy/builtin/` 还包含一个仅供挖掘 worker 使用的受控因子排名研究模板。它不出现在普通选股列表，也不能通过普通策略 API 直接运行或保存 override；挖掘结果发布时会生成独立策略。详见 [因子与策略挖掘](./mining.md)。 + 内置目录 `backend/app/strategy/builtin/` 由项目维护,**AI 生成的策略不会落入此目录**。 --- diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index feca72d..d7a9268 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -25,6 +25,7 @@ import { Star, ScanSearch, History, + Pickaxe, FileText, Settings, Key, @@ -79,6 +80,7 @@ const nav = [ { to: '/watchlist', label: '自选', icon: Star }, { to: '/screener', label: '策略', icon: ScanSearch }, { to: '/backtest', label: '回测', icon: History }, + { to: '/mining', label: '挖掘', icon: Pickaxe }, { to: '/stock-analysis', label: '个股分析', icon: TrendingUp }, { to: '/limit-ladder', label: '连板梯队', icon: Flame }, { to: '/concept-analysis', label: '概念分析', icon: Layers3 }, @@ -424,12 +426,11 @@ export function Layout() { : (dataSources?.custom?.find(s => s.name === activeProvider)?.display_name || activeProvider) const isCustomActive = activeProvider !== 'tickflow' - // 轮询触发记录总数 → 更新监控中心徽标 (每 15 秒) + // 轮询触发记录总数 → 更新监控中心徽标 (每 15 秒; 后台标签页由 SSE 事件驱动, 不轮询) const alertsTotalQuery = useQuery({ queryKey: ['alerts-total'], queryFn: () => api.alertsList({ days: 7, limit: 1 }), refetchInterval: 15000, - refetchIntervalInBackground: true, select: (data) => data.total, }) // 只在拿到真实总数时同步徽标 (避免 data=undefined 时传 0 重置 lastSeen) @@ -456,11 +457,27 @@ export function Layout() { const navItems = savedOrder.length > 0 ? (() => { const byTo = new Map(allNav.map(n => [n.to, n])) - const ordered = savedOrder + const ordered = (savedOrder .map(id => byTo.get(id) ?? byTo.get(`/analysis/${id}`)) - .filter(Boolean) - const seen = new Set(ordered.map(n => n!.to)) - return [...ordered as typeof allNav, ...allNav.filter(n => !seen.has(n.to))] + .filter(Boolean)) as typeof allNav + const seen = new Set(ordered.map(n => n.to)) + const merged = [...ordered] + for (const item of allNav) { + if (seen.has(item.to)) continue + // 未保存过排序的新条目: 内置页插回默认位置(排在已保存的默认前驱之后), + // 分析/扩展菜单仍追加到末尾 + const defaultIndex = nav.findIndex(n => n.to === item.to) + let anchor = -1 + if (defaultIndex > 0) { + for (let i = defaultIndex - 1; i >= 0 && anchor < 0; i -= 1) { + anchor = merged.findIndex(n => n.to === nav[i].to) + } + } + if (anchor >= 0) merged.splice(anchor + 1, 0, item) + else if (defaultIndex >= 0) merged.unshift(item) + else merged.push(item) + } + return merged })() : allNav diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 8433d58..38e0ef9 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -473,6 +473,15 @@ export interface RegimeRow { speculation_score?: number resilience_score?: number trend_score?: number + // 情绪周期阶段与梯队指标(重算后才有; 旧数据可能缺) + phase?: MarketPhase | null + first_board?: number | null + ge2_count?: number | null + ge3_count?: number | null + ge5_count?: number | null + ladder_completeness?: number | null + promo_rate?: number | null + promo_pool?: number | null } export interface RegimeHistory { @@ -498,6 +507,90 @@ export interface RegimeCoverage { latest_date: string | null } +// ── 市场阶段(情绪周期) 与 主线 ── +export type MarketPhase = 'ice' | 'ignite' | 'rally' | 'climax' | 'ebb' | 'repair' + +export const MARKET_PHASE_LABELS: Record = { + ice: '冰点', + ignite: '启动', + rally: '主升', + climax: '高潮', + ebb: '退潮', + repair: '修复', +} + +export const MARKET_PHASE_COLORS: Record = { + ice: '#38bdf8', // 天蓝(冻结) + ignite: '#f59e0b', // 琥珀(升温) + rally: '#ef4444', // 红(主升) + climax: '#d946ef', // 品红(极端) + ebb: '#14b8a6', // 青(退潮) + repair: '#94a3b8', // 灰(修复) +} + +export const MARKET_PHASE_ORDER: MarketPhase[] = ['ice', 'ignite', 'rally', 'climax', 'ebb', 'repair'] + +export interface MainlineMemberStat { + member: string + top5_days: number + score_sum: number + max_boards: number + leader_symbol: string +} + +export interface PhaseSegment { + phase: MarketPhase + label: string + start: string + end: string + days: number + avg_height: number + avg_first_board: number + avg_ge2: number + avg_promo: number | null + avg_seal_rate: number + top_mainlines: MainlineMemberStat[] +} + +export interface PhaseSegments { + segments: PhaseSegment[] + total: number +} + +export interface MainlineRow { + date: string + kind: string + member: string + limit_up_count: number + ge2_count: number + max_boards: number + boards_sum: number + rungs_filled: number + leader_symbol: string + score: number + rank: number +} + +export interface MainlineLeader { + member: string + top1_days: number + avg_score: number + max_boards: number +} + +export interface MainlineFilter { + min_members: number + max_members: number + blacklist: string[] +} + +export interface MainlineResult { + rows: MainlineRow[] + leaders: MainlineLeader[] + membership_note: string + filter: MainlineFilter +} + // ===== 大盘复盘 ===== export interface AiReviewReport { id: string @@ -846,6 +939,217 @@ export interface FactorBatchResult { error: string | null } +// ===== Factor / strategy mining ===== +export type MiningBudgetProfile = 'exploratory' | 'balanced' | 'strict' +export type MiningRunStatus = + | 'queued' + | 'running' + | 'cancelling' + | 'succeeded' + | 'succeeded_with_budget_exhausted' + | 'failed' + | 'cancelled' + | 'interrupted' + | 'skipped_prerequisite' + +export interface MiningAvailability { + asset_type: 'stock' | 'etf' + budget_profile: MiningBudgetProfile + trading_bars: number + required_bars: number + outer_folds: number + required_outer_folds: number + eligible: boolean + available_start: string | null + available_end: string | null + effective_start: string | null + effective_end: string | null + suggested_start: string | null +} + +export interface MiningRequestV1 { + factor_names: string[] + strategy_ids?: string[] + symbols?: string[] | null + asset_type?: 'stock' | 'etf' + start?: string | null + end?: string | null + budget_profile?: MiningBudgetProfile + commission_pct?: number + stamp_tax_pct?: number + slippage_bps?: number + correlation_threshold?: number + max_combination_factors?: number + beam_width?: number + max_finalists?: number + force?: boolean +} + +export interface MiningRunProgress { + phase: string + label?: string + done?: number + total?: number + percent?: number + elapsed_ms?: number + message?: string +} + +export interface MiningRun { + run_id: string + signature: string + status: MiningRunStatus + request: MiningRequestV1 + source?: 'manual' | 'scheduled' + created_at: string + updated_at: string + started_at?: string | null + finished_at?: string | null + data_as_of?: string | null + progress?: MiningRunProgress | null + error?: string | null + reused?: boolean + summary?: MiningResultSummary | null +} + +export interface MiningResultSummary { + factor_count: number + selected_factor_count: number + candidate_count: number + valid_fold_count: number + skipped_fold_count: number + confidence: 'low' | 'standard' | 'high' + budget_exhausted?: boolean + elapsed_ms?: number + peak_rss_bytes?: number +} + +export interface MiningFactorRow { + factor_name: string + label?: string + direction: 1 | -1 + score: number | null + ic_mean: number | null + ir: number | null + coverage: number | null + turnover: number | null + spread_return?: number | null + spread_sharpe?: number | null + selected: boolean + excluded_reason?: string | null +} + +export interface MiningRegimeRow { + state: 'overall' | 'strong' | 'range' | 'weak' | string + label: string + n_dates: number + total_return: number | null + sharpe: number | null + max_drawdown: number | null +} + +export interface MiningFoldRow { + fold: number + label?: string + train_start?: string + train_end?: string + test_start?: string + test_end?: string + selected_factors?: string[] + total_return: number | null + sharpe: number | null + max_drawdown?: number | null + n_trades?: number | null + skipped?: boolean + reason?: string | null + evaluation_kind?: 'selected' | 'cross' | 'benchmark' | null +} + +export interface MiningCandidateGate { + qualified: boolean + reasons: string[] +} + +export interface MiningCandidateRow { + signature: string + name: string + kind: 'factor_combination' | 'existing_strategy' + factor_names?: string[] + strategy_id?: string | null + regime_state?: string | null + score: number | null + oos_return: number | null + oos_sharpe: number | null + oos_max_drawdown: number | null + oos_positive_fold_ratio: number | null + oos_n_trades: number | null + confidence: 'low' | 'standard' | 'high' + valid_folds?: number | null + skipped_folds?: number | null + promoted_candidate_id?: string | null + published_strategy_id?: string | null + gate?: MiningCandidateGate | null + folds?: MiningFoldRow[] +} + +export interface MiningTelemetry { + elapsed_ms?: number + peak_rss_bytes?: number + panel_scans?: number + matrix_bytes?: number + cache_hits?: number + fold_reuses?: number + serialized_result_bytes?: number + phase_ms?: Record +} + +export interface MiningRequestSummary { + asset_type: string + budget_profile: string + start: string | null + end: string | null + factor_count: number + strategy_count: number + commission_pct: number | null + stamp_tax_pct: number | null + slippage_bps: number | null + correlation_threshold: number | null +} + +export interface MiningResult { + run_id: string + methodology_version: string + algorithm_version: string + data_as_of: string | null + summary: MiningResultSummary + request_summary?: MiningRequestSummary | null + factors: MiningFactorRow[] + correlation: { + labels: string[] + matrix: (number | null)[][] + pair_counts?: (number | null)[][] + threshold: number + } + regimes: MiningRegimeRow[] + candidates: MiningCandidateRow[] + folds: MiningFoldRow[] + telemetry: MiningTelemetry +} + +export interface MiningEvent { + id: number + type: string + timestamp?: string + payload?: Record + message?: string +} + +export interface MiningScheduleConfig { + mining_schedule_enabled: boolean + mining_schedule_weekday: number + mining_budget_profile: Exclude +} + export type ResearchCandidateKind = 'factor' | 'strategy' export type ResearchCandidateStatus = 'pending' | 'validated' | 'rejected' @@ -1751,8 +2055,28 @@ export const api = { if (start) params.set('start', start) if (end) params.set('end', end) const qs = params.toString() - return request<{ ok: boolean; computed: number }>(`/api/regime/recompute${qs ? `?${qs}` : ''}`, { method: 'POST' }) + return request<{ ok: boolean; computed: number; phase_days?: number; mainline_rows?: number }>(`/api/regime/recompute${qs ? `?${qs}` : ''}`, { method: 'POST' }) }, + regimePhases: (start?: string, end?: string) => { + const params = new URLSearchParams() + if (start) params.set('start', start) + if (end) params.set('end', end) + const qs = params.toString() + return request(`/api/regime/phases${qs ? `?${qs}` : ''}`) + }, + regimeMainline: (start?: string, end?: string, top = 10, kind: 'concept' | 'industry' = 'concept') => { + const params = new URLSearchParams({ top: String(top), kind }) + if (start) params.set('start', start) + if (end) params.set('end', end) + return request(`/api/regime/mainline?${params.toString()}`) + }, + regimeMainlineRecompute: () => + request<{ ok: boolean; rows: number }>('/api/regime/mainline/recompute', { method: 'POST' }), + mainlineFilterUpdate: (payload: { min_members?: number; max_members?: number; blacklist?: string[] }) => + request('/api/settings/preferences/mainline-filter', { + method: 'PUT', + body: JSON.stringify(payload), + }), limitLadder: (asOf?: string, extColumns?: string, direction?: 'up' | 'down') => { const params = new URLSearchParams() @@ -1820,6 +2144,64 @@ export const api = { body: JSON.stringify(payload), }), + miningRuns: () => + request<{ items: MiningRun[] }>('/api/backtest/mining/runs'), + + miningAvailability: (params: { + assetType: 'stock' | 'etf' + budgetProfile: MiningBudgetProfile + start?: string + end?: string + }) => { + const query = new URLSearchParams({ + asset_type: params.assetType, + budget_profile: params.budgetProfile, + }) + if (params.start) query.set('start', params.start) + if (params.end) query.set('end', params.end) + return request(`/api/backtest/mining/availability?${query}`, { + quiet: true, + }) + }, + + miningRun: (runId: string) => + request(`/api/backtest/mining/runs/${encodeURIComponent(runId)}`), + + miningStart: (payload: MiningRequestV1) => + request('/api/backtest/mining/runs', { + method: 'POST', + body: JSON.stringify(payload), + }), + + miningResult: (runId: string) => + request(`/api/backtest/mining/runs/${encodeURIComponent(runId)}/result`), + + miningCancel: (runId: string) => + request(`/api/backtest/mining/runs/${encodeURIComponent(runId)}/cancel`, { + method: 'POST', + }), + + miningPromote: (runId: string, signature: string) => + request( + `/api/backtest/mining/runs/${encodeURIComponent(runId)}/candidates/${encodeURIComponent(signature)}/promote`, + { method: 'POST' }, + ), + + miningPublish: (runId: string, signature: string) => + request<{ ok: boolean; strategy_id: string }>( + `/api/backtest/mining/runs/${encodeURIComponent(runId)}/candidates/${encodeURIComponent(signature)}/publish`, + { method: 'POST' }, + ), + + miningConfig: () => + request('/api/backtest/mining/config'), + + updateMiningConfig: (payload: Partial) => + request('/api/backtest/mining/config', { + method: 'PATCH', + body: JSON.stringify(payload), + }), + researchCandidates: () => request<{ items: ResearchCandidate[] }>('/api/backtest/candidates'), diff --git a/frontend/src/lib/miningTask.ts b/frontend/src/lib/miningTask.ts new file mode 100644 index 0000000..10b23e8 --- /dev/null +++ b/frontend/src/lib/miningTask.ts @@ -0,0 +1,376 @@ +import { useSyncExternalStore } from 'react' +import { api, type MiningResult, type MiningRun, type MiningRunProgress, type MiningRunStatus } from './api' + +export interface MiningTask { + runId: string | null + isPending: boolean + cancelling: boolean + reconnecting: boolean + run: MiningRun | null + progress: MiningRunProgress | null + result: MiningResult | null + previousResult: MiningResult | null + error: string | null +} + +const ACTIVE_RUN_KEY = 'mining_active_run_id' +const TERMINAL_STATES = new Set([ + 'succeeded', + 'succeeded_with_budget_exhausted', + 'failed', + 'cancelled', + 'interrupted', + 'skipped_prerequisite', +]) +const SUCCESS_STATES = new Set([ + 'succeeded', + 'succeeded_with_budget_exhausted', +]) +const STATUS_POLL_INTERVAL_MS = 2000 + +let current: MiningTask = { + runId: null, + isPending: false, + cancelling: false, + reconnecting: false, + run: null, + progress: null, + result: null, + previousResult: null, + error: null, +} +let eventSource: EventSource | null = null +let connectionToken = 0 +let statusPoll: { + runId: string + token: number + timer: ReturnType | null +} | null = null +const listeners = new Set<() => void>() + +function emit() { + listeners.forEach(listener => listener()) +} + +function update(patch: Partial) { + current = { ...current, ...patch } + emit() +} + +function subscribe(listener: () => void) { + listeners.add(listener) + return () => listeners.delete(listener) +} + +function stopStatusPolling() { + if (statusPoll?.timer) clearTimeout(statusPoll.timer) + statusPoll = null +} + +function closeEvents() { + connectionToken += 1 + stopStatusPolling() + eventSource?.close() + eventSource = null +} + +function eventPayload(event: MessageEvent): Record { + try { + const parsed = JSON.parse(event.data) + if (parsed && typeof parsed === 'object') { + return parsed.payload && typeof parsed.payload === 'object' + ? { ...parsed, ...parsed.payload } + : parsed + } + } catch { /* ignore malformed progress events */ } + return {} +} + +async function refreshTerminalRun( + runId: string, + fallbackStatus?: MiningRunStatus, + knownRun?: MiningRun, + token = connectionToken, + fallbackError?: string, +) { + try { + const run = knownRun ?? await api.miningRun(runId) + let result: MiningResult | null = null + if (SUCCESS_STATES.has(run.status)) { + result = await api.miningResult(runId) + if (result.run_id !== runId) throw new Error('任务结果与运行 ID 不匹配') + } + if (current.runId !== runId || connectionToken !== token) return + localStorage.removeItem(ACTIVE_RUN_KEY) + update({ + run, + progress: run.progress ?? current.progress, + result, + isPending: false, + cancelling: false, + reconnecting: false, + error: run.error || fallbackError || null, + }) + } catch (error) { + if (current.runId !== runId || connectionToken !== token) return + localStorage.removeItem(ACTIVE_RUN_KEY) + const run = current.run && fallbackStatus + ? { ...current.run, status: fallbackStatus, error: fallbackError || current.run.error } + : current.run + update({ + run, + result: null, + isPending: false, + cancelling: false, + reconnecting: false, + error: fallbackStatus === 'cancelled' + ? '任务已取消' + : fallbackError || String((error as Error).message || error), + }) + } +} + +function startStatusPolling(runId: string, token: number, restart = false) { + if (restart) stopStatusPolling() + if (statusPoll?.runId === runId && statusPoll.token === token) return + stopStatusPolling() + + const poll = { runId, token, timer: null as ReturnType | null } + statusPoll = poll + + const pollStatus = async () => { + if ( + statusPoll !== poll + || current.runId !== runId + || connectionToken !== token + || !current.isPending + ) return + + try { + const run = await api.miningRun(runId) + if (statusPoll !== poll || current.runId !== runId || connectionToken !== token) return + update({ + run, + progress: run.progress ?? current.progress, + cancelling: current.cancelling || run.status === 'cancelling', + }) + if (TERMINAL_STATES.has(run.status)) { + closeEvents() + const terminalToken = connectionToken + await refreshTerminalRun(runId, run.status, run, terminalToken) + return + } + } catch { + // EventSource keeps reconnecting; polling is only a bounded status fallback. + } + + if ( + statusPoll !== poll + || current.runId !== runId + || connectionToken !== token + || !current.isPending + ) return + poll.timer = setTimeout(() => { + poll.timer = null + void pollStatus() + }, STATUS_POLL_INTERVAL_MS) + } + + void pollStatus() +} + +function connect(runId: string) { + closeEvents() + const token = connectionToken + const source = new EventSource(`/api/backtest/mining/runs/${encodeURIComponent(runId)}/events`) + eventSource = source + + source.onopen = () => { + if (token !== connectionToken) return + update({ reconnecting: false }) + if (!current.cancelling) stopStatusPolling() + } + + source.addEventListener('progress', event => { + if (token !== connectionToken) return + update({ + progress: eventPayload(event as MessageEvent) as unknown as MiningRunProgress, + reconnecting: false, + }) + if (!current.cancelling) stopStatusPolling() + }) + + const onTerminal = (event: Event) => { + if (token !== connectionToken) return + const payload = eventPayload(event as MessageEvent) + const eventType = (event as MessageEvent).type + const status = (payload.status || eventType) as MiningRunStatus + closeEvents() + const terminalToken = connectionToken + void refreshTerminalRun( + runId, + status, + undefined, + terminalToken, + typeof payload.message === 'string' ? payload.message : undefined, + ) + } + for (const type of [ + 'succeeded', + 'succeeded_with_budget_exhausted', + 'failed', + 'cancelled', + 'interrupted', + 'skipped_prerequisite', + ]) { + source.addEventListener(type, onTerminal) + } + + source.onerror = () => { + if (token !== connectionToken || !current.isPending) return + update({ reconnecting: true }) + startStatusPolling(runId, token) + } +} + +export async function startMining(payload: Parameters[0]) { + closeEvents() + localStorage.removeItem(ACTIVE_RUN_KEY) + const token = connectionToken + update({ + runId: null, + isPending: true, + cancelling: false, + reconnecting: false, + run: null, + progress: { phase: 'queued', label: '创建任务' }, + result: null, + previousResult: current.result ?? current.previousResult, + error: null, + }) + try { + const run = await api.miningStart(payload) + if (connectionToken !== token || current.runId !== null) return + localStorage.setItem(ACTIVE_RUN_KEY, run.run_id) + update({ + runId: run.run_id, + run, + progress: run.progress ?? current.progress, + isPending: !TERMINAL_STATES.has(run.status), + }) + if (TERMINAL_STATES.has(run.status)) { + await refreshTerminalRun(run.run_id, run.status, run, token) + } else { + connect(run.run_id) + } + } catch (error) { + if (connectionToken !== token || current.runId !== null) return + update({ + isPending: false, + error: String((error as Error).message || error), + }) + } +} + +export async function cancelMining() { + if (!current.runId || !current.isPending || current.cancelling) return + const runId = current.runId + const token = connectionToken + update({ cancelling: true, error: null }) + startStatusPolling(runId, token, true) + try { + const run = await api.miningCancel(runId) + if (current.runId !== runId || connectionToken !== token) return + update({ + run, + progress: run.progress ?? current.progress, + cancelling: !TERMINAL_STATES.has(run.status), + }) + if (TERMINAL_STATES.has(run.status)) { + closeEvents() + const terminalToken = connectionToken + await refreshTerminalRun(runId, run.status, run, terminalToken) + } + } catch (error) { + if (current.runId !== runId || connectionToken !== token) return + update({ + cancelling: true, + reconnecting: true, + error: String((error as Error).message || error), + }) + startStatusPolling(runId, token, true) + } +} + +export async function attachMiningRun(runId: string): Promise { + closeEvents() + const token = connectionToken + const previousResult = current.result ?? current.previousResult + update({ + runId, + isPending: true, + cancelling: false, + reconnecting: true, + run: null, + progress: { phase: 'reconnecting', label: '读取任务状态' }, + result: null, + previousResult, + error: null, + }) + try { + const run = await api.miningRun(runId) + if (current.runId !== runId || connectionToken !== token) return false + update({ + run, + progress: run.progress ?? null, + isPending: !TERMINAL_STATES.has(run.status), + cancelling: run.status === 'cancelling', + reconnecting: false, + error: run.error ?? null, + }) + if (TERMINAL_STATES.has(run.status)) { + await refreshTerminalRun(runId, run.status, run, token) + } else { + localStorage.setItem(ACTIVE_RUN_KEY, runId) + connect(runId) + } + return true + } catch (error) { + if (current.runId !== runId || connectionToken !== token) return false + localStorage.removeItem(ACTIVE_RUN_KEY) + update({ + isPending: false, + reconnecting: false, + error: String((error as Error).message || error), + }) + return false + } +} + +export function tryReconnectMining(): boolean { + const runId = localStorage.getItem(ACTIVE_RUN_KEY) + if (!runId) return false + void attachMiningRun(runId) + return true +} + +export function clearMiningTask() { + closeEvents() + localStorage.removeItem(ACTIVE_RUN_KEY) + current = { + runId: null, + isPending: false, + cancelling: false, + reconnecting: false, + run: null, + progress: null, + result: null, + previousResult: current.result ?? current.previousResult, + error: null, + } + emit() +} + +export function useMiningTask(): MiningTask { + return useSyncExternalStore(subscribe, () => current, () => current) +} diff --git a/frontend/src/lib/queryKeys.ts b/frontend/src/lib/queryKeys.ts index 073eef0..1e048a4 100644 --- a/frontend/src/lib/queryKeys.ts +++ b/frontend/src/lib/queryKeys.ts @@ -26,7 +26,11 @@ export const QK = { watchlistGroups: ['watchlist-groups'] as const, watchlistQuotes: ['watchlist-quotes'] as const, watchlistEnriched: (ext?: string) => ['watchlist-enriched', ext] as const, - watchlistKlineBatch: (symbols: string) => ['watchlist-kline-batch', symbols] as const, + // 不用 watchlist- 前缀: 日K历史盘中几乎不变, 若被 SSE quotes_updated 高频失效 + // (expert 1s) 会导致全自选日K每秒重拉, staleTime 形同虚设。 + // 刷新点: staleTime 过期 + Watchlist 增删自选/改蜡烛天数时的手动失效; + // 当日最后一根蜡烛由 Watchlist 用 enriched 实时 OHLC 前端修补 (零额外请求)。 + watchlistKlineBatch: (symbols: string) => ['kline-batch', symbols] as const, // 不用 watchlist- 前缀: 避免被 SSE quotes_updated 高频失效(expert 1s/pro 2s) // 导致每次都拉 TickFlow 触限流。分时图用固定 refetchInterval 刷新即可。 minuteBatch: (symbols: string) => ['minute-batch', symbols] as const, @@ -45,8 +49,16 @@ export const QK = { // Backtest backtestStatus: ['backtest-status'] as const, factorColumns: ['backtest-factor-columns'] as const, + miningRuns: ['backtest-mining-runs'] as const, + miningAvailability: (assetType: string, profile: string, start: string, end: string) => + ['backtest-mining-availability', assetType, profile, start, end] as const, + miningRun: (id: string) => ['backtest-mining-run', id] as const, + miningResult: (id: string) => ['backtest-mining-result', id] as const, + miningConfig: ['backtest-mining-config'] as const, researchCandidates: ['research-candidates'] as const, - strategyLinkOptions: ['strategy-link-options'] as const, + strategyLinkOptions: (assetType?: 'stock' | 'etf') => assetType + ? ['strategy-link-options', assetType] as const + : ['strategy-link-options'] as const, strategyDetail: (id: string) => ['strategy-detail', id] as const, // Data / Pipeline @@ -96,6 +108,8 @@ export const QK = { regimeLatest: ['regime-latest'] as const, regimeStates: (days: number) => ['regime-states', days] as const, regimeCoverage: ['regime-coverage'] as const, + regimePhases: (start?: string, end?: string) => ['regime-phases', start ?? '', end ?? ''] as const, + regimeMainline: (kind: string, start?: string, end?: string) => ['regime-mainline', kind, start ?? '', end ?? ''] as const, } as const // ===== SSE 应该 invalidate 的 key 前缀列表 ===== @@ -107,7 +121,11 @@ export const QK = { // 且在 monitor "重算" 窗口内读到空结果, 造成策略列表闪烁 (变 0 → 空失效 → 又出现)。 export const SSE_INVALIDATE_PREFIXES = [ - 'watchlist', + // 精确前缀: 只命中自选页的实时数据 (quotes/enriched)。不能用宽泛的 'watchlist' —— + // 会误伤 ['watchlist'] (自选列表) 和 ['watchlist-groups'] (分组配置, 只随手动操作变化)。 + // 旧设置里的 'watchlist' 单开关由 useQuoteStream 兼容读取。 + 'watchlist-quotes', + 'watchlist-enriched', 'quote-status', 'index-quotes', 'overview-market', diff --git a/frontend/src/lib/useQuoteStream.ts b/frontend/src/lib/useQuoteStream.ts index 867c46f..8eb5432 100644 --- a/frontend/src/lib/useQuoteStream.ts +++ b/frontend/src/lib/useQuoteStream.ts @@ -137,6 +137,13 @@ export function useQuoteStream( const activePrefixes = SSE_INVALIDATE_PREFIXES.filter((p) => { // 'quote-status' 始终刷新 (全局状态) if (p === 'quote-status') return true + // 兼容旧配置: 'watchlist' 拆成两个精确前缀后, 未单独设置时沿用旧 'watchlist' 开关 + if ( + (p === 'watchlist-quotes' || p === 'watchlist-enriched') && + pages[p] === undefined + ) { + return pages['watchlist'] !== false + } return pages[p] !== false }) qc.invalidateQueries({ diff --git a/frontend/src/pages/Backtest.tsx b/frontend/src/pages/Backtest.tsx index 89bf7c3..0936c85 100644 --- a/frontend/src/pages/Backtest.tsx +++ b/frontend/src/pages/Backtest.tsx @@ -1,4 +1,5 @@ import { useState } from 'react' +import { Navigate, useSearchParams } from 'react-router-dom' import { BarChart3, BookmarkCheck, FlaskConical, ShieldCheck } from 'lucide-react' import { PageHeader } from '@/components/PageHeader' import { FactorDiscovery } from './backtest/FactorDiscovery' @@ -27,9 +28,28 @@ const MODES: Record('strategy') + const [searchParams, setSearchParams] = useSearchParams() + const requestedTab = searchParams.get('tab') const [candidatesOpen, setCandidatesOpen] = useState(false) + // 旧链接兼容: 挖掘已升级为一级路由 /mining, 保留 run/candidate 参数重定向 + if (requestedTab === 'mining') { + const next = new URLSearchParams(searchParams) + next.delete('tab') + const search = next.toString() + return + } + + const activeTab: Tab = requestedTab && requestedTab in MODES + ? requestedTab as Tab + : 'strategy' + + const changeTab = (tab: Tab) => { + const next = new URLSearchParams(searchParams) + next.set('tab', tab) + setSearchParams(next, { replace: true }) + } + return (
setActiveTab(tab)} + onClick={() => changeTab(tab)} aria-current={active ? 'page' : undefined} className={`inline-flex h-7 items-center gap-1 rounded-[5px] px-1.5 text-[11px] font-medium transition-colors sm:gap-1.5 sm:px-2.5 sm:text-xs ${active ? 'bg-accent text-white shadow-sm' diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index 641942a..c55eada 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -102,7 +102,6 @@ function MonitorWidget({ onStockClick }: { onStockClick: (event: AlertEvent) => queryKey: ['alerts', ''], queryFn: () => api.alertsList({ days: 7, limit: 10 }), refetchInterval: 10000, - refetchIntervalInBackground: true, }) const events: AlertEvent[] = alerts.data?.alerts ?? [] diff --git a/frontend/src/pages/LimitUpLadder.tsx b/frontend/src/pages/LimitUpLadder.tsx index c9b7f3a..b8b17d9 100644 --- a/frontend/src/pages/LimitUpLadder.tsx +++ b/frontend/src/pages/LimitUpLadder.tsx @@ -1517,7 +1517,10 @@ export function LimitUpLadder() { const extColumnsParam = useMemo(() => buildExtColumnsParam(extFields), [extFields]) const { data, isLoading, refetch, isFetching } = useQuery({ - queryKey: [QK.limitLadder(asOf || undefined), extColumnsParam, direction], + // key 必须拍平 (spread 展开): key[0] 为字符串 'limit-ladder' 才能被 SSE 前缀失效 + // 命中实现实时刷新, depth_updated 事件 (invalidate ['limit-ladder']) 也才能匹配本查询。 + // 嵌套数组 key 会导致前者靠 String() 侥幸命中、后者永远失配。 + queryKey: [...QK.limitLadder(asOf || undefined), extColumnsParam, direction], queryFn: () => api.limitLadder(asOf || undefined, extColumnsParam, direction), staleTime: 5 * 60_000, }) diff --git a/frontend/src/pages/Mining.tsx b/frontend/src/pages/Mining.tsx new file mode 100644 index 0000000..9c7fba2 --- /dev/null +++ b/frontend/src/pages/Mining.tsx @@ -0,0 +1,37 @@ +import { useState } from 'react' +import { BookmarkCheck } from 'lucide-react' +import { PageHeader } from '@/components/PageHeader' +import { MiningWorkbench } from './backtest/MiningWorkbench' +import { ResearchCandidatesDialog } from './backtest/ResearchCandidatesDialog' + +export function Mining() { + const [candidatesOpen, setCandidatesOpen] = useState(false) + + return ( +
+ 嵌套样本外因子与策略挖掘} + className="shrink-0 flex-wrap gap-x-4 gap-y-2 bg-base/95 px-3 lg:flex-nowrap lg:px-5" + right={( + + )} + /> + +
+ +
+ + {candidatesOpen && setCandidatesOpen(false)} />} +
+ ) +} diff --git a/frontend/src/pages/Monitor.tsx b/frontend/src/pages/Monitor.tsx index 5af1d56..ee0b744 100644 --- a/frontend/src/pages/Monitor.tsx +++ b/frontend/src/pages/Monitor.tsx @@ -135,8 +135,8 @@ export function Monitor() { const alertsQuery = useQuery({ queryKey: [...QK.alerts(filter === 'all' ? undefined : filter), extColumnsParam ?? ''], queryFn: () => api.alertsList({ days: 7, limit: 500, source: filter === 'all' ? undefined : filter, extColumns: extColumnsParam }), + // 10s 轮询仅作 SSE strategy_alert 事件的兜底; 后台标签页不再拉 500 条全量 refetchInterval: 10000, - refetchIntervalInBackground: true, }) const total = alertsQuery.data?.total ?? 0 @@ -351,7 +351,7 @@ function AlertsList({ alertsQuery, confirmClear, setConfirmClear, total, enterTs const isNew = ev.ts > enterTs return ( api.regimeStates(days), staleTime: 5 * 60 * 1000, }) + // 情绪周期阶段段 + 主线排行(与 history 同一时间范围) + const phases = useQuery({ + queryKey: QK.regimePhases(histRange.start, histRange.end), + queryFn: () => api.regimePhases(histRange.start, histRange.end), + staleTime: 5 * 60 * 1000, + }) + const [mainlineKind, setMainlineKind] = useState<'concept' | 'industry'>('concept') + const [filterOpen, setFilterOpen] = useState(false) + const mainline = useQuery({ + queryKey: QK.regimeMainline(mainlineKind, histRange.start, histRange.end), + queryFn: () => api.regimeMainline(histRange.start, histRange.end, 10, mainlineKind), + staleTime: 5 * 60 * 1000, + }) const [recomputing, setRecomputing] = useState(false) const rows: RegimeRow[] = history.data?.rows ?? [] const latest = rows.length > 0 ? rows[rows.length - 1] : null + const hasPhaseData = rows.length > 0 && rows.some(r => r.phase != null) + const segments = phases.data?.segments ?? [] + + // 当前阶段持续天数(末尾连续同阶段) + 当前主线(最新交易日 top3) + const phaseStreak = useMemo(() => { + if (!hasPhaseData) return null + const lastPhase = rows[rows.length - 1].phase + let streak = 1 + for (let i = rows.length - 2; i >= 0; i--) { + if (rows[i].phase === lastPhase) streak++ + else break + } + return { phase: lastPhase as MarketPhase, streak } + }, [rows, hasPhaseData]) + const latestMainlines = useMemo(() => { + const mlRows = mainline.data?.rows ?? [] + if (mlRows.length === 0) return [] + const lastDate = mlRows[mlRows.length - 1].date + return mlRows.filter(r => r.date === lastDate && r.rank <= 3) + }, [mainline.data]) // ── 当前势头: 末尾连续同态天数 + score 5日斜率(改善/恶化) + 上次弱势距今 ── const momentum = useMemo(() => { @@ -185,6 +219,77 @@ export function Regime() { + // 阶段时间轴: 高度折线 + 2板以上宽度柱 + 晋级率曲线, 背景色带=情绪周期阶段 + const phaseOption = useMemo(() => { + if (rows.length === 0 || !hasPhaseData) return null + const dates = rows.map(r => r.date) + const heights = rows.map(r => r.max_consecutive) + const ge2 = rows.map(r => r.ge2_count ?? null) + const promo = rows.map(r => (r.promo_rate != null ? Math.round(r.promo_rate * 100) : null)) + const phaseBands: any[] = [] + let bandStart = rows[0]?.date + let prevPhase = rows[0]?.phase + rows.forEach((r, i) => { + if (r.phase !== prevPhase || i === rows.length - 1) { + const bandEnd = i === rows.length - 1 ? r.date : rows[i - 1].date + if (prevPhase && MARKET_PHASE_COLORS[prevPhase as MarketPhase]) { + phaseBands.push([ + { xAxis: bandStart, itemStyle: { color: MARKET_PHASE_COLORS[prevPhase as MarketPhase], opacity: 0.10 } }, + { xAxis: bandEnd }, + ]) + } + bandStart = r.date + prevPhase = r.phase + } + }) + return { + backgroundColor: 'transparent', + tooltip: { + trigger: 'axis', backgroundColor: ct.tooltipBg, borderColor: ct.tooltipBorder, + textStyle: { color: ct.tooltipText }, + formatter: (params: any) => { + const p0 = Array.isArray(params) ? params[0] : params + const i = dates.indexOf(p0.axisValue) + const r = rows[i] + if (!r) return '' + const phase = r.phase ? MARKET_PHASE_LABELS[r.phase] : '—' + return [ + `${r.date} · ${phase}`, + `高度 ${r.max_consecutive} · 首板 ${r.first_board ?? '—'} · 2板+ ${r.ge2_count ?? '—'}`, + `晋级率 ${r.promo_rate != null ? (r.promo_rate * 100).toFixed(1) + '%' : '—'} · 封板率 ${r.seal_rate != null ? (r.seal_rate * 100).toFixed(1) + '%' : '—'}`, + ].join('
') + }, + }, + legend: { + data: ['高度', '2板+', '晋级率'], textStyle: { color: ct.text, fontSize: 10 }, top: 0, + }, + grid: { left: 44, right: 44, top: 32, bottom: 44 }, + xAxis: { + type: 'category', data: dates, boundaryGap: false, + axisLabel: { color: ct.text, fontSize: 10, formatter: (v: string) => v.slice(5) }, + axisLine: { lineStyle: { color: ct.grid } }, + }, + yAxis: [ + { type: 'value', name: '高度/宽度', position: 'left', axisLabel: { color: ct.text, fontSize: 10 }, splitLine: { show: false }, nameTextStyle: { color: ct.text } }, + { type: 'value', name: '晋级率%', min: 0, max: 100, position: 'right', axisLabel: { color: ct.text, fontSize: 10 }, splitLine: { lineStyle: { color: ct.grid } }, nameTextStyle: { color: ct.text } }, + ], + dataZoom: [ + { type: 'inside', start: Math.max(0, 100 - (60 / days) * 100) }, + { type: 'slider', bottom: 6, height: 14, borderColor: ct.border, fillerColor: ct.zoomFill, textStyle: { color: ct.text } }, + ], + series: [ + { name: '2板+', type: 'bar', data: ge2, yAxisIndex: 0, barMaxWidth: 5, + itemStyle: { color: '#f59e0b', opacity: 0.4 }, z: 1 }, + { name: '高度', type: 'line', data: heights, smooth: true, symbol: 'none', yAxisIndex: 0, + lineStyle: { width: 1.6, color: '#ef4444' }, z: 3, + markArea: { silent: true, data: phaseBands } }, + { name: '晋级率', type: 'line', data: promo, smooth: true, symbol: 'none', yAxisIndex: 1, + lineStyle: { width: 1.2, color: '#3b82f6', type: 'dotted' }, z: 2 }, + ], + } + }, [rows, days, ct, hasPhaseData]) + const phaseChartRef = useEChart(phaseOption, [phaseOption]) + // 趋势图: 综合分主线 + 4 子维度曲线(可切换) + 状态背景色带 + 涨停数柱状 const trendOption = useMemo(() => { if (rows.length === 0) return null @@ -366,6 +471,8 @@ export function Regime() { qc.invalidateQueries({ queryKey: ['regime-history'] }), qc.invalidateQueries({ queryKey: ['regime-states'] }), qc.invalidateQueries({ queryKey: ['regime-latest'] }), + qc.invalidateQueries({ queryKey: ['regime-phases'] }), + qc.invalidateQueries({ queryKey: ['regime-mainline'] }), qc.invalidateQueries({ queryKey: QK.regimeCoverage }), ]) } catch (e) { @@ -429,6 +536,207 @@ export function Regime() {
+ {/* ── 市场阶段概览 (情绪周期 + 梯队指标 + 当前主线) ── */} + {hasPhaseData && latest ? ( +
+ {/* 当前阶段 */} +
+
+ 当前阶段 · {latest.date} +
+
+ + {MARKET_PHASE_LABELS[phaseStreak?.phase ?? 'repair']} + + {phaseStreak && 第 {phaseStreak.streak} 天} +
+
+ {latestMainlines.length > 0 ? latestMainlines.map(m => ( + + {m.member} + + )) : 暂无主线数据} +
+
+ {([ + { label: '市场高度', val: latest.max_consecutive, unit: '板', color: '#ef4444' }, + { label: '首板宽度', val: latest.first_board, unit: '家', color: '#f97316' }, + { label: '2板+宽度', val: latest.ge2_count, unit: '家', color: '#f59e0b' }, + { label: '晋级率', val: latest.promo_rate != null ? `${(latest.promo_rate * 100).toFixed(0)}%` : '—', + unit: '', color: '#3b82f6', + sub: latest.promo_pool != null ? `池 ${latest.promo_pool} 家` : undefined }, + { label: '梯队完整度', val: latest.ladder_completeness != null ? `${(latest.ladder_completeness * 100).toFixed(0)}%` : '—', + unit: '', color: '#a855f7', sub: '2板→最高板不断档' }, + ] as { label: string; val: React.ReactNode; unit: string; color: string; sub?: string }[]).map(k => ( +
+
+ {k.label} +
+
+ {k.val}{k.unit} +
+ {k.sub &&
{k.sub}
} +
+ ))} +
+ ) : ( +
+ 市场阶段(情绪周期)数据尚未生成 — 点击右上角「重算」即可回填全部历史阶段与主线 +
+ )} + + {/* ── 情绪周期时间轴 (阶段色带 + 高度/宽度/晋级率) ── */} + {hasPhaseData && rows.length > 0 && ( +
+ +
+
+ {rows.map(r => ( +
+ ))} +
+
+ {MARKET_PHASE_ORDER.map(p => ( + + + {MARKET_PHASE_LABELS[p]} + + ))} +
+
+ )} + + {/* ── 阶段 × 主线 (什么阶段走什么主升) ── */} + {segments.length > 0 && ( +
+ +
+ + + + + + + + + + + + + + + {[...segments].reverse().map((seg, i) => ( + + + + + + + + + + + ))} + +
阶段区间天数高度2板+晋级率封板率主导主线
+ + {seg.label} + + + {seg.start.slice(5)} ~ {seg.end.slice(5)} + {seg.days}{seg.avg_height}{seg.avg_ge2} + {seg.avg_promo != null ? `${(seg.avg_promo * 100).toFixed(0)}%` : '—'} + + {seg.avg_seal_rate != null ? `${(seg.avg_seal_rate * 100).toFixed(0)}%` : '—'} + +
+ {seg.top_mainlines.length > 0 ? seg.top_mainlines.map(m => ( + + {m.member}{m.top5_days}d + + )) : —} +
+
+
+
+ )} + + {/* ── 主线排行 (窗口内持续性 + 过滤设置) ── */} +
+ + {mainline.data?.membership_note} + + + } + /> +
+
+ {([['concept', '概念'], ['industry', '行业']] as const).map(([k, label]) => ( + + ))} +
+ 窗口内 top1 天数排序 · 点击「过滤」配置宽基概念屏蔽 +
+ {filterOpen && ( + { + await qc.invalidateQueries({ queryKey: ['regime-mainline'] }) + await qc.invalidateQueries({ queryKey: ['regime-phases'] }) + }} + /> + )} +
+ + + + + + + + + + + + {(mainline.data?.leaders ?? []).map((l, i) => ( + + + + + + + + ))} + {(mainline.data?.leaders ?? []).length === 0 && ( + + )} + +
#主线top1 天数日均分最高板
{i + 1}{l.member}{l.top1_days}{l.avg_score}{l.max_boards} 板
+ {mainline.isLoading ? '加载中…' : '暂无主线数据 — 点击「重算」回填, 或检查过滤设置'} +
+
+
+ {/* ── 最新日概览 (4 个指标卡, 去掉与看板重复的涨停/涨跌/成交额) ── */} {latest ? (
@@ -661,6 +969,91 @@ export function Regime() { ) } +// ── 主线过滤设置面板 ────────────────────────────────────── +// 宽基/风格标签(融资融券/沪深股通等数千成分)会霸占主线榜首。默认按成员数 +// 上限过滤; 用户可调阈值并按名称屏蔽特定概念, 保存后自动重算主线。 +function MainlineFilterPanel({ filter, onDone }: { + filter: { min_members: number; max_members: number; blacklist: string[] } | undefined + onDone: () => Promise +}) { + const [minMembers, setMinMembers] = useState(String(filter?.min_members ?? 4)) + const [maxMembers, setMaxMembers] = useState(String(filter?.max_members ?? 600)) + const [blacklist, setBlacklist] = useState(filter?.blacklist ?? []) + const [input, setInput] = useState('') + const [saving, setSaving] = useState(false) + + const addTag = () => { + const v = input.trim() + if (v && !blacklist.includes(v)) setBlacklist([...blacklist, v]) + setInput('') + } + + const save = async () => { + setSaving(true) + try { + await api.mainlineFilterUpdate({ + min_members: Math.max(1, Number(minMembers) || 4), + max_members: Math.max(50, Number(maxMembers) || 600), + blacklist, + }) + await api.regimeMainlineRecompute() + toast('过滤已保存, 主线已重算', 'success') + await onDone() + } catch (e) { + toast(`保存失败 · ${String((e as Error)?.message || e)}`, 'error') + } finally { + setSaving(false) + } + } + + return ( +
+
+ + +
+ 按名称屏蔽(回车添加) +
+ setInput(e.target.value)} + onKeyDown={e => { if (e.key === 'Enter') { e.preventDefault(); addTag() } }} + placeholder="如: 融资融券、沪股通" + className="h-7 flex-1 rounded-input border border-border bg-base px-2 text-xs text-foreground outline-none focus:border-accent" /> +
+ {blacklist.length > 0 && ( +
+ {blacklist.map(b => ( + + {b} + + + ))} +
+ )} +
+ +
+
+ 说明: 成分股数超过上限的概念(如 融资融券~7700家/沪深股通~3300家)视为宽基/风格标签, 不参与主线排名; 修改后自动重算全部历史主线(秒级)。 +
+
+ ) +} + // ── 自定义天数输入弹窗 ──────────────────────────────────── function CustomDaysModal({ current, onClose, onApply }: { current: number diff --git a/frontend/src/pages/Screener.tsx b/frontend/src/pages/Screener.tsx index b6b533f..0b93868 100644 --- a/frontend/src/pages/Screener.tsx +++ b/frontend/src/pages/Screener.tsx @@ -36,6 +36,9 @@ import { type ColumnConfig, } from '@/lib/screener-columns' +// 获取策略为占位功能, 暂时隐藏入口; 恢复时改回 true +const SHOW_STRATEGY_STORE = false + export function Screener() { const [assetType, setAssetType] = useState<'stock' | 'etf'>('stock') const [activeStrategy, setActiveStrategy] = useState(null) @@ -115,10 +118,12 @@ export function Screener() { setFilter(filterMap.current.get(strategyId) ?? { ...defaultFilter }) }, []) - // 对原始结果应用过滤 - const filteredRows = result - ? applyFilter(result.rows, filter) - : [] + // 对原始结果应用过滤 (memo: 否则每次渲染都对全部结果行过滤, + // 且新数组身份会击穿下游 displayRows 的 memo) + const filteredRows = useMemo( + () => (result ? applyFilter(result.rows, filter) : []), + [result, filter], + ) const { data: prefs } = usePreferences() const screenerAutoRun = prefs?.screener_auto_run ?? true @@ -698,16 +703,18 @@ export function Screener() { 创建策略 · AI - {/* 获取策略（占位，敬请期待） */} - + {/* 获取策略（占位，敬请期待）— 暂时隐藏 */} + {SHOW_STRATEGY_STORE && ( + + )}
} /> diff --git a/frontend/src/pages/Watchlist.tsx b/frontend/src/pages/Watchlist.tsx index 5118768..41643a9 100644 --- a/frontend/src/pages/Watchlist.tsx +++ b/frontend/src/pages/Watchlist.tsx @@ -787,7 +787,32 @@ export function Watchlist() { staleTime: 5 * 60_000, // 5 分钟内不重请求 }) - const klineData = dailyKVisible ? (klineBatch.data?.data ?? {}) : {} + // 当日蜡烛实时修补: 历史 K 线按 staleTime 周期拉取 (见 queryKeys 注释), 最后一根 + // 蜡烛用每 tick 刷新的 enriched 当日 OHLC 前端覆盖/追加, 蜡烛随实时行情跳动, 零额外请求。 + const klineData = useMemo(() => { + const base = dailyKVisible ? (klineBatch.data?.data ?? {}) : {} + const liveRows = enriched.data?.rows + const asOf = enriched.data?.as_of + if (!dailyKVisible || !liveRows?.length || !asOf) return base + const liveBySymbol = new Map(liveRows.map((r: any) => [r.symbol, r])) + const patched: Record = {} + for (const sym of Object.keys(base)) { + const arr = base[sym] + if (!Array.isArray(arr) || arr.length === 0) { patched[sym] = arr; continue } + const live = liveBySymbol.get(sym) + const { open, high, low, close } = live ?? {} + if (open == null || high == null || low == null || close == null) { patched[sym] = arr; continue } + const last = arr[arr.length - 1] + if (last.date === asOf) { + patched[sym] = [...arr.slice(0, -1), { ...last, open, high, low, close }] + } else if (last.date < asOf) { + patched[sym] = [...arr, { date: asOf, open, high, low, close }] + } else { + patched[sym] = arr + } + } + return patched + }, [dailyKVisible, klineBatch.data, enriched.data]) // 批量分时数据 (Pro+ 用户, 列可见时才拉) // 刷新策略: 仅当实时行情运行 且 用户在实时监控设置里开启 minute_intraday_refresh 时 @@ -811,7 +836,7 @@ export function Watchlist() { qc.setQueryData(QK.watchlist, data) qc.invalidateQueries({ queryKey: QK.watchlist }) qc.invalidateQueries({ queryKey: ['watchlist-enriched'] }) - qc.invalidateQueries({ queryKey: ['watchlist-kline-batch'] }) + qc.invalidateQueries({ queryKey: ['kline-batch'] }) }, }) @@ -826,7 +851,7 @@ export function Watchlist() { // 2. 清除 list 缓存，触发后台 refetch qc.invalidateQueries({ queryKey: QK.watchlist }) qc.invalidateQueries({ queryKey: ['watchlist-enriched'] }) - qc.invalidateQueries({ queryKey: ['watchlist-kline-batch'] }) + qc.invalidateQueries({ queryKey: ['kline-batch'] }) }, }) @@ -836,7 +861,7 @@ export function Watchlist() { qc.setQueryData(QK.watchlist, data) qc.invalidateQueries({ queryKey: QK.watchlist }) qc.invalidateQueries({ queryKey: ['watchlist-enriched'] }) - qc.invalidateQueries({ queryKey: ['watchlist-kline-batch'] }) + qc.invalidateQueries({ queryKey: ['kline-batch'] }) qc.invalidateQueries({ queryKey: QK.preferences }) qc.invalidateQueries({ queryKey: QK.quoteStatus }) }, @@ -850,7 +875,7 @@ export function Watchlist() { qc.setQueryData(['watchlist-enriched', extColumnsParam], { rows: [], as_of: null, elapsed_ms: 0 }) qc.invalidateQueries({ queryKey: QK.watchlist }) qc.invalidateQueries({ queryKey: ['watchlist-enriched'] }) - qc.invalidateQueries({ queryKey: ['watchlist-kline-batch'] }) + qc.invalidateQueries({ queryKey: ['kline-batch'] }) }, }) diff --git a/frontend/src/pages/backtest/MiningWorkbench.tsx b/frontend/src/pages/backtest/MiningWorkbench.tsx new file mode 100644 index 0000000..d76f4be --- /dev/null +++ b/frontend/src/pages/backtest/MiningWorkbench.tsx @@ -0,0 +1,698 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { useSearchParams } from 'react-router-dom' +import { + AlertTriangle, + CheckCircle2, + Clock3, + Database, + FlaskConical, + Gauge, + Link2, + LoaderCircle, + Play, + RefreshCw, + Rocket, + Save, + Settings2, + Square, +} from 'lucide-react' +import { EmptyState } from '@/components/EmptyState' +import { toast } from '@/components/Toast' +import { + api, + type FactorColumn, + type MiningBudgetProfile, + type MiningCandidateGate, + type MiningCandidateRow, + type MiningRequestV1, + type MiningResult, + type MiningRun, + type MiningScheduleConfig, +} from '@/lib/api' +import { + attachMiningRun, + cancelMining, + startMining, + tryReconnectMining, + useMiningTask, +} from '@/lib/miningTask' +import { QK } from '@/lib/queryKeys' +import { FactorCorrelationHeatmap } from './charts/FactorCorrelationHeatmap' +import { MiningOosChart } from './charts/MiningOosChart' +import { RegimeComparisonChart } from './charts/RegimeComparisonChart' + +const DRAFT_KEY = 'mining_workbench_draft_v1' +const TODAY = new Date().toISOString().slice(0, 10) +const INPUT = 'h-8 w-full rounded-input border border-border bg-surface px-2 text-xs text-foreground outline-none transition-colors focus:border-accent' +const LABEL = 'mb-1 block text-[10px] font-medium text-secondary' +const SUCCESS = new Set(['succeeded', 'succeeded_with_budget_exhausted']) +const ACTIVE = new Set(['queued', 'running', 'cancelling']) +const PROFILE_LABELS: Record = { + exploratory: '探索档', + balanced: '均衡档', + strict: '严格档', +} + +interface MiningDraft { + assetType: 'stock' | 'etf' + start: string + end: string + profile: MiningBudgetProfile + factorNames: string[] + strategyIds: string[] + commissionBps: string + stampTaxBps: string + slippageBps: string + correlationThreshold: string + maxCombinationFactors: string + beamWidth: string + maxFinalists: string + force: boolean +} + +function yearsAgo(years: number) { + const value = new Date() + value.setFullYear(value.getFullYear() - years) + return value.toISOString().slice(0, 10) +} + +const DEFAULT_DRAFT: MiningDraft = { + assetType: 'stock', + start: yearsAgo(4), + end: TODAY, + profile: 'exploratory', + factorNames: [], + strategyIds: [], + commissionBps: '2', + stampTaxBps: '5', + slippageBps: '5', + correlationThreshold: '0.75', + maxCombinationFactors: '4', + beamWidth: '12', + maxFinalists: '8', + force: false, +} + +function loadDraft(): MiningDraft { + try { + const value = JSON.parse(localStorage.getItem(DRAFT_KEY) || '') + if (!value || typeof value !== 'object') return DEFAULT_DRAFT + return { ...DEFAULT_DRAFT, ...value } + } catch { + return DEFAULT_DRAFT + } +} + +function parseBoundedNumber(value: string, label: string, min: number, max: number, integer = false) { + const parsed = Number(value) + if (!Number.isFinite(parsed) || parsed < min || parsed > max || (integer && !Number.isInteger(parsed))) { + toast(`${label}需为 ${min}–${max}${integer ? ' 的整数' : ''}`, 'error') + return null + } + return parsed +} + +function formatNumber(value: number | null | undefined, digits = 2) { + return typeof value === 'number' && Number.isFinite(value) ? value.toFixed(digits) : '—' +} + +function formatPct(value: number | null | undefined, digits = 2) { + return typeof value === 'number' && Number.isFinite(value) ? `${(value * 100).toFixed(digits)}%` : '—' +} + +function formatBytes(value: number | null | undefined) { + if (typeof value !== 'number' || !Number.isFinite(value)) return '—' + if (value >= 1024 ** 3) return `${(value / 1024 ** 3).toFixed(2)} GB` + if (value >= 1024 ** 2) return `${(value / 1024 ** 2).toFixed(1)} MB` + if (value >= 1024) return `${(value / 1024).toFixed(1)} KB` + return `${value} B` +} + +function statusLabel(status?: string) { + return ({ + queued: '排队中', + running: '挖掘中', + cancelling: '正在取消', + succeeded: '已完成', + succeeded_with_budget_exhausted: '已完成（预算耗尽）', + failed: '失败', + cancelled: '已取消', + interrupted: '启动中断', + skipped_prerequisite: '前置条件不足', + } as Record)[status || ''] || '未运行' +} + +function confidenceLabel(value?: string) { + return value === 'high' ? '高' : value === 'standard' ? '标准' : '低' +} + +function definitionLabel(candidate: MiningCandidateRow) { + if (candidate.kind === 'existing_strategy') return candidate.strategy_id || '已有策略' + return candidate.factor_names?.join(' + ') || '因子组合' +} + +function gateTitle(gate?: MiningCandidateGate | null) { + if (!gate || !gate.reasons.length) return undefined + return gate.reasons.join('\n') +} + +function RequestSummaryLine({ result }: { result: MiningResult }) { + const request = result.request_summary + if (!request) return null + const items: [string, string][] = [ + ['资产', request.asset_type === 'etf' ? 'ETF' : '股票'], + ['档位', PROFILE_LABELS[request.budget_profile as MiningBudgetProfile] || request.budget_profile], + ['区间', `${request.start || '—'} → ${request.end || '—'}`], + ['因子', String(request.factor_count)], + ['对照策略', String(request.strategy_count)], + ['成本', `佣金 ${formatBps(request.commission_pct)} / 印花 ${formatBps(request.stamp_tax_pct)} / 滑点 ${typeof request.slippage_bps === 'number' && Number.isFinite(request.slippage_bps) ? `${request.slippage_bps.toFixed(1)}bp` : '—'}`], + ['相关阈值', request.correlation_threshold == null ? '—' : request.correlation_threshold.toFixed(2)], + ] + return ( +
+ {items.map(([label, value]) => ( + + {label}{' '} + {value} + + ))} +
+ ) +} + +function formatBps(ratio: number | null | undefined, digits = 1) { + return typeof ratio === 'number' && Number.isFinite(ratio) ? `${(ratio * 10000).toFixed(digits)}bp` : '—' +} + +function foldKindLabel(kind?: string | null) { + if (kind === 'cross') return '跨折' + if (kind === 'benchmark') return '对照' + return undefined +} + +function SummaryStrip({ result }: { result: MiningResult }) { + const items = [ + ['因子', `${result.summary.selected_factor_count}/${result.summary.factor_count}`], + ['候选', String(result.summary.candidate_count)], + ['有效折', String(result.summary.valid_fold_count)], + ['跳过折', String(result.summary.skipped_fold_count)], + ['置信度', confidenceLabel(result.summary.confidence)], + ['耗时', result.summary.elapsed_ms == null ? '—' : `${(result.summary.elapsed_ms / 1000).toFixed(1)}s`], + ['峰值内存', formatBytes(result.summary.peak_rss_bytes)], + ] + return ( +
+ {items.map(([label, value]) => ( +
+
{label}
+
{value}
+
+ ))} +
+ ) +} + +function RunStatus({ run, progress, error, reconnecting }: { run: MiningRun | null; progress?: { label?: string; phase?: string; percent?: number } | null; error?: string | null; reconnecting?: boolean }) { + const active = !!run && ACTIVE.has(run.status) + const success = !!run && SUCCESS.has(run.status) + const Icon = reconnecting ? RefreshCw : active ? LoaderCircle : success ? CheckCircle2 : error ? AlertTriangle : Clock3 + return ( +
+ +
+
+ {reconnecting ? '连接恢复中' : run ? statusLabel(run.status) : '研究任务'} + {progress?.label ? ` · ${progress.label}` : ''} +
+ {(error || progress?.phase) && ( +
+ {error || progress?.phase} +
+ )} +
+ {typeof progress?.percent === 'number' && ( + {Math.round(progress.percent)}% + )} + {run?.run_id && {run.run_id}} +
+ ) +} + +function FactorTable({ result }: { result: MiningResult }) { + return ( +
+
+ 因子方向评分ICIR覆盖换手价差Sharpe状态 +
+ {result.factors.map(row => ( +
+ {row.label || row.factor_name} + {row.direction === 1 ? '正向' : '反向'} + {formatNumber(row.score, 3)} + {formatNumber(row.ic_mean, 4)} + {formatNumber(row.ir, 2)} + {formatPct(row.coverage, 1)} + {formatPct(row.turnover, 1)} + {formatPct(row.spread_return, 2)} + {formatNumber(row.spread_sharpe, 2)} + + {row.selected ? '入选' : row.excluded_reason || '未入选'} + +
+ ))} +
+ ) +} + +export function MiningWorkbench() { + const queryClient = useQueryClient() + const [searchParams, setSearchParams] = useSearchParams() + const initializedFactors = useRef(false) + const [draft, setDraft] = useState(loadDraft) + const [scheduleDraft, setScheduleDraft] = useState(null) + const [correlationScope, setCorrelationScope] = useState<'all' | 'selected'>('selected') + const task = useMiningTask() + const runFromUrl = searchParams.get('run') || '' + const selectedCandidate = searchParams.get('candidate') || '' + + const factorQuery = useQuery({ queryKey: QK.factorColumns, queryFn: api.factorColumns }) + const strategyQuery = useQuery({ + queryKey: QK.strategyLinkOptions(draft.assetType), + queryFn: () => api.strategyList(draft.assetType), + }) + const runsQuery = useQuery({ + queryKey: QK.miningRuns, + queryFn: api.miningRuns, + refetchInterval: task.isPending ? 5000 : false, + }) + const validDateRange = !draft.start || !draft.end || draft.start <= draft.end + const availabilityQuery = useQuery({ + queryKey: QK.miningAvailability( + draft.assetType, + draft.profile, + draft.start, + draft.end, + ), + queryFn: () => api.miningAvailability({ + assetType: draft.assetType, + budgetProfile: draft.profile, + start: draft.start || undefined, + end: draft.end || undefined, + }), + enabled: validDateRange, + staleTime: 30_000, + }) + const configQuery = useQuery({ queryKey: QK.miningConfig, queryFn: api.miningConfig }) + + useEffect(() => { + localStorage.setItem(DRAFT_KEY, JSON.stringify(draft)) + }, [draft]) + + useEffect(() => { + if (initializedFactors.current || !factorQuery.data?.columns.length) return + initializedFactors.current = true + if (!draft.factorNames.length) { + setDraft(current => ({ ...current, factorNames: factorQuery.data!.columns.slice(0, 48).map(item => item.id) })) + } + }, [factorQuery.data, draft.factorNames.length]) + + useEffect(() => { + if (configQuery.data && !scheduleDraft) setScheduleDraft(configQuery.data) + }, [configQuery.data, scheduleDraft]) + + useEffect(() => { + if (runFromUrl) { + if (task.isPending && !task.runId) return + if (task.runId !== runFromUrl) void attachMiningRun(runFromUrl) + return + } + if (task.runId) { + const params = new URLSearchParams(searchParams) + params.set('run', task.runId) + setSearchParams(params, { replace: true }) + return + } + tryReconnectMining() + }, [runFromUrl, searchParams, setSearchParams, task.runId]) + + const currentResult = task.runId + && (!runFromUrl || runFromUrl === task.runId) + && task.run?.run_id === task.runId + && SUCCESS.has(task.run.status) + && task.result?.run_id === task.runId + ? task.result + : null + const showingPrevious = !!( + task.runId + && (!runFromUrl || runFromUrl === task.runId) + && task.previousResult + && task.previousResult.run_id !== task.runId + && ( + task.isPending + || !!task.error + || (!!task.run && !SUCCESS.has(task.run.status)) + ) + ) + const result = currentResult ?? (showingPrevious ? task.previousResult : null) + const candidates = result?.candidates ?? [] + const activeCandidate = candidates.find(item => item.signature === selectedCandidate) ?? candidates[0] ?? null + const candidateFolds = activeCandidate?.folds?.length ? activeCandidate.folds : result?.folds ?? [] + const correlation = useMemo(() => { + if (!result || correlationScope === 'all') return result?.correlation ?? null + const selected = new Set(result.factors.filter(item => item.selected).map(item => item.factor_name)) + const indexes = result.correlation.labels + .map((label, index) => selected.has(label) ? index : -1) + .filter(index => index >= 0) + if (!indexes.length) return result.correlation + return { + ...result.correlation, + labels: indexes.map(index => result.correlation.labels[index]), + matrix: indexes.map(row => indexes.map(column => result.correlation.matrix[row]?.[column] ?? null)), + pair_counts: indexes.map(row => indexes.map(column => result.correlation.pair_counts?.[row]?.[column] ?? null)), + } + }, [result, correlationScope]) + + const setSelectedCandidate = (signature: string) => { + const params = new URLSearchParams(searchParams) + if (signature) params.set('candidate', signature) + else params.delete('candidate') + setSearchParams(params, { replace: true }) + } + + useEffect(() => { + if (!selectedCandidate && candidates[0]) setSelectedCandidate(candidates[0].signature) + if (selectedCandidate && candidates.length && !candidates.some(item => item.signature === selectedCandidate)) { + setSelectedCandidate(candidates[0]?.signature || '') + } + }, [candidates, selectedCandidate]) + + const factorGroups = useMemo(() => { + const groups: Record = {} + for (const item of factorQuery.data?.columns ?? []) (groups[item.group] ??= []).push(item) + return groups + }, [factorQuery.data]) + const strategies = strategyQuery.data?.strategies.filter(item => item.execution_backend === 'matrix_native') ?? [] + + useEffect(() => { + if (!strategyQuery.data) return + const compatibleIds = new Set(strategies.map(item => item.id)) + setDraft(current => { + const strategyIds = current.strategyIds.filter(id => compatibleIds.has(id)) + return strategyIds.length === current.strategyIds.length ? current : { ...current, strategyIds } + }) + }, [strategyQuery.data]) + + const updateDraft = (key: K, value: MiningDraft[K]) => { + setDraft(current => ({ ...current, [key]: value })) + } + const changeAssetType = (assetType: MiningDraft['assetType']) => { + setDraft(current => ({ + ...current, + assetType, + strategyIds: [], + })) + } + const toggleFactor = (id: string) => { + setDraft(current => ({ + ...current, + factorNames: current.factorNames.includes(id) + ? current.factorNames.filter(value => value !== id) + : current.factorNames.length < 48 ? [...current.factorNames, id] : current.factorNames, + })) + } + const toggleStrategy = (id: string) => { + setDraft(current => ({ + ...current, + strategyIds: current.strategyIds.includes(id) + ? current.strategyIds.filter(value => value !== id) + : current.strategyIds.length < 8 ? [...current.strategyIds, id] : current.strategyIds, + })) + } + + const runMining = () => { + if (!draft.factorNames.length) { + toast('至少选择一个因子', 'error') + return + } + if (draft.start && draft.end && draft.start > draft.end) { + toast('开始日期不能晚于结束日期', 'error') + return + } + if (availabilityQuery.isPending || availabilityQuery.isFetching) { + toast('正在核验有效交易日，请稍候', 'error') + return + } + if (availabilityQuery.isError || !availabilityQuery.data) { + toast('无法核验有效交易日，请检查数据状态后重试', 'error') + return + } + if (!availabilityQuery.data.eligible) { + toast( + `${PROFILE_LABELS[draft.profile]}至少需要 ${availabilityQuery.data.required_bars} 个交易日，当前范围仅 ${availabilityQuery.data.trading_bars} 个`, + 'error', + ) + return + } + const commissionBps = parseBoundedNumber(draft.commissionBps, '佣金', 0, 500) + const stampTaxBps = parseBoundedNumber(draft.stampTaxBps, '印花税', 0, 500) + const slippageBps = parseBoundedNumber(draft.slippageBps, '滑点', 0, 1000) + const correlationThreshold = parseBoundedNumber(draft.correlationThreshold, '相关阈值', Number.EPSILON, 1) + const maxCombinationFactors = parseBoundedNumber(draft.maxCombinationFactors, '组合上限', 1, 4, true) + const beamWidth = parseBoundedNumber(draft.beamWidth, 'Beam', 1, 12, true) + const maxFinalists = parseBoundedNumber(draft.maxFinalists, 'Finalists', 1, 8, true) + if ([commissionBps, stampTaxBps, slippageBps, correlationThreshold, maxCombinationFactors, beamWidth, maxFinalists].some(value => value == null)) return + const payload: MiningRequestV1 = { + factor_names: draft.factorNames, + strategy_ids: draft.strategyIds, + asset_type: draft.assetType, + start: draft.start || null, + end: draft.end || null, + budget_profile: draft.profile, + commission_pct: commissionBps! / 10000, + stamp_tax_pct: stampTaxBps! / 10000, + slippage_bps: slippageBps!, + correlation_threshold: correlationThreshold!, + max_combination_factors: maxCombinationFactors!, + beam_width: beamWidth!, + max_finalists: maxFinalists!, + force: draft.force, + } + const params = new URLSearchParams(searchParams) + params.delete('run') + params.delete('candidate') + setSearchParams(params, { replace: true }) + void startMining(payload).then(() => queryClient.invalidateQueries({ queryKey: QK.miningRuns })) + } + + const promote = useMutation({ + mutationFn: (candidate: MiningCandidateRow) => { + if (!currentResult) throw new Error('只能保存当前已成功运行的候选') + return api.miningPromote(currentResult.run_id, candidate.signature) + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: QK.researchCandidates }) + toast('候选已保存，状态为待验证', 'success') + }, + onError: error => toast(`保存失败 · ${String((error as Error).message || error)}`, 'error'), + }) + const publish = useMutation({ + mutationFn: (candidate: MiningCandidateRow) => { + if (!currentResult) throw new Error('只能发布当前已成功运行的候选') + return api.miningPublish(currentResult.run_id, candidate.signature) + }, + onSuccess: value => { + queryClient.invalidateQueries({ queryKey: QK.screenerStrategies(draft.assetType) }) + toast(`策略已发布 · ${value.strategy_id}`, 'success') + }, + onError: error => toast(`发布失败 · ${String((error as Error).message || error)}`, 'error'), + }) + const saveSchedule = useMutation({ + mutationFn: (value: MiningScheduleConfig) => api.updateMiningConfig(value), + onSuccess: value => { + setScheduleDraft(value) + queryClient.setQueryData(QK.miningConfig, value) + toast('自动挖掘配置已保存', 'success') + }, + onError: error => toast(`保存失败 · ${String((error as Error).message || error)}`, 'error'), + }) + + const attachRun = (run: MiningRun) => { + const params = new URLSearchParams(searchParams) + params.set('run', run.run_id) + params.delete('candidate') + setSearchParams(params, { replace: true }) + } + + return ( +
+ + +
+ + {showingPrevious &&
历史结果 · run {result?.run_id}。当前 run {task.runId} {task.isPending ? '仍在执行' : '未成功完成'}，以下内容仅供参考，候选操作已禁用。
} + + {!result ? ( +
+ ) : ( +
+ + + +
+

因子排名

{result.methodology_version}
+
+
+ +
+
+
+

市场环境对比

+ {activeCandidate && candidates[0] && activeCandidate.signature !== candidates[0].signature && 环境数据属于排名首位候选（{candidates[0].name}）} +
+ ({ state: row.state, label: row.label, nDates: row.n_dates, sharpe: row.sharpe, return: row.total_return, maxDrawdown: row.max_drawdown }))} /> +
+

逐折样本外

({ fold: row.fold, label: row.label || `Fold ${row.fold}`, return: row.total_return, sharpe: row.sharpe, skipped: row.skipped, reason: row.reason || undefined }))} />
+
+ +
+
+
+

相关矩阵

+
阈值 {result.correlation.threshold.toFixed(2)} · 按日截面 Rank
+
+
+ {(['selected', 'all'] as const).map(scope => ( + + ))} +
+
+
+
+ +
+
+
+ +

策略候选

对照策略在全部 outer 折独立评估 · 发布始终需要人工确认
{candidates.map(candidate => )}{!candidates.length &&
暂无晋级候选
}
{activeCandidate ?
{activeCandidate.name}{activeCandidate.gate && {activeCandidate.gate.qualified ? '达标' : '未达标'}}
{definitionLabel(activeCandidate)}
{activeCandidate.gate && !activeCandidate.gate.qualified &&
未达晋级门槛，仅可保存为待定候选：{activeCandidate.gate.reasons.join('；')}
}
{[['平均每折收益', formatPct(activeCandidate.oos_return)], ['Sharpe', formatNumber(activeCandidate.oos_sharpe)], ['最大回撤', formatPct(activeCandidate.oos_max_drawdown)], ['正收益折', formatPct(activeCandidate.oos_positive_fold_ratio)], ['有效折', activeCandidate.valid_folds == null ? '—' : String(activeCandidate.valid_folds)], ['交易数', activeCandidate.oos_n_trades == null ? '—' : String(activeCandidate.oos_n_trades)]].map(([label, value]) =>
{label}
{value}
)}
{candidateFolds.map(fold =>
Fold {fold.fold}{foldKindLabel(fold.evaluation_kind) && {foldKindLabel(fold.evaluation_kind)}}{fold.train_start || '—'} → {fold.train_end || '—'}{fold.test_start || '—'} → {fold.test_end || '—'}{formatPct(fold.total_return)}{formatNumber(fold.sharpe)}{formatPct(fold.max_drawdown)}{fold.skipped ? fold.reason || '跳过' : `${fold.n_trades ?? '—'} 笔`}
)}
:
选择候选查看定义与逐折结果
}
+ +

性能与复用

{[['总耗时', result.telemetry.elapsed_ms == null ? '—' : `${(result.telemetry.elapsed_ms / 1000).toFixed(1)}s`], ['峰值 RSS', formatBytes(result.telemetry.peak_rss_bytes)], ['面板扫描', result.telemetry.panel_scans ?? '—'], ['矩阵字节', formatBytes(result.telemetry.matrix_bytes)], ['缓存命中', result.telemetry.cache_hits ?? '—'], ['Fold 复用', result.telemetry.fold_reuses ?? '—'], ['IPC 结果', formatBytes(result.telemetry.serialized_result_bytes)]].map(([label, value]) =>
{label}
{String(value)}
)}
{result.telemetry.phase_ms &&
{Object.entries(result.telemetry.phase_ms).map(([phase, ms]) => {phase} {ms.toFixed(1)}ms)}
}
+
+ )} + + {task.runId &&
刷新后通过持久 run ID 自动重连；浏览器断开不会取消 worker。
} +
+
+ ) +} diff --git a/frontend/src/pages/backtest/ResearchCandidatesDialog.tsx b/frontend/src/pages/backtest/ResearchCandidatesDialog.tsx index 0d8534b..2aa7b74 100644 --- a/frontend/src/pages/backtest/ResearchCandidatesDialog.tsx +++ b/frontend/src/pages/backtest/ResearchCandidatesDialog.tsx @@ -46,7 +46,7 @@ export function ResearchCandidatesDialog({ onClose }: { onClose: () => void }) { const [kind, setKind] = useState<'all' | 'factor' | 'strategy'>('all') const [linkDraft, setLinkDraft] = useState(null) const candidates = useQuery({ queryKey: QK.researchCandidates, queryFn: api.researchCandidates }) - const strategies = useQuery({ queryKey: QK.strategyLinkOptions, queryFn: () => api.strategyList() }) + const strategies = useQuery({ queryKey: QK.strategyLinkOptions(), queryFn: () => api.strategyList() }) const factorColumns = useQuery({ queryKey: QK.factorColumns, queryFn: api.factorColumns }) const supportedFactors = useMemo( () => new Set((factorColumns.data?.columns ?? []).map(item => item.id)), @@ -98,7 +98,7 @@ export function ResearchCandidatesDialog({ onClose }: { onClose: () => void }) { onSuccess: result => { queryClient.invalidateQueries({ queryKey: QK.strategyDetail(result.strategyId) }) queryClient.invalidateQueries({ queryKey: ['screener-strategies'] }) - queryClient.invalidateQueries({ queryKey: QK.strategyLinkOptions }) + queryClient.invalidateQueries({ queryKey: QK.strategyLinkOptions() }) setLinkDraft(null) toast(`已加入“${result.strategyName}”评分方案`, 'success') }, diff --git a/frontend/src/pages/backtest/charts/FactorCorrelationHeatmap.tsx b/frontend/src/pages/backtest/charts/FactorCorrelationHeatmap.tsx new file mode 100644 index 0000000..fd0c039 --- /dev/null +++ b/frontend/src/pages/backtest/charts/FactorCorrelationHeatmap.tsx @@ -0,0 +1,191 @@ +import { useEffect, useMemo, useState } from 'react' +import type { EChartsOption } from 'echarts' +import { useChartTheme } from '@/lib/theme' +import { useECharts } from './useECharts' + +export interface FactorCorrelationHeatmapProps { + labels: string[] + matrix: (number | null)[][] + pairCounts?: (number | null)[][] + threshold?: number +} + +interface HeatmapDatum { + value: [number, number, number, number | null] + itemStyle?: { opacity: number } +} + +interface PreparedHeatmap { + labels: string[] + data: HeatmapDatum[] +} + +const EMPTY_HEATMAP: PreparedHeatmap = { labels: [], data: [] } + +function escapeHtml(value: string): string { + return value.replace(/[&<>'"]/g, char => ({ + '&': '&', + '<': '<', + '>': '>', + "'": ''', + '"': '"', + })[char] ?? char) +} + +export function FactorCorrelationHeatmap({ + labels, + matrix, + pairCounts, + threshold, +}: FactorCorrelationHeatmapProps) { + const ct = useChartTheme() + const [prepared, setPrepared] = useState(EMPTY_HEATMAP) + + // Flattening can dominate render time for wide factor sets, so keep it out of render. + useEffect(() => { + if (!labels.length || !matrix.length) { + setPrepared(EMPTY_HEATMAP) + return + } + + const nextLabels = labels.slice() + const data: HeatmapDatum[] = [] + const thresholdValue = typeof threshold === 'number' && Number.isFinite(threshold) + ? Math.min(1, Math.max(0, Math.abs(threshold))) + : null + + for (let rowIndex = 0; rowIndex < nextLabels.length; rowIndex += 1) { + const row = matrix[rowIndex] + if (!row) continue + + for (let columnIndex = 0; columnIndex < nextLabels.length; columnIndex += 1) { + const rho = row[columnIndex] + if (typeof rho !== 'number' || !Number.isFinite(rho)) continue + + const rawPairCount = pairCounts?.[rowIndex]?.[columnIndex] + const pairCount = typeof rawPairCount === 'number' && Number.isFinite(rawPairCount) + ? rawPairCount + : null + const passesThreshold = thresholdValue == null + || rowIndex === columnIndex + || Math.abs(rho) >= thresholdValue + + data.push({ + value: [columnIndex, rowIndex, rho, pairCount], + ...(passesThreshold ? {} : { itemStyle: { opacity: 0.3 } }), + }) + } + } + + setPrepared({ labels: nextLabels, data }) + }, [labels, matrix, pairCounts, threshold]) + + const option = useMemo(() => { + if (!prepared.data.length) return null + + return { + animation: false, + grid: { left: 78, right: 16, top: 14, bottom: 72 }, + tooltip: { + trigger: 'item', + confine: true, + backgroundColor: ct.tooltipBg, + borderColor: ct.tooltipBorder, + textStyle: { color: ct.tooltipText, fontSize: 12 }, + formatter: (params: any) => { + const value = params.value as HeatmapDatum['value'] | undefined + if (!value) return '' + const [columnIndex, rowIndex, rho, pairCount] = value + const rowLabel = escapeHtml(prepared.labels[rowIndex] ?? '') + const columnLabel = escapeHtml(prepared.labels[columnIndex] ?? '') + return `
${rowLabel} × ${columnLabel}
+
rho${rho.toFixed(4)}
+
配对数${pairCount == null ? '—' : pairCount.toLocaleString('zh-CN')}
` + }, + }, + xAxis: { + type: 'category', + data: prepared.labels, + axisLabel: { + color: ct.text, + fontSize: 10, + interval: 0, + rotate: prepared.labels.length > 6 ? 40 : 0, + width: 68, + overflow: 'truncate', + }, + axisLine: { lineStyle: { color: ct.border } }, + axisTick: { show: false }, + splitArea: { show: false }, + }, + yAxis: { + type: 'category', + data: prepared.labels, + inverse: true, + axisLabel: { + color: ct.text, + fontSize: 10, + width: 64, + overflow: 'truncate', + }, + axisLine: { lineStyle: { color: ct.border } }, + axisTick: { show: false }, + splitArea: { show: false }, + }, + visualMap: { + type: 'continuous', + min: -1, + max: 1, + dimension: 2, + orient: 'horizontal', + left: 'center', + bottom: 4, + itemWidth: 100, + itemHeight: 8, + calculable: false, + precision: 1, + text: ['1', '-1'], + textGap: 6, + textStyle: { color: ct.text, fontSize: 10 }, + inRange: { + color: ['#2563eb', ct.fillSubtle, '#ef4444'], + }, + }, + series: [{ + name: '相关性', + type: 'heatmap', + data: prepared.data, + progressive: 2000, + itemStyle: { + borderColor: ct.border, + borderWidth: 1, + }, + emphasis: { + itemStyle: { + borderColor: ct.textStrong, + borderWidth: 1, + }, + }, + }], + } + }, [prepared, ct]) + + const chartRef = useECharts(option, [prepared, ct]) + const isEmpty = prepared.data.length === 0 + + return ( +
+
+ {isEmpty && ( +
+ 暂无相关性数据 +
+ )} +
+ ) +} diff --git a/frontend/src/pages/backtest/charts/MiningOosChart.tsx b/frontend/src/pages/backtest/charts/MiningOosChart.tsx new file mode 100644 index 0000000..dae25ba --- /dev/null +++ b/frontend/src/pages/backtest/charts/MiningOosChart.tsx @@ -0,0 +1,201 @@ +import { useMemo } from 'react' +import type { EChartsOption } from 'echarts' +import { useChartTheme } from '@/lib/theme' +import { useECharts } from './useECharts' + +export interface MiningOosFold { + fold: number | string + label: string + return: number | null + sharpe: number | null + skipped?: boolean + reason?: string +} + +export interface MiningOosChartProps { + folds: MiningOosFold[] +} + +function finiteOrNull(value: number | null): number | null { + return typeof value === 'number' && Number.isFinite(value) ? value : null +} + +function escapeHtml(value: string): string { + return value.replace(/[&<>'"]/g, char => ({ + '&': '&', + '<': '<', + '>': '>', + "'": ''', + '"': '"', + })[char] ?? char) +} + +export function MiningOosChart({ folds }: MiningOosChartProps) { + const ct = useChartTheme() + const preparedFolds = useMemo(() => folds.map(fold => ({ + ...fold, + return: fold.skipped ? null : finiteOrNull(fold.return), + sharpe: fold.skipped ? null : finiteOrNull(fold.sharpe), + })), [folds]) + + const hasDisplayData = preparedFolds.some(fold => ( + fold.skipped || fold.return != null || fold.sharpe != null + )) + + const option = useMemo(() => { + if (!preparedFolds.length || !hasDisplayData) return null + + return { + animation: false, + grid: { left: 54, right: 48, top: 48, bottom: 52 }, + legend: { + top: 2, + left: 'center', + itemWidth: 12, + itemHeight: 8, + itemGap: 14, + textStyle: { color: ct.text, fontSize: 10 }, + data: ['样本外收益', 'Sharpe', '已跳过'], + }, + tooltip: { + trigger: 'axis', + confine: true, + axisPointer: { type: 'shadow' }, + backgroundColor: ct.tooltipBg, + borderColor: ct.tooltipBorder, + textStyle: { color: ct.tooltipText, fontSize: 12 }, + formatter: (params: any) => { + const items = Array.isArray(params) ? params : [params] + const skippedItem = items.find(item => Array.isArray(item.value) && typeof item.value[2] === 'number') + const index = skippedItem + ? skippedItem.value[2] as number + : items[0]?.dataIndex as number | undefined + const fold = index == null ? undefined : preparedFolds[index] + if (!fold) return '' + + const heading = fold.label || `Fold ${fold.fold}` + if (fold.skipped) { + const reason = fold.reason ? escapeHtml(fold.reason) : '未提供原因' + return `
${escapeHtml(heading)}
+
已跳过
+
${reason}
` + } + + return `
${escapeHtml(heading)}
+
样本外收益${fold.return == null ? '—' : `${(fold.return * 100).toFixed(2)}%`}
+
Sharpe${fold.sharpe == null ? '—' : fold.sharpe.toFixed(2)}
` + }, + }, + xAxis: { + type: 'category', + data: preparedFolds.map(fold => fold.label || `Fold ${fold.fold}`), + axisLabel: { + interval: 0, + fontSize: 10, + width: 68, + overflow: 'truncate', + formatter: (_value: string, index: number) => preparedFolds[index]?.skipped + ? `{skipped|${preparedFolds[index]?.label || `Fold ${preparedFolds[index]?.fold ?? ''}`}}` + : `{normal|${preparedFolds[index]?.label || `Fold ${preparedFolds[index]?.fold ?? ''}`}}`, + rich: { + normal: { color: ct.text }, + skipped: { color: ct.text, opacity: 0.5 }, + }, + }, + axisLine: { lineStyle: { color: ct.border } }, + axisTick: { show: false }, + }, + yAxis: [ + { + type: 'value', + name: '收益', + nameTextStyle: { color: ct.text, fontSize: 10 }, + axisLabel: { + color: ct.text, + fontSize: 10, + formatter: (value: number) => `${value.toFixed(0)}%`, + }, + axisLine: { show: false }, + splitLine: { lineStyle: { color: ct.grid } }, + }, + { + type: 'value', + name: 'Sharpe', + nameTextStyle: { color: ct.text, fontSize: 10 }, + axisLabel: { color: ct.text, fontSize: 10 }, + axisLine: { show: false }, + splitLine: { show: false }, + }, + ], + series: [ + { + name: '样本外收益', + type: 'bar', + data: preparedFolds.map(fold => fold.return == null + ? null + : { + value: fold.return * 100, + itemStyle: { color: fold.return >= 0 ? '#ef4444' : '#22c55e' }, + }), + barMaxWidth: 28, + }, + { + name: 'Sharpe', + type: 'line', + yAxisIndex: 1, + data: preparedFolds.map(fold => fold.sharpe), + connectNulls: false, + symbol: 'circle', + symbolSize: 6, + lineStyle: { color: '#3b82f6', width: 1.5 }, + itemStyle: { color: '#3b82f6' }, + z: 5, + }, + { + name: '已跳过', + type: 'custom', + data: preparedFolds.flatMap((fold, index) => fold.skipped ? [[index, 0, index]] : []), + renderItem: (params: any, api: any) => { + const x = api.coord([api.value(0), 0])[0] + const coordinateSystem = params.coordSys as { y: number } + return { + type: 'text', + x, + y: coordinateSystem.y + 7, + style: { + text: '跳过', + fill: ct.text, + opacity: 0.65, + fontSize: 10, + align: 'center', + verticalAlign: 'top', + }, + } + }, + z: 10, + }, + ], + } + }, [preparedFolds, hasDisplayData, ct]) + + const chartRef = useECharts(option, [preparedFolds, ct]) + const emptyMessage = folds.length === 0 + ? '暂无样本外验证数据' + : '暂无可展示的样本外指标' + + return ( +
+
+ {!hasDisplayData && ( +
+ {emptyMessage} +
+ )} +
+ ) +} diff --git a/frontend/src/pages/backtest/charts/RegimeComparisonChart.tsx b/frontend/src/pages/backtest/charts/RegimeComparisonChart.tsx new file mode 100644 index 0000000..3c9ce4a --- /dev/null +++ b/frontend/src/pages/backtest/charts/RegimeComparisonChart.tsx @@ -0,0 +1,182 @@ +import { useMemo } from 'react' +import type { EChartsOption } from 'echarts' +import { useChartTheme } from '@/lib/theme' +import { useECharts } from './useECharts' + +export interface RegimeComparisonRow { + state: string + label: string + nDates: number + sharpe: number | null + return: number | null + maxDrawdown: number | null +} + +export interface RegimeComparisonChartProps { + rows: RegimeComparisonRow[] +} + +function finiteOrNull(value: number | null): number | null { + return typeof value === 'number' && Number.isFinite(value) ? value : null +} + +function escapeHtml(value: string): string { + return value.replace(/[&<>'"]/g, char => ({ + '&': '&', + '<': '<', + '>': '>', + "'": ''', + '"': '"', + })[char] ?? char) +} + +export function RegimeComparisonChart({ rows }: RegimeComparisonChartProps) { + const ct = useChartTheme() + const preparedRows = useMemo(() => rows.map(row => { + const hasEnoughSamples = Number.isFinite(row.nDates) && row.nDates >= 2 + return { + ...row, + hasEnoughSamples, + sharpe: hasEnoughSamples ? finiteOrNull(row.sharpe) : null, + return: hasEnoughSamples ? finiteOrNull(row.return) : null, + maxDrawdown: hasEnoughSamples ? finiteOrNull(row.maxDrawdown) : null, + } + }), [rows]) + + const hasMetrics = preparedRows.some(row => ( + row.return != null || row.sharpe != null || row.maxDrawdown != null + )) + + const option = useMemo(() => { + if (!preparedRows.length || !hasMetrics) return null + + return { + animation: false, + color: ['#ef4444', '#22c55e', '#3b82f6'], + grid: { left: 54, right: 48, top: 48, bottom: 48 }, + legend: { + top: 2, + left: 'center', + itemWidth: 12, + itemHeight: 8, + itemGap: 12, + textStyle: { color: ct.text, fontSize: 10 }, + data: ['收益', '最大回撤', 'Sharpe'], + }, + tooltip: { + trigger: 'axis', + confine: true, + axisPointer: { type: 'shadow' }, + backgroundColor: ct.tooltipBg, + borderColor: ct.tooltipBorder, + textStyle: { color: ct.tooltipText, fontSize: 12 }, + formatter: (params: any) => { + const items = Array.isArray(params) ? params : [params] + const index = items[0]?.dataIndex as number | undefined + const row = index == null ? undefined : preparedRows[index] + if (!row) return '' + + const state = row.state && row.state !== row.label + ? `${escapeHtml(row.state)}` + : '' + const status = row.hasEnoughSamples ? '' : '
样本不足
' + return `
${escapeHtml(row.label)} ${state}
+
交易日${row.nDates.toLocaleString('zh-CN')}
+
收益${row.return == null ? '—' : `${(row.return * 100).toFixed(2)}%`}
+
最大回撤${row.maxDrawdown == null ? '—' : `${(row.maxDrawdown * 100).toFixed(2)}%`}
+
Sharpe${row.sharpe == null ? '—' : row.sharpe.toFixed(2)}
${status}` + }, + }, + xAxis: { + type: 'category', + data: preparedRows.map(row => row.label), + axisLabel: { + interval: 0, + fontSize: 10, + width: 72, + overflow: 'truncate', + formatter: (_value: string, index: number) => preparedRows[index]?.hasEnoughSamples + ? `{normal|${preparedRows[index]?.label ?? ''}}` + : `{muted|${preparedRows[index]?.label ?? ''}}`, + rich: { + normal: { color: ct.text }, + muted: { color: ct.text, opacity: 0.45 }, + }, + }, + axisLine: { lineStyle: { color: ct.border } }, + axisTick: { show: false }, + }, + yAxis: [ + { + type: 'value', + name: '比例', + nameTextStyle: { color: ct.text, fontSize: 10 }, + axisLabel: { + color: ct.text, + fontSize: 10, + formatter: (value: number) => `${value.toFixed(0)}%`, + }, + axisLine: { show: false }, + splitLine: { lineStyle: { color: ct.grid } }, + }, + { + type: 'value', + name: 'Sharpe', + nameTextStyle: { color: ct.text, fontSize: 10 }, + axisLabel: { color: ct.text, fontSize: 10 }, + axisLine: { show: false }, + splitLine: { show: false }, + }, + ], + series: [ + { + name: '收益', + type: 'bar', + data: preparedRows.map(row => row.return == null ? null : row.return * 100), + barMaxWidth: 24, + itemStyle: { color: '#ef4444' }, + }, + { + name: '最大回撤', + type: 'bar', + data: preparedRows.map(row => row.maxDrawdown == null ? null : row.maxDrawdown * 100), + barMaxWidth: 24, + itemStyle: { color: '#22c55e' }, + }, + { + name: 'Sharpe', + type: 'line', + yAxisIndex: 1, + data: preparedRows.map(row => row.sharpe), + connectNulls: false, + symbol: 'circle', + symbolSize: 6, + lineStyle: { color: '#3b82f6', width: 1.5 }, + itemStyle: { color: '#3b82f6' }, + z: 5, + }, + ], + } + }, [preparedRows, hasMetrics, ct]) + + const chartRef = useECharts(option, [preparedRows, ct]) + const emptyMessage = rows.length === 0 + ? '暂无市场状态数据' + : '样本不足，暂无可比较指标' + + return ( +
+
+ {!hasMetrics && ( +
+ {emptyMessage} +
+ )} +
+ ) +} diff --git a/frontend/src/pages/backtest/charts/useECharts.ts b/frontend/src/pages/backtest/charts/useECharts.ts index 186e2eb..5952124 100644 --- a/frontend/src/pages/backtest/charts/useECharts.ts +++ b/frontend/src/pages/backtest/charts/useECharts.ts @@ -21,12 +21,13 @@ export function useECharts( // 初始化 / 销毁 useEffect(() => { if (!chartRef.current) return - instanceRef.current = echarts.init(chartRef.current, undefined, { renderer: 'canvas' }) - const handleResize = () => instanceRef.current?.resize() - window.addEventListener('resize', handleResize) + const container = chartRef.current + instanceRef.current = echarts.init(container, undefined, { renderer: 'canvas' }) + const resizeObserver = new ResizeObserver(() => instanceRef.current?.resize()) + resizeObserver.observe(container) return () => { - window.removeEventListener('resize', handleResize) + resizeObserver.disconnect() instanceRef.current?.dispose() instanceRef.current = null } diff --git a/frontend/src/pages/settings/MenuSettings.tsx b/frontend/src/pages/settings/MenuSettings.tsx index b2e51e5..2127d18 100644 --- a/frontend/src/pages/settings/MenuSettings.tsx +++ b/frontend/src/pages/settings/MenuSettings.tsx @@ -35,6 +35,7 @@ const BUILTIN_PAGES: NavEntry[] = [ { id: '/watchlist', label: '自选', type: 'builtin', visible: true }, { id: '/screener', label: '策略', type: 'builtin', visible: true }, { id: '/backtest', label: '回测', type: 'builtin', visible: true }, + { id: '/mining', label: '挖掘', type: 'builtin', visible: true }, { id: '/limit-ladder', label: '连板梯队', type: 'builtin', visible: true }, { id: '/concept-analysis', label: '概念分析', type: 'builtin', visible: true }, { id: '/industry-analysis', label: '行业分析', type: 'builtin', visible: true }, @@ -187,7 +188,18 @@ export function SettingsMenuSettingsPanel() { } } for (const e of [...BUILTIN_PAGES, ...analysisEntries]) { - if (!seen.has(e.id)) ordered.push(e) + if (seen.has(e.id)) continue + // 未保存过排序的新条目: 内置页插回默认位置, 分析菜单追加到末尾 + const defaultIndex = BUILTIN_PAGES.findIndex(p => p.id === e.id) + let anchor = -1 + if (defaultIndex > 0) { + for (let i = defaultIndex - 1; i >= 0 && anchor < 0; i -= 1) { + anchor = ordered.findIndex(o => o.id === BUILTIN_PAGES[i].id) + } + } + if (anchor >= 0) ordered.splice(anchor + 1, 0, e) + else if (defaultIndex >= 0) ordered.unshift(e) + else ordered.push(e) } return ordered }, [prefs?.nav_order, analysisEntries]) @@ -207,7 +219,18 @@ export function SettingsMenuSettingsPanel() { if (e) { result.push(e); seen.add(id) } } for (const e of allEntries) { - if (!seen.has(e.id)) result.push(e) + if (seen.has(e.id)) continue + // 与 allEntries 同一语义: 未保存的新内置页插回默认位置而非追加到末尾 + const defaultIndex = BUILTIN_PAGES.findIndex(p => p.id === e.id) + let anchor = -1 + if (defaultIndex > 0) { + for (let i = defaultIndex - 1; i >= 0 && anchor < 0; i -= 1) { + anchor = result.findIndex(o => o.id === BUILTIN_PAGES[i].id) + } + } + if (anchor >= 0) result.splice(anchor + 1, 0, e) + else if (defaultIndex >= 0) result.unshift(e) + else result.push(e) } return result }, [localOrder, prefs?.nav_order, allEntries]) diff --git a/frontend/src/router.tsx b/frontend/src/router.tsx index dcfea77..8fb0297 100644 --- a/frontend/src/router.tsx +++ b/frontend/src/router.tsx @@ -18,6 +18,7 @@ import { const Watchlist = lazy(() => import('./pages/Watchlist').then(m => ({ default: m.Watchlist }))) const Screener = lazy(() => import('./pages/Screener').then(m => ({ default: m.Screener }))) const Backtest = lazy(() => import('./pages/Backtest').then(m => ({ default: m.Backtest }))) +const Mining = lazy(() => import('./pages/Mining').then(m => ({ default: m.Mining }))) const Financials = lazy(() => import('./pages/Financials').then(m => ({ default: m.Financials }))) const Data = lazy(() => import('./pages/Data').then(m => ({ default: m.Data }))) const Monitor = lazy(() => import('./pages/Monitor').then(m => ({ default: m.Monitor }))) @@ -48,6 +49,7 @@ const CORE_ROUTE_PATHS = new Set([ '/watchlist', '/screener', '/backtest', + '/mining', '/financials', '/data', '/monitor', @@ -119,6 +121,7 @@ export const router = createBrowserRouter([ { path: 'watchlist', element: }, { path: 'screener', element: }, { path: 'backtest', element: }, + { path: 'mining', element: }, { path: 'financials', element: }, { path: 'data', element: }, { path: 'monitor', element: }, diff --git a/gui-test-screenshots/phase_t1_overview.png b/gui-test-screenshots/phase_t1_overview.png new file mode 100644 index 0000000..7eaf5b6 Binary files /dev/null and b/gui-test-screenshots/phase_t1_overview.png differ diff --git a/gui-test-screenshots/phase_t2_segments.png b/gui-test-screenshots/phase_t2_segments.png new file mode 100644 index 0000000..7eaf5b6 Binary files /dev/null and b/gui-test-screenshots/phase_t2_segments.png differ diff --git a/gui-test-screenshots/phase_t3_filter_panel.png b/gui-test-screenshots/phase_t3_filter_panel.png new file mode 100644 index 0000000..7df8fbb Binary files /dev/null and b/gui-test-screenshots/phase_t3_filter_panel.png differ diff --git a/gui-test-screenshots/phase_t4_mobile.png b/gui-test-screenshots/phase_t4_mobile.png new file mode 100644 index 0000000..352dfa2 Binary files /dev/null and b/gui-test-screenshots/phase_t4_mobile.png differ diff --git a/gui-test-screenshots/t1-desktop-empty-dark.png b/gui-test-screenshots/t1-desktop-empty-dark.png new file mode 100644 index 0000000..cb5d763 Binary files /dev/null and b/gui-test-screenshots/t1-desktop-empty-dark.png differ diff --git a/gui-test-screenshots/t2-validation-error-dark.png b/gui-test-screenshots/t2-validation-error-dark.png new file mode 100644 index 0000000..6bfb7be Binary files /dev/null and b/gui-test-screenshots/t2-validation-error-dark.png differ diff --git a/gui-test-screenshots/t3-run-failed-dark.png b/gui-test-screenshots/t3-run-failed-dark.png new file mode 100644 index 0000000..3d4c7ee Binary files /dev/null and b/gui-test-screenshots/t3-run-failed-dark.png differ diff --git a/gui-test-screenshots/t4-1024-dark.png b/gui-test-screenshots/t4-1024-dark.png new file mode 100644 index 0000000..6fe9d5e Binary files /dev/null and b/gui-test-screenshots/t4-1024-dark.png differ diff --git a/gui-test-screenshots/t5-390-dark.png b/gui-test-screenshots/t5-390-dark.png new file mode 100644 index 0000000..6fa697c Binary files /dev/null and b/gui-test-screenshots/t5-390-dark.png differ diff --git a/gui-test-screenshots/t6-360-dark.png b/gui-test-screenshots/t6-360-dark.png new file mode 100644 index 0000000..a0a6dd1 Binary files /dev/null and b/gui-test-screenshots/t6-360-dark.png differ diff --git a/gui-test-screenshots/t7-360-light.png b/gui-test-screenshots/t7-360-light.png new file mode 100644 index 0000000..555cfe8 Binary files /dev/null and b/gui-test-screenshots/t7-360-light.png differ diff --git a/gui-test-screenshots/t8-desktop-light.png b/gui-test-screenshots/t8-desktop-light.png new file mode 100644 index 0000000..dcaac48 Binary files /dev/null and b/gui-test-screenshots/t8-desktop-light.png differ