mirror of
https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git
synced 2026-09-12 16:44:15 +08:00
feat(v0.2): 市场阶段与主线识别 + 因子挖掘全链路 + 数据层完善
- 市场环境: 新增情绪周期6阶段(冰点/启动/主升/高潮/退潮/修复, 连板梯队驱动, EMA平滑+2日确认+弱档否决, 平均段长9.7天)与概念/行业主线排名(涨停梯队聚合, 可配置宽基/风格标签过滤); 市场环境页重构, regime 透明加列, 与5档state并存 - 挖掘: 因子与策略挖掘全链路(API/worker/进程锁/候选库/前端工作台/文档), 周度调度默认关闭且永不自动发布 - 回测: 财务快照因子(点时口径), 批量回测预计算共享下期收益, 信号路径矩阵列依赖展开修复(consecutive_limit_ups 缺列报错) - 数据/性能: enriched 生成与预热治理, 重任务限流, 行情/K线缓存复用, 时区修复 - 测试: 后端全量 914 通过; GUI 黑盒验证截图存证 gui-test-screenshots/
This commit is contained in:
shy3130
@@ -0,0 +1,736 @@
|
||||
"""Persistent factor and strategy mining HTTP API."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from datetime import date
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
import polars as pl
|
||||
from fastapi import APIRouter, Header, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
from app.backtest.factor import FACTOR_COLUMNS
|
||||
from app.backtest.mining import (
|
||||
MAX_BEAM_WIDTH,
|
||||
MAX_COMBINATION_SIZE,
|
||||
MAX_FINALISTS,
|
||||
evaluate_candidate_gate,
|
||||
)
|
||||
from app.services import preferences
|
||||
from app.services.mining_jobs import (
|
||||
RUN_STATUSES,
|
||||
SUCCESS_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
MiningRunStore,
|
||||
MiningRunStoreError,
|
||||
MiningRunValidationError,
|
||||
)
|
||||
from app.services.mining_preflight import (
|
||||
mining_availability,
|
||||
require_mining_availability,
|
||||
)
|
||||
from app.services.mining_schedule import (
|
||||
MINING_ALGORITHM_VERSION,
|
||||
build_data_fingerprint,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/backtest/mining", tags=["backtest"])
|
||||
_FACTOR_IDS = frozenset(str(item["id"]) for item in FACTOR_COLUMNS)
|
||||
_MAX_ARTIFACT_BYTES = 64 * 1024 * 1024
|
||||
_SSE_POLL_SECONDS = 0.5
|
||||
_SSE_HEARTBEAT_SECONDS = 15.0
|
||||
|
||||
|
||||
class MiningStartRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
factor_names: list[str] = Field(min_length=1, max_length=48)
|
||||
strategy_ids: list[str] = Field(default_factory=list, max_length=8)
|
||||
symbols: list[str] | None = None
|
||||
asset_type: Literal["stock", "etf"] = "stock"
|
||||
start: date | None = None
|
||||
end: date | None = None
|
||||
budget_profile: Literal["exploratory", "balanced", "strict"] = "balanced"
|
||||
commission_pct: float = Field(0.0002, ge=0.0, le=0.05, allow_inf_nan=False)
|
||||
stamp_tax_pct: float = Field(0.0005, ge=0.0, le=0.05, allow_inf_nan=False)
|
||||
slippage_bps: float = Field(5.0, ge=0.0, le=1000.0, allow_inf_nan=False)
|
||||
correlation_threshold: float = Field(0.75, gt=0.0, le=1.0, allow_inf_nan=False)
|
||||
max_combination_factors: int = Field(4, ge=1, le=MAX_COMBINATION_SIZE)
|
||||
beam_width: int = Field(12, ge=1, le=MAX_BEAM_WIDTH)
|
||||
max_finalists: int = Field(MAX_FINALISTS, ge=1, le=MAX_FINALISTS)
|
||||
force: bool = False
|
||||
|
||||
@field_validator("start", "end", mode="before")
|
||||
@classmethod
|
||||
def _iso_dates(cls, value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return date.fromisoformat(value)
|
||||
except ValueError as exc:
|
||||
raise ValueError("dates must use ISO YYYY-MM-DD format") from exc
|
||||
return value
|
||||
|
||||
@field_validator("factor_names", "strategy_ids")
|
||||
@classmethod
|
||||
def _unique_ids(cls, values: list[str]) -> list[str]:
|
||||
if any(not value or len(value) > 120 for value in values):
|
||||
raise ValueError("IDs must contain 1 to 120 characters")
|
||||
if len(set(values)) != len(values):
|
||||
raise ValueError("IDs must be unique")
|
||||
return values
|
||||
|
||||
@field_validator("factor_names")
|
||||
@classmethod
|
||||
def _known_factors(cls, values: list[str]) -> list[str]:
|
||||
unknown = sorted(set(values) - _FACTOR_IDS)
|
||||
if unknown:
|
||||
raise ValueError(f"unknown mining factors: {unknown}")
|
||||
return values
|
||||
|
||||
@field_validator("symbols")
|
||||
@classmethod
|
||||
def _symbols(cls, values: list[str] | None) -> list[str] | None:
|
||||
if values is None:
|
||||
return None
|
||||
cleaned = [value for value in values if value]
|
||||
if len(cleaned) > 10_000:
|
||||
raise ValueError("symbols contains more than 10000 entries")
|
||||
if len(set(cleaned)) != len(cleaned):
|
||||
raise ValueError("symbols must be unique")
|
||||
return cleaned or None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _date_range(self) -> MiningStartRequest:
|
||||
if self.start is not None and self.end is not None and self.start > self.end:
|
||||
raise ValueError("start must not be after end")
|
||||
return self
|
||||
|
||||
|
||||
class MiningSchedulePatch(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
mining_schedule_enabled: bool | None = None
|
||||
mining_schedule_weekday: int | None = Field(None, ge=0, le=4)
|
||||
mining_budget_profile: Literal["balanced", "strict"] | None = None
|
||||
|
||||
|
||||
@router.get("/availability")
|
||||
def get_availability(
|
||||
request: Request,
|
||||
asset_type: Annotated[Literal["stock", "etf"], Query()] = "stock",
|
||||
budget_profile: Annotated[
|
||||
Literal["exploratory", "balanced", "strict"], Query()
|
||||
] = "balanced",
|
||||
start: Annotated[date | None, Query()] = None,
|
||||
end: Annotated[date | None, Query()] = None,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return mining_availability(
|
||||
request.app.state.repo.store.data_dir,
|
||||
asset_type=asset_type,
|
||||
budget_profile=budget_profile,
|
||||
start=start,
|
||||
end=end,
|
||||
).to_dict()
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.get("/runs")
|
||||
def list_runs(
|
||||
request: Request,
|
||||
limit: Annotated[int, Query(ge=1, le=200)] = 50,
|
||||
status: Annotated[list[str] | None, Query()] = None,
|
||||
) -> dict[str, Any]:
|
||||
manager = _manager(request)
|
||||
statuses = None
|
||||
if status:
|
||||
unknown = sorted(set(status) - RUN_STATUSES)
|
||||
if unknown:
|
||||
raise HTTPException(status_code=400, detail=f"unsupported mining statuses: {unknown}")
|
||||
statuses = status
|
||||
try:
|
||||
manifests = manager.store.list_runs(limit=limit, statuses=statuses)
|
||||
return {"items": [_project_run(manager.store, item) for item in manifests]}
|
||||
except MiningRunValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except MiningRunStoreError as exc:
|
||||
raise HTTPException(status_code=500, detail="failed to read mining runs") from exc
|
||||
|
||||
|
||||
@router.post("/runs")
|
||||
def start_run(payload: MiningStartRequest, request: Request) -> dict[str, Any]:
|
||||
manager = _manager(request)
|
||||
worker_request = payload.model_dump(mode="json", exclude={"force"})
|
||||
try:
|
||||
_validate_selected_strategies(
|
||||
request.app.state.strategy_engine,
|
||||
payload.strategy_ids,
|
||||
payload.asset_type,
|
||||
)
|
||||
require_mining_availability(
|
||||
request.app.state.repo.store.data_dir,
|
||||
asset_type=payload.asset_type,
|
||||
budget_profile=payload.budget_profile,
|
||||
start=payload.start,
|
||||
end=payload.end,
|
||||
)
|
||||
fingerprint = build_data_fingerprint(
|
||||
request.app.state.repo,
|
||||
request.app.state,
|
||||
worker_request,
|
||||
)
|
||||
existing = None
|
||||
if not payload.force:
|
||||
from app.services.mining_jobs import (
|
||||
ACTIVE_RUN_STATUSES,
|
||||
SUCCESS_RUN_STATUSES,
|
||||
compute_run_signature,
|
||||
)
|
||||
|
||||
signature = compute_run_signature(worker_request, fingerprint)
|
||||
existing = manager.store.find_by_signature(
|
||||
signature,
|
||||
statuses=ACTIVE_RUN_STATUSES | SUCCESS_RUN_STATUSES,
|
||||
)
|
||||
manifest = manager.start(
|
||||
worker_request,
|
||||
fingerprint,
|
||||
force=payload.force,
|
||||
source="manual",
|
||||
)
|
||||
projected = _project_run(manager.store, manifest)
|
||||
projected["reused"] = existing is not None
|
||||
return projected
|
||||
except (MiningRunValidationError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except MiningRunStoreError as exc:
|
||||
raise HTTPException(status_code=500, detail="failed to persist mining run") from exc
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}")
|
||||
def get_run(run_id: str, request: Request) -> dict[str, Any]:
|
||||
store = _manager(request).store
|
||||
return _project_run(store, _required_manifest(store, run_id))
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/cancel")
|
||||
def cancel_run(run_id: str, request: Request) -> dict[str, Any]:
|
||||
manager = _manager(request)
|
||||
try:
|
||||
return _project_run(manager.store, manager.cancel(run_id))
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="mining run not found") from exc
|
||||
except MiningRunValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/result")
|
||||
def get_result(run_id: str, request: Request) -> dict[str, Any]:
|
||||
store = _manager(request).store
|
||||
manifest = _required_manifest(store, run_id)
|
||||
status = str(manifest["status"])
|
||||
if status not in SUCCESS_RUN_STATUSES:
|
||||
status_code = 409 if status not in TERMINAL_RUN_STATUSES else 422
|
||||
raise HTTPException(
|
||||
status_code=status_code,
|
||||
detail=f"mining result is unavailable for status {status}",
|
||||
)
|
||||
try:
|
||||
summary = store.read_summary(run_id)
|
||||
frames = {
|
||||
name: _read_registered_artifact(store, manifest, name)
|
||||
for name in ("factors", "correlation", "candidates", "folds")
|
||||
}
|
||||
return _project_result(manifest, summary, frames)
|
||||
except (
|
||||
MiningRunStoreError,
|
||||
OSError,
|
||||
pl.exceptions.PolarsError,
|
||||
ValueError,
|
||||
) as exc:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="mining result artifacts are unavailable",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/events")
|
||||
def stream_events(
|
||||
run_id: str,
|
||||
request: Request,
|
||||
last_event_id: str | None = Header(None, alias="Last-Event-ID"),
|
||||
) -> EventSourceResponse:
|
||||
store = _manager(request).store
|
||||
_required_manifest(store, run_id)
|
||||
cursor = _event_cursor(last_event_id)
|
||||
|
||||
async def generate() -> AsyncIterator[dict[str, str]]:
|
||||
nonlocal cursor
|
||||
last_emit = asyncio.get_running_loop().time()
|
||||
terminal_sent = False
|
||||
first_batch = True
|
||||
while not await request.is_disconnected():
|
||||
events = await asyncio.to_thread(store.read_events, run_id, after_id=cursor)
|
||||
if first_batch and events and int(events[0]["id"]) > cursor + 1:
|
||||
summary = await asyncio.to_thread(store.read_summary, run_id)
|
||||
progress = summary.get("progress")
|
||||
if isinstance(progress, Mapping):
|
||||
yield {
|
||||
"id": str(cursor),
|
||||
"event": "progress",
|
||||
"data": json.dumps(progress, ensure_ascii=False, allow_nan=False),
|
||||
}
|
||||
last_emit = asyncio.get_running_loop().time()
|
||||
first_batch = False
|
||||
for event in events:
|
||||
cursor = int(event["id"])
|
||||
event_type = "failed" if event.get("type") == "error" else str(event["type"])
|
||||
payload = dict(event.get("payload") or {})
|
||||
if event_type in TERMINAL_RUN_STATUSES:
|
||||
payload.setdefault("status", event_type)
|
||||
terminal_sent = True
|
||||
yield {
|
||||
"id": str(cursor),
|
||||
"event": event_type,
|
||||
"data": json.dumps(payload, ensure_ascii=False, allow_nan=False),
|
||||
}
|
||||
last_emit = asyncio.get_running_loop().time()
|
||||
manifest = await asyncio.to_thread(store.get, run_id)
|
||||
if manifest is None:
|
||||
return
|
||||
status = str(manifest["status"])
|
||||
if status in TERMINAL_RUN_STATUSES:
|
||||
if not terminal_sent:
|
||||
event_type = "failed" if status == "failed" else status
|
||||
yield {
|
||||
"id": str(cursor),
|
||||
"event": event_type,
|
||||
"data": json.dumps(
|
||||
{"status": status, "message": manifest.get("error")},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
return
|
||||
now = asyncio.get_running_loop().time()
|
||||
if now - last_emit >= _SSE_HEARTBEAT_SECONDS:
|
||||
yield {"event": "heartbeat", "data": "{}"}
|
||||
last_emit = now
|
||||
await asyncio.sleep(_SSE_POLL_SECONDS)
|
||||
|
||||
return EventSourceResponse(generate(), ping=_SSE_HEARTBEAT_SECONDS)
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/candidates/{signature}/promote")
|
||||
def promote_candidate(run_id: str, signature: str, request: Request) -> dict[str, Any]:
|
||||
service = _candidate_service(request)
|
||||
try:
|
||||
return service.promote(run_id, signature)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="mining run or candidate not found") from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/candidates/{signature}/publish")
|
||||
def publish_candidate(run_id: str, signature: str, request: Request) -> dict[str, Any]:
|
||||
service = _candidate_service(request)
|
||||
try:
|
||||
return service.publish(run_id, signature)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail="mining run or candidate not found") from exc
|
||||
except FileExistsError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.get("/config")
|
||||
def get_config() -> dict[str, Any]:
|
||||
return preferences.get_mining_schedule()
|
||||
|
||||
|
||||
@router.patch("/config")
|
||||
def update_config(payload: MiningSchedulePatch) -> dict[str, Any]:
|
||||
current = preferences.get_mining_schedule()
|
||||
updates = payload.model_dump(exclude_none=True)
|
||||
if not updates:
|
||||
raise HTTPException(status_code=400, detail="at least one mining config field is required")
|
||||
merged = {**current, **updates}
|
||||
try:
|
||||
return preferences.set_mining_schedule(
|
||||
merged["mining_schedule_enabled"],
|
||||
merged["mining_schedule_weekday"],
|
||||
merged["mining_budget_profile"],
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
def _manager(request: Request):
|
||||
manager = getattr(request.app.state, "mining_manager", None)
|
||||
if manager is None:
|
||||
raise HTTPException(status_code=503, detail="mining manager is unavailable")
|
||||
return manager
|
||||
|
||||
|
||||
def _candidate_service(request: Request):
|
||||
service = getattr(request.app.state, "mining_candidate_service", None)
|
||||
if service is not None:
|
||||
return service
|
||||
from app.backtest.candidates import CandidateStore
|
||||
from app.services.mining_candidates import MiningCandidateService
|
||||
|
||||
manager = _manager(request)
|
||||
data_dir = request.app.state.repo.store.data_dir
|
||||
monitor_engine = getattr(request.app.state, "monitor_engine", None)
|
||||
service = MiningCandidateService(
|
||||
data_dir,
|
||||
manager.store,
|
||||
CandidateStore(data_dir),
|
||||
request.app.state.strategy_engine,
|
||||
monitor_state_invalidator=(
|
||||
monitor_engine.invalidate_strategy_state
|
||||
if monitor_engine is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
request.app.state.mining_candidate_service = service
|
||||
return service
|
||||
|
||||
|
||||
def _required_manifest(store: MiningRunStore, run_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
manifest = store.get(run_id)
|
||||
except MiningRunValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except MiningRunStoreError as exc:
|
||||
raise HTTPException(status_code=500, detail="failed to read mining run") from exc
|
||||
if manifest is None:
|
||||
raise HTTPException(status_code=404, detail="mining run not found")
|
||||
return manifest
|
||||
|
||||
|
||||
def _project_run(store: MiningRunStore, manifest: Mapping[str, Any]) -> dict[str, Any]:
|
||||
run_id = str(manifest["run_id"])
|
||||
summary = store.read_summary(run_id)
|
||||
events = store.read_events(run_id)
|
||||
source = next(
|
||||
(
|
||||
event.get("payload", {}).get("source")
|
||||
for event in events
|
||||
if event.get("type") == "queued" and event.get("payload", {}).get("source")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if source is None and isinstance(manifest.get("data_fingerprint"), Mapping):
|
||||
source = manifest["data_fingerprint"].get("source")
|
||||
compact = _summary_projection(summary) if manifest["status"] in SUCCESS_RUN_STATUSES else None
|
||||
return {
|
||||
"run_id": run_id,
|
||||
"signature": manifest["run_signature"],
|
||||
"status": manifest["status"],
|
||||
"request": manifest.get("request") or {},
|
||||
"source": source or "manual",
|
||||
"created_at": manifest.get("created_at"),
|
||||
"updated_at": manifest.get("updated_at"),
|
||||
"started_at": manifest.get("started_at"),
|
||||
"finished_at": manifest.get("finished_at"),
|
||||
"data_as_of": summary.get("data_as_of"),
|
||||
"progress": (
|
||||
summary.get("progress")
|
||||
if isinstance(summary.get("progress"), Mapping)
|
||||
else None
|
||||
),
|
||||
"error": manifest.get("error"),
|
||||
"summary": compact,
|
||||
}
|
||||
|
||||
|
||||
def _request_summary(manifest: Mapping[str, Any]) -> dict[str, Any]:
|
||||
request = manifest.get("request") or {}
|
||||
factor_names = request.get("factor_names")
|
||||
strategy_ids = request.get("strategy_ids")
|
||||
return {
|
||||
"asset_type": request.get("asset_type") or "stock",
|
||||
"budget_profile": request.get("budget_profile") or "balanced",
|
||||
"start": request.get("start"),
|
||||
"end": request.get("end"),
|
||||
"factor_count": len(factor_names) if isinstance(factor_names, list) else 0,
|
||||
"strategy_count": len(strategy_ids) if isinstance(strategy_ids, list) else 0,
|
||||
"commission_pct": _finite(request.get("commission_pct")),
|
||||
"stamp_tax_pct": _finite(request.get("stamp_tax_pct")),
|
||||
"slippage_bps": _finite(request.get("slippage_bps")),
|
||||
"correlation_threshold": _finite(request.get("correlation_threshold")),
|
||||
}
|
||||
|
||||
|
||||
def _summary_projection(summary: Mapping[str, Any]) -> dict[str, Any]:
|
||||
worker = summary.get("worker") if isinstance(summary.get("worker"), Mapping) else {}
|
||||
return {
|
||||
"factor_count": int(summary.get("factor_count") or 0),
|
||||
"selected_factor_count": int(summary.get("selected_factor_count") or 0),
|
||||
"candidate_count": int(summary.get("candidate_count") or 0),
|
||||
"valid_fold_count": int(summary.get("valid_fold_count") or 0),
|
||||
"skipped_fold_count": int(summary.get("skipped_fold_count") or 0),
|
||||
"confidence": summary.get("confidence") or "low",
|
||||
"budget_exhausted": bool(summary.get("budget_exhausted", False)),
|
||||
"elapsed_ms": _finite(summary.get("elapsed_ms")),
|
||||
"peak_rss_bytes": _optional_int(worker.get("peak_rss_bytes")),
|
||||
}
|
||||
|
||||
|
||||
def _read_registered_artifact(
|
||||
store: MiningRunStore,
|
||||
manifest: Mapping[str, Any],
|
||||
name: str,
|
||||
) -> pl.DataFrame:
|
||||
artifacts = manifest.get("artifacts")
|
||||
if not isinstance(artifacts, Mapping) or name not in artifacts:
|
||||
raise ValueError(f"mining artifact is not registered: {name}")
|
||||
raw_path = artifacts[name]
|
||||
if not isinstance(raw_path, str):
|
||||
raise ValueError(f"mining artifact registration is invalid: {name}")
|
||||
run_dir = store.artifact_path(str(manifest["run_id"]), name).parent # type: ignore[arg-type]
|
||||
registered = (run_dir / raw_path).resolve()
|
||||
if not registered.is_relative_to(run_dir.resolve()):
|
||||
raise ValueError(f"mining artifact escapes its run directory: {name}")
|
||||
if registered.suffix.lower() != ".parquet" or not registered.is_file():
|
||||
raise ValueError(f"mining artifact is unavailable: {name}")
|
||||
if registered.stat().st_size > _MAX_ARTIFACT_BYTES:
|
||||
raise ValueError(f"mining artifact exceeds size limit: {name}")
|
||||
return pl.read_parquet(registered)
|
||||
|
||||
|
||||
def _project_result(
|
||||
manifest: Mapping[str, Any],
|
||||
summary: Mapping[str, Any],
|
||||
frames: Mapping[str, pl.DataFrame],
|
||||
) -> dict[str, Any]:
|
||||
factors = [_clean_record(row) for row in frames["factors"].to_dicts()]
|
||||
correlation = _project_correlation(frames["correlation"])
|
||||
fold_records = [_project_fold(row) for row in frames["folds"].to_dicts()]
|
||||
candidates = _project_candidates(frames["candidates"], fold_records)
|
||||
selected_signature = candidates[0]["signature"] if candidates else None
|
||||
folds = [
|
||||
_public_fold(row)
|
||||
for row in fold_records
|
||||
if row["regime_state"] == "overall"
|
||||
and (selected_signature is None or row["candidate_signature"] == selected_signature)
|
||||
]
|
||||
regimes = _project_regimes(fold_records, selected_signature)
|
||||
worker = summary.get("worker") if isinstance(summary.get("worker"), Mapping) else {}
|
||||
threshold = _finite((manifest.get("request") or {}).get("correlation_threshold"))
|
||||
correlation["threshold"] = threshold if threshold is not None else 0.75
|
||||
return {
|
||||
"run_id": manifest["run_id"],
|
||||
"methodology_version": summary.get("methodology_version") or "factor_v2",
|
||||
"algorithm_version": summary.get("algorithm_version") or MINING_ALGORITHM_VERSION,
|
||||
"data_as_of": summary.get("data_as_of"),
|
||||
"request_summary": _request_summary(manifest),
|
||||
"summary": _summary_projection(summary),
|
||||
"factors": factors,
|
||||
"correlation": correlation,
|
||||
"regimes": regimes,
|
||||
"candidates": candidates,
|
||||
"folds": folds,
|
||||
"telemetry": {
|
||||
"elapsed_ms": _finite(summary.get("elapsed_ms")),
|
||||
"peak_rss_bytes": _optional_int(worker.get("peak_rss_bytes")),
|
||||
"panel_scans": _optional_int(summary.get("panel_scans")),
|
||||
"matrix_bytes": _optional_int(summary.get("matrix_bytes")),
|
||||
"serialized_result_bytes": _optional_int(worker.get("serialized_result_bytes")),
|
||||
"phase_ms": _finite_mapping(summary.get("phase_ms")),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _project_correlation(frame: pl.DataFrame) -> dict[str, Any]:
|
||||
required = {"factor_x", "factor_y", "rho", "pair_count"}
|
||||
if not required.issubset(frame.columns):
|
||||
raise ValueError("correlation artifact schema is invalid")
|
||||
labels = sorted(set(frame["factor_x"].to_list()) | set(frame["factor_y"].to_list()))
|
||||
positions = {str(label): index for index, label in enumerate(labels)}
|
||||
matrix: list[list[float | None]] = [[None for _ in labels] for _ in labels]
|
||||
counts: list[list[int | None]] = [[None for _ in labels] for _ in labels]
|
||||
for row in frame.iter_rows(named=True):
|
||||
left = positions[str(row["factor_x"])]
|
||||
right = positions[str(row["factor_y"])]
|
||||
matrix[left][right] = _finite(row["rho"])
|
||||
counts[left][right] = _optional_int(row["pair_count"])
|
||||
return {"labels": labels, "matrix": matrix, "pair_counts": counts}
|
||||
|
||||
|
||||
def _project_fold(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
projected = _clean_record(row)
|
||||
projected["selected_factors"] = _json_string_list(row.get("selected_factors_json"))
|
||||
projected["candidate_signature"] = row.get("candidate_signature")
|
||||
projected["regime_state"] = str(row.get("regime_state") or "overall")
|
||||
projected["n_dates"] = int(row.get("n_dates") or 0)
|
||||
return projected
|
||||
|
||||
|
||||
def _public_fold(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
key: row.get(key)
|
||||
for key in (
|
||||
"fold", "label", "train_start", "train_end", "test_start", "test_end",
|
||||
"selected_factors", "total_return", "sharpe", "max_drawdown", "n_trades",
|
||||
"skipped", "reason", "evaluation_kind",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _project_candidates(
|
||||
frame: pl.DataFrame,
|
||||
folds: Sequence[Mapping[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
required = {"signature", "name", "kind", "factor_names_json", "confidence"}
|
||||
if not required.issubset(frame.columns):
|
||||
raise ValueError("candidates artifact schema is invalid")
|
||||
candidates = []
|
||||
for row in frame.to_dicts():
|
||||
candidate = _clean_record(row)
|
||||
candidate.pop("definition_json", None)
|
||||
candidate.pop("factor_names_json", None)
|
||||
candidate["factor_names"] = _json_string_list(row.get("factor_names_json"))
|
||||
signature = str(row["signature"])
|
||||
candidate["folds"] = [
|
||||
_public_fold(fold)
|
||||
for fold in folds
|
||||
if fold["regime_state"] == "overall"
|
||||
and fold["candidate_signature"] == signature
|
||||
]
|
||||
gate = evaluate_candidate_gate(
|
||||
confidence=row.get("confidence"),
|
||||
valid_folds=row.get("valid_folds"),
|
||||
positive_fold_ratio=row.get("oos_positive_fold_ratio"),
|
||||
sharpe=row.get("oos_sharpe"),
|
||||
max_drawdown=row.get("oos_max_drawdown"),
|
||||
n_trades=row.get("oos_n_trades"),
|
||||
)
|
||||
candidate["gate"] = {
|
||||
"qualified": gate.qualified,
|
||||
"reasons": list(gate.reasons),
|
||||
}
|
||||
candidates.append(candidate)
|
||||
candidates.sort(
|
||||
key=lambda item: (
|
||||
-(item.get("oos_sharpe") if item.get("oos_sharpe") is not None else -math.inf),
|
||||
str(item["signature"]),
|
||||
)
|
||||
)
|
||||
return candidates
|
||||
|
||||
|
||||
def _project_regimes(
|
||||
folds: Sequence[Mapping[str, Any]],
|
||||
signature: str | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
labels = {"overall": "整体", "strong": "强势", "range": "震荡", "weak": "弱势"}
|
||||
result = []
|
||||
for state in ("overall", "strong", "range", "weak"):
|
||||
rows = [
|
||||
row
|
||||
for row in folds
|
||||
if row["regime_state"] == state
|
||||
and (signature is None or row["candidate_signature"] == signature)
|
||||
and not row.get("skipped")
|
||||
]
|
||||
result.append({
|
||||
"state": state,
|
||||
"label": labels[state],
|
||||
"n_dates": sum(int(row.get("n_dates") or 0) for row in rows),
|
||||
"total_return": _mean(row.get("total_return") for row in rows),
|
||||
"sharpe": _mean(row.get("sharpe") for row in rows),
|
||||
"max_drawdown": _minimum(row.get("max_drawdown") for row in rows),
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
def _validate_selected_strategies(
|
||||
strategy_engine: Any,
|
||||
strategy_ids: Sequence[str],
|
||||
asset_type: str,
|
||||
) -> None:
|
||||
for strategy_id in strategy_ids:
|
||||
strategy = strategy_engine.get(strategy_id)
|
||||
if strategy.meta.get("research_only"):
|
||||
raise ValueError(f"research template cannot be mined as existing: {strategy_id}")
|
||||
if strategy.execution_backend != "matrix_native":
|
||||
raise ValueError(f"mining strategy is not matrix-native: {strategy_id}")
|
||||
if "1d" not in strategy.meta.get("timeframes", ["1d"]):
|
||||
raise ValueError(f"mining strategy is not daily-compatible: {strategy_id}")
|
||||
if asset_type not in strategy.meta.get("asset_types", ["stock"]):
|
||||
raise ValueError(f"mining strategy does not support {asset_type}: {strategy_id}")
|
||||
|
||||
|
||||
def _event_cursor(value: str | None) -> int:
|
||||
if value in (None, ""):
|
||||
return 0
|
||||
try:
|
||||
cursor = int(value)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail="Last-Event-ID must be an integer") from exc
|
||||
if cursor < 0:
|
||||
raise HTTPException(status_code=400, detail="Last-Event-ID must be non-negative")
|
||||
return cursor
|
||||
|
||||
|
||||
def _json_string_list(value: Any) -> list[str]:
|
||||
if not isinstance(value, str):
|
||||
return []
|
||||
parsed = json.loads(value)
|
||||
if not isinstance(parsed, list) or any(not isinstance(item, str) for item in parsed):
|
||||
raise ValueError("artifact JSON list is invalid")
|
||||
return parsed
|
||||
|
||||
|
||||
def _clean_record(row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
str(key): (_finite(value) if isinstance(value, float) else value)
|
||||
for key, value in row.items()
|
||||
}
|
||||
|
||||
|
||||
def _finite(value: Any) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return number if math.isfinite(number) else None
|
||||
|
||||
|
||||
def _optional_int(value: Any) -> int | None:
|
||||
number = _finite(value)
|
||||
return int(number) if number is not None else None
|
||||
|
||||
|
||||
def _finite_mapping(value: Any) -> dict[str, float] | None:
|
||||
if not isinstance(value, Mapping):
|
||||
return None
|
||||
return {
|
||||
str(key): number
|
||||
for key, item in value.items()
|
||||
if (number := _finite(item)) is not None
|
||||
}
|
||||
|
||||
|
||||
def _mean(values: Sequence[Any] | Any) -> float | None:
|
||||
finite = [number for value in values if (number := _finite(value)) is not None]
|
||||
return sum(finite) / len(finite) if finite else None
|
||||
|
||||
|
||||
def _minimum(values: Sequence[Any] | Any) -> float | None:
|
||||
finite = [number for value in values if (number := _finite(value)) is not None]
|
||||
return min(finite) if finite else None
|
||||
Reference in New Issue
Block a user