mirror of
https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git
synced 2026-09-12 17:54:15 +08:00
feat(v0.2): 市场阶段与主线识别 + 因子挖掘全链路 + 数据层完善
- 市场环境: 新增情绪周期6阶段(冰点/启动/主升/高潮/退潮/修复, 连板梯队驱动, EMA平滑+2日确认+弱档否决, 平均段长9.7天)与概念/行业主线排名(涨停梯队聚合, 可配置宽基/风格标签过滤); 市场环境页重构, regime 透明加列, 与5档state并存 - 挖掘: 因子与策略挖掘全链路(API/worker/进程锁/候选库/前端工作台/文档), 周度调度默认关闭且永不自动发布 - 回测: 财务快照因子(点时口径), 批量回测预计算共享下期收益, 信号路径矩阵列依赖展开修复(consecutive_limit_ups 缺列报错) - 数据/性能: enriched 生成与预热治理, 重任务限流, 行情/K线缓存复用, 时区修复 - 测试: 后端全量 914 通过; GUI 黑盒验证截图存证 gui-test-screenshots/
This commit is contained in:
shy3130
@@ -1,13 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import threading
|
||||
from datetime import date, timedelta
|
||||
from types import SimpleNamespace
|
||||
|
||||
import polars as pl
|
||||
import pytest
|
||||
|
||||
from app.backtest import worker as worker_module
|
||||
from app.backtest.mining import benchmark_candidate
|
||||
from app.backtest.optimizer import OptimizeConfig
|
||||
from app.backtest.strategy import StrategyBacktestConfig
|
||||
from app.backtest.walkforward import WalkForwardConfig
|
||||
from app.backtest.worker import make_worker_task, run_worker_task
|
||||
from app.backtest.worker import BacktestWorkerError, make_worker_task, run_worker_task
|
||||
from app.enriched_generation import bump_enriched_generation, get_enriched_generation
|
||||
from app.services.mining_jobs import MiningRunStore
|
||||
|
||||
|
||||
def _write_worker_strategy(data_dir) -> None:
|
||||
@@ -88,6 +96,49 @@ def _write_market_data(data_dir, start: date, days: int = 3) -> None:
|
||||
}).write_parquet(instruments_dir / "part.parquet")
|
||||
|
||||
|
||||
def _write_mining_market_data(
|
||||
data_dir,
|
||||
start: date,
|
||||
*,
|
||||
days: int = 219,
|
||||
assets: int = 4,
|
||||
) -> None:
|
||||
symbols = [f"60000{asset}.SH" for asset in range(assets)]
|
||||
for offset in range(days):
|
||||
current = start + timedelta(days=offset)
|
||||
rows = []
|
||||
for asset_id, symbol in enumerate(symbols):
|
||||
close = 10.0 + asset_id + offset * (0.01 + asset_id * 0.002)
|
||||
rows.append({
|
||||
"symbol": symbol,
|
||||
"date": current,
|
||||
"open": close,
|
||||
"high": close * 1.01,
|
||||
"low": close * 0.99,
|
||||
"close": close,
|
||||
"volume": 1000.0 + asset_id * 100.0,
|
||||
"amount": close * (100000.0 + asset_id * 1000.0),
|
||||
"raw_close": close,
|
||||
"raw_high": close * 1.01,
|
||||
"raw_low": close * 0.99,
|
||||
"turnover_rate": 1.0 + asset_id * 0.5 + offset * 0.001,
|
||||
"consecutive_limit_ups": 0,
|
||||
"consecutive_limit_downs": 0,
|
||||
})
|
||||
partition = data_dir / "kline_daily_enriched" / f"date={current.isoformat()}"
|
||||
partition.mkdir(parents=True)
|
||||
pl.DataFrame(rows).write_parquet(partition / "part.parquet")
|
||||
|
||||
instruments_dir = data_dir / "instruments"
|
||||
instruments_dir.mkdir(parents=True)
|
||||
pl.DataFrame({
|
||||
"symbol": symbols,
|
||||
"name": [f"测试{asset}" for asset in range(assets)],
|
||||
"total_shares": [1_000_000_000.0] * assets,
|
||||
"float_shares": [1_000_000_000.0] * assets,
|
||||
}).write_parquet(instruments_dir / "part.parquet")
|
||||
|
||||
|
||||
def test_spawn_worker_returns_compact_result_and_memory_metrics(tmp_path):
|
||||
start = date(2024, 1, 1)
|
||||
data_dir = tmp_path / "data"
|
||||
@@ -187,6 +238,216 @@ def test_spawn_walkforward_reuses_shared_matrix_across_folds(tmp_path):
|
||||
assert result["worker"]["worker_exitcode"] == 0
|
||||
|
||||
|
||||
def test_spawn_mining_writes_four_artifacts_and_returns_compact_summary(tmp_path):
|
||||
start = date(2023, 1, 2)
|
||||
data_dir = tmp_path / "data"
|
||||
_write_mining_market_data(data_dir, start)
|
||||
store = MiningRunStore(data_dir)
|
||||
manifest = store.create(
|
||||
{
|
||||
"factor_names": ["turnover_rate"],
|
||||
"strategy_ids": [],
|
||||
"symbols": None,
|
||||
"asset_type": "stock",
|
||||
"start": (start - timedelta(days=7)).isoformat(),
|
||||
"end": (start + timedelta(days=225)).isoformat(),
|
||||
"budget_profile": "exploratory",
|
||||
"forward_horizon": 1,
|
||||
"commission_pct": 0.0,
|
||||
"stamp_tax_pct": 0.0,
|
||||
"slippage_bps": 0.0,
|
||||
"correlation_threshold": 0.75,
|
||||
"max_combination_factors": 1,
|
||||
"beam_width": 2,
|
||||
"max_finalists": 2,
|
||||
"require_regime": False,
|
||||
},
|
||||
{"generation": get_enriched_generation(data_dir, "stock")},
|
||||
run_id="spawn_mining",
|
||||
)
|
||||
payload = {
|
||||
"run_id": manifest["run_id"],
|
||||
"request": manifest["request"],
|
||||
"data_fingerprint": manifest["data_fingerprint"],
|
||||
"source": "manual",
|
||||
}
|
||||
|
||||
result = run_worker_task(make_worker_task("mining", data_dir, payload))
|
||||
|
||||
assert result["status"] in {"succeeded", "succeeded_with_budget_exhausted"}
|
||||
assert result["factor_count"] == 1
|
||||
assert result["data_as_of"] == (start + timedelta(days=218)).isoformat()
|
||||
assert result["panel_scans"] == 1
|
||||
assert result["matrix_bytes"] > 0
|
||||
assert result["worker"]["worker_exitcode"] == 0
|
||||
assert result["worker"]["serialized_result_bytes"] < 100_000
|
||||
registered = store.get("spawn_mining")["artifacts"] # type: ignore[index]
|
||||
assert set(registered) == {"factors", "correlation", "candidates", "folds"}
|
||||
for name in registered:
|
||||
artifact = store.artifact_path("spawn_mining", name)
|
||||
assert artifact.is_file()
|
||||
frame = pl.read_parquet(artifact)
|
||||
assert frame.columns
|
||||
if name == "folds":
|
||||
assert "n_dates" in frame.columns
|
||||
assert frame.filter(pl.col("regime_state") == "overall")["n_dates"].min() > 0
|
||||
|
||||
def test_spawn_mining_benchmarks_strategy_on_every_outer_fold(tmp_path):
|
||||
start = date(2023, 1, 2)
|
||||
data_dir = tmp_path / "data"
|
||||
_write_mining_market_data(data_dir, start)
|
||||
store = MiningRunStore(data_dir)
|
||||
manifest = store.create(
|
||||
{
|
||||
"factor_names": ["turnover_rate"],
|
||||
"strategy_ids": ["low_volatility_leader"],
|
||||
"symbols": None,
|
||||
"asset_type": "stock",
|
||||
"start": (start - timedelta(days=7)).isoformat(),
|
||||
"end": (start + timedelta(days=225)).isoformat(),
|
||||
"budget_profile": "exploratory",
|
||||
"forward_horizon": 1,
|
||||
"commission_pct": 0.0,
|
||||
"stamp_tax_pct": 0.0,
|
||||
"slippage_bps": 0.0,
|
||||
"correlation_threshold": 0.75,
|
||||
"max_combination_factors": 1,
|
||||
"beam_width": 2,
|
||||
"max_finalists": 2,
|
||||
"require_regime": False,
|
||||
},
|
||||
{"generation": get_enriched_generation(data_dir, "stock")},
|
||||
run_id="spawn_mining_benchmark",
|
||||
)
|
||||
payload = {
|
||||
"run_id": manifest["run_id"],
|
||||
"request": manifest["request"],
|
||||
"data_fingerprint": manifest["data_fingerprint"],
|
||||
"source": "manual",
|
||||
}
|
||||
|
||||
result = run_worker_task(make_worker_task("mining", data_dir, payload))
|
||||
|
||||
assert result["status"] in {"succeeded", "succeeded_with_budget_exhausted"}
|
||||
folds = pl.read_parquet(store.artifact_path("spawn_mining_benchmark", "folds"))
|
||||
benchmark_signature = benchmark_candidate("low_volatility_leader").candidate_id
|
||||
benchmark_rows = folds.filter(
|
||||
(pl.col("evaluation_kind") == "benchmark")
|
||||
& (pl.col("candidate_signature") == benchmark_signature)
|
||||
& (pl.col("regime_state") == "overall")
|
||||
)
|
||||
outer_folds = result["valid_fold_count"] + result["skipped_fold_count"]
|
||||
assert outer_folds >= 1
|
||||
assert benchmark_rows.height == outer_folds
|
||||
selected_rows = folds.filter(
|
||||
(pl.col("evaluation_kind") == "selected")
|
||||
& (pl.col("regime_state") == "overall")
|
||||
)
|
||||
assert selected_rows.height == outer_folds
|
||||
candidates = pl.read_parquet(store.artifact_path("spawn_mining_benchmark", "candidates"))
|
||||
assert benchmark_signature in candidates["signature"].to_list()
|
||||
assert "existing_strategy" in candidates["kind"].to_list()
|
||||
|
||||
|
||||
def test_spawn_mining_rejects_generation_change_after_queue(tmp_path):
|
||||
start = date(2023, 1, 2)
|
||||
data_dir = tmp_path / "data"
|
||||
_write_mining_market_data(data_dir, start)
|
||||
store = MiningRunStore(data_dir)
|
||||
queued_generation = get_enriched_generation(data_dir, "stock")
|
||||
manifest = store.create(
|
||||
{
|
||||
"factor_names": ["turnover_rate"],
|
||||
"strategy_ids": [],
|
||||
"symbols": None,
|
||||
"asset_type": "stock",
|
||||
"start": (start - timedelta(days=7)).isoformat(),
|
||||
"end": (start + timedelta(days=225)).isoformat(),
|
||||
"budget_profile": "exploratory",
|
||||
"forward_horizon": 1,
|
||||
"commission_pct": 0.0,
|
||||
"stamp_tax_pct": 0.0,
|
||||
"slippage_bps": 0.0,
|
||||
"correlation_threshold": 0.75,
|
||||
"max_combination_factors": 1,
|
||||
"beam_width": 2,
|
||||
"max_finalists": 2,
|
||||
"require_regime": False,
|
||||
},
|
||||
{"generation": queued_generation},
|
||||
run_id="stale_generation_mining",
|
||||
)
|
||||
bump_enriched_generation(data_dir, "stock")
|
||||
payload = {
|
||||
"run_id": manifest["run_id"],
|
||||
"request": manifest["request"],
|
||||
"data_fingerprint": manifest["data_fingerprint"],
|
||||
"source": "manual",
|
||||
}
|
||||
|
||||
with pytest.raises(
|
||||
BacktestWorkerError,
|
||||
match="changed after the run was queued",
|
||||
):
|
||||
run_worker_task(make_worker_task("mining", data_dir, payload))
|
||||
|
||||
assert store.get("stale_generation_mining")["artifacts"] == {} # type: ignore[index]
|
||||
|
||||
|
||||
def test_worker_terminates_child_after_cancel_grace(monkeypatch, tmp_path):
|
||||
class FakeQueue:
|
||||
def get(self, timeout):
|
||||
raise queue.Empty
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def join_thread(self):
|
||||
pass
|
||||
|
||||
class FakeEvent:
|
||||
def set(self):
|
||||
pass
|
||||
|
||||
class FakeProcess:
|
||||
def __init__(self):
|
||||
self.alive = True
|
||||
self.exitcode = None
|
||||
|
||||
def start(self):
|
||||
pass
|
||||
|
||||
def is_alive(self):
|
||||
return self.alive
|
||||
|
||||
def join(self, timeout=None):
|
||||
pass
|
||||
|
||||
def terminate(self):
|
||||
self.alive = False
|
||||
self.exitcode = -15
|
||||
|
||||
process = FakeProcess()
|
||||
context = SimpleNamespace(
|
||||
Queue=FakeQueue,
|
||||
Event=FakeEvent,
|
||||
Process=lambda **_kwargs: process,
|
||||
)
|
||||
clock = iter([0.0, 0.0, 0.0, 6.0])
|
||||
monkeypatch.setattr(worker_module.mp, "get_context", lambda _method: context)
|
||||
monkeypatch.setattr(worker_module.time, "monotonic", lambda: next(clock))
|
||||
cancel_event = threading.Event()
|
||||
cancel_event.set()
|
||||
|
||||
with pytest.raises(BacktestWorkerError, match="after cancellation"):
|
||||
run_worker_task(
|
||||
{"kind": "mining", "data_dir": str(tmp_path), "config": {}},
|
||||
cancel_event=cancel_event,
|
||||
)
|
||||
|
||||
assert process.exitcode == -15
|
||||
|
||||
|
||||
def test_spawn_walkforward_skips_folds_before_available_matrix_data(tmp_path):
|
||||
configured_start = date(2024, 1, 1)
|
||||
market_start = configured_start + timedelta(days=4)
|
||||
|
||||
Reference in New Issue
Block a user