aeroxw/tick-stock-panel
1
0
Fork 0
mirror of https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git synced 2026-09-12 17:54:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(v0.2): 市场阶段与主线识别 + 因子挖掘全链路 + 数据层完善

Browse Source 
- 市场环境: 新增情绪周期6阶段(冰点/启动/主升/高潮/退潮/修复, 连板梯队驱动,
  EMA平滑+2日确认+弱档否决, 平均段长9.7天)与概念/行业主线排名(涨停梯队聚合,
  可配置宽基/风格标签过滤); 市场环境页重构, regime 透明加列, 与5档state并存
- 挖掘: 因子与策略挖掘全链路(API/worker/进程锁/候选库/前端工作台/文档),
  周度调度默认关闭且永不自动发布
- 回测: 财务快照因子(点时口径), 批量回测预计算共享下期收益,
  信号路径矩阵列依赖展开修复(consecutive_limit_ups 缺列报错)
- 数据/性能: enriched 生成与预热治理, 重任务限流, 行情/K线缓存复用, 时区修复
- 测试: 后端全量 914 通过; GUI 黑盒验证截图存证 gui-test-screenshots/
This commit is contained in:
shy3130
2026-08-16 23:39:07 +08:00
parent eb869c7ad2
commit 697c27bb02
129 changed files with 20324 additions and 602 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/backtest/matrix.py
+168
View File
@@ -59,6 +59,17 @@ _ROLLING_MATERIALIZED_WINDOW_BUDGET_BYTES = 32 * 1024 * 1024
_MATRIX_DISK_CACHE_DEFAULT_MAX_BYTES = 512 * 1024 * 1024
logger = logging.getLogger(__name__)
class MatrixPrewarmCancelledError(RuntimeError):
"""A matrix cache prewarm was cancelled during application shutdown."""
def _raise_if_matrix_cancelled(cancel_event: threading.Event | None) -> None:
if cancel_event is not None and cancel_event.is_set():
raise MatrixPrewarmCancelledError("matrix cache prewarm cancelled")
_MATRIX_DISK_CACHE_LOCK = threading.RLock()
_MATRIX_DISK_CACHE_LEASES: dict[str, int] = {}
_MATRIX_DISK_CACHE_PENDING_DELETE: set[str] = set()
@@ -679,8 +690,10 @@ def load_market_data_matrix_from_parquet(
cache_max_bytes: int = _MATRIX_DISK_CACHE_DEFAULT_MAX_BYTES,
profile_generation: str = "default",
source_generation: str | None = None,
cancel_event: threading.Event | None = None,
) -> MarketDataMatrix:
"""Load a daily market matrix, reusing a covering read-only mmap when possible."""
_raise_if_matrix_cancelled(cancel_event)
if start > end:
raise ValueError("matrix parquet range start must not exceed end")
root = Path(parquet_root)
@@ -729,6 +742,7 @@ def load_market_data_matrix_from_parquet(
instruments,
batch_size=batch_size,
cache_status="disabled",
cancel_event=cancel_event,
)
cache_dir = Path(cache_root)
@@ -807,7 +821,9 @@ def load_market_data_matrix_from_parquet(
source_generation,
batch_size=batch_size,
axis_cache_root=cache_dir,
cancel_event=cancel_event,
)
_raise_if_matrix_cancelled(cancel_event)
_prune_matrix_disk_cache(
cache_dir,
keep=cache_path,
@@ -916,12 +932,15 @@ def _build_market_data_matrix_from_dataset(
*,
batch_size: int,
cache_status: str,
cancel_event: threading.Event | None = None,
) -> MarketDataMatrix:
_raise_if_matrix_cancelled(cancel_event)
filter_expr = _matrix_filter_expression(start, end, symbols)
actual_dates, actual_symbols = _collect_parquet_axes(
dataset,
filter_expr,
batch_size=batch_size,
cancel_event=cancel_event,
)
if not actual_dates or not actual_symbols:
raise ValueError("matrix parquet range contains no market data")
@@ -953,7 +972,9 @@ def _build_market_data_matrix_from_dataset(
parquet_fields,
seen,
batch_size=batch_size,
cancel_event=cancel_event,
)
_raise_if_matrix_cancelled(cancel_event)
names, latest_limits = _populate_matrix_derived_arrays(
actual_symbols,
arrays,
@@ -1037,7 +1058,9 @@ def _build_market_data_matrix_cache_from_dataset(
*,
batch_size: int,
axis_cache_root: Path,
cancel_event: threading.Event | None = None,
) -> None:
_raise_if_matrix_cancelled(cancel_event)
build_started = time.perf_counter()
timing_ms: dict[str, float] = {}
cache_path.parent.mkdir(parents=True, exist_ok=True)
@@ -1057,7 +1080,9 @@ def _build_market_data_matrix_cache_from_dataset(
filter_expr,
batch_size=batch_size,
cache_root=axis_cache_root,
cancel_event=cancel_event,
)
_raise_if_matrix_cancelled(cancel_event)
if not actual_dates or not actual_symbols:
raise ValueError("matrix parquet range contains no market data")
timing_ms["axes"] = round((time.perf_counter() - stage_started) * 1000, 1)
@@ -1101,7 +1126,9 @@ def _build_market_data_matrix_cache_from_dataset(
parquet_fields,
seen,
batch_size=batch_size,
cancel_event=cancel_event,
)
_raise_if_matrix_cancelled(cancel_event)
if not seen.any():
raise ValueError("matrix parquet range contains no requested market data")
timing_ms["scan"] = round((time.perf_counter() - stage_started) * 1000, 1)
@@ -1118,6 +1145,7 @@ def _build_market_data_matrix_cache_from_dataset(
vector_fields=vector_fields,
)
_mask_unseen_staging_fields(fields, seen)
_raise_if_matrix_cancelled(cancel_event)
if "price_limit_pct" in fields:
write_numpy_price_limit_matrix(
fields["price_limit_pct"],
@@ -1147,6 +1175,7 @@ def _build_market_data_matrix_cache_from_dataset(
apply_latest_limits=actual_dates[-1] == _latest_partition_date(root),
)
timing_ms["derived"] = round((time.perf_counter() - stage_started) * 1000, 1)
_raise_if_matrix_cancelled(cancel_event)
stage_started = time.perf_counter()
for values in mapped:
values.flush()
@@ -1181,6 +1210,7 @@ def _build_market_data_matrix_cache_from_dataset(
json.dumps(manifest, ensure_ascii=False, separators=(",", ":")),
encoding="utf-8",
)
_raise_if_matrix_cancelled(cancel_event)
try:
os.replace(temporary, cache_path)
except OSError:
@@ -1313,6 +1343,7 @@ def _scan_matrix_values(
seen: np.ndarray,
*,
batch_size: int,
cancel_event: threading.Event | None = None,
) -> None:
date_to_id = {value: index for index, value in enumerate(actual_dates)}
symbol_to_id = {value: index for index, value in enumerate(actual_symbols)}
@@ -1343,6 +1374,7 @@ def _scan_matrix_values(
**{name: fields[name] for name in parquet_fields},
}
for batch in scanner.to_batches():
_raise_if_matrix_cancelled(cancel_event)
time_ids = _arrow_axis_ids(_batch_column(batch, "date"), date_to_id)
asset_ids = _arrow_axis_ids(_batch_column(batch, "symbol"), symbol_to_id)
flat_ids = time_ids.astype(np.int64) * asset_count + asset_ids
@@ -1898,7 +1930,9 @@ def _load_or_build_matrix_axes(
*,
batch_size: int,
cache_root: Path,
cancel_event: threading.Event | None = None,
) -> tuple[list[date], list[str]]:
_raise_if_matrix_cancelled(cancel_event)
path = _matrix_axis_cache_path(cache_root, parquet_root, start, end, symbols)
previous: dict[str, Any] | None = None
if path.exists():
@@ -1931,6 +1965,7 @@ def _load_or_build_matrix_axes(
dataset,
filter_expr,
batch_size=batch_size,
cancel_event=cancel_event,
)
changed_labels = set()
retained_dates = {value.isoformat() for value in actual_dates}
@@ -1956,6 +1991,7 @@ def _load_or_build_matrix_axes(
)
symbols_set = set(actual_symbols)
for batch in scanner.to_batches():
_raise_if_matrix_cancelled(cancel_event)
retained_dates.update(
value.isoformat()
for value in pc.unique(_batch_column(batch, "date")).to_pylist()
@@ -1971,8 +2007,10 @@ def _load_or_build_matrix_axes(
dataset,
filter_expr,
batch_size=batch_size,
cancel_event=cancel_event,
)
_raise_if_matrix_cancelled(cancel_event)
payload = {
"version": _MATRIX_AXIS_INDEX_VERSION,
"source_partitions": dict(source_partitions),
@@ -1993,6 +2031,7 @@ def _collect_parquet_axes(
filter_expr,
*,
batch_size: int,
cancel_event: threading.Event | None = None,
) -> tuple[list[date], list[str]]:
dates: set[date] = set()
symbols: set[str] = set()
@@ -2003,6 +2042,7 @@ def _collect_parquet_axes(
use_threads=True,
)
for batch in scanner.to_batches():
_raise_if_matrix_cancelled(cancel_event)
dates.update(pc.unique(_batch_column(batch, "date")).to_pylist())
symbols.update(
str(value)
@@ -2998,6 +3038,7 @@ _VALID_REDUCE_MIN = 0
_VALID_REDUCE_MAX = 1
_VALID_REDUCE_MEAN = 2
_VALID_REDUCE_STD = 3
_VALID_REDUCE_SUM = 4
@njit(cache=True, nogil=True, parallel=True)
@@ -3049,6 +3090,8 @@ def _valid_rolling_kernel(
mean = total / window_value
if operation == _VALID_REDUCE_MEAN:
out[row, asset_id] = mean
elif operation == _VALID_REDUCE_SUM:
out[row, asset_id] = total
else:
squared = 0.0
for offset in range(window):
@@ -3187,6 +3230,30 @@ def valid_rolling_std(
)
def valid_rolling_sum(
values: np.ndarray,
valid_mask: np.ndarray,
window: int,
*,
bar_index: ValidBarIndex | None = None,
) -> np.ndarray:
source = np.asarray(values, dtype=np.float32)
valid = np.asarray(valid_mask, dtype=bool) & np.isfinite(source)
index = _resolve_valid_bar_index(source, valid, bar_index)
return _cached_matrix_operation(
"valid_rolling_sum",
(source, valid, index.offsets, index.rows),
{"window": int(window)},
lambda: _valid_rolling_reduce(
source,
valid,
window,
_VALID_REDUCE_SUM,
bar_index=index,
),
)
def rolling_quantile(values: np.ndarray, window: int, quantile: float) -> np.ndarray:
source = np.asarray(values, dtype=np.float32)
q = float(quantile)
@@ -3675,6 +3742,10 @@ _MATRIX_COMPUTED_FEATURES = frozenset({
"turnover_ratio_5d", "log_amount", "amount_ratio_5d",
"gap_return", "intraday_return", "close_position",
"distance_to_high_60d", "distance_from_low_60d",
"max_ret_20d", "ret_skew_20d", "up_days_20d",
"amihud_20d", "turnover_z_60d", "vol_price_corr_20d",
"vwap_bias", "vol_trend_5_60",
"limit_up_count_20d", "limit_up_count_60d",
})
@@ -3872,9 +3943,106 @@ def _compute_matrix_feature(market: MarketDataMatrix, name: str) -> np.ndarray:
return _matrix_relative(market.close, matrix_feature(market, "high_60d"))
if name == "distance_from_low_60d":
return _matrix_relative(market.close, matrix_feature(market, "low_60d"))
if name == "max_ret_20d":
daily = matrix_feature(market, "change_pct")
return valid_rolling_max(daily, np.isfinite(daily), 20)
if name == "ret_skew_20d":
return _matrix_rolling_skew(matrix_feature(market, "change_pct"), 20)
if name == "up_days_20d":
daily = matrix_feature(market, "change_pct")
up = np.where(daily > 0, np.float32(1.0), np.float32(0.0)).astype(np.float32)
up[~np.isfinite(daily)] = np.nan
return valid_rolling_sum(up, np.isfinite(up), 20)
if name == "amihud_20d":
daily = matrix_feature(market, "change_pct")
amount = market.field("amount")
amount_yi = amount / np.float32(1e8)
illiquidity = _matrix_ratio(np.abs(daily), amount_yi)
return valid_rolling_mean(
illiquidity,
close_valid & np.isfinite(illiquidity),
20,
)
if name == "turnover_z_60d":
turnover = market.field("turnover_rate")
valid = close_valid & np.isfinite(turnover)
previous = valid_shift(turnover, 1, valid)
baseline_valid = np.isfinite(previous)
mean = valid_rolling_mean(previous, baseline_valid, 60)
std = valid_rolling_std(previous, baseline_valid, 60, ddof=1)
deviation = _matrix_ratio(turnover - mean, std)
deviation[np.isfinite(std) & (std <= 0)] = np.nan
return deviation
if name == "vol_price_corr_20d":
daily = matrix_feature(market, "change_pct")
return _matrix_rolling_corr(daily, market.volume, close_valid, 20)
if name == "vwap_bias":
amount = market.field("amount")
shares = market.volume * np.float32(100.0)
valid = close_valid & np.isfinite(amount) & (market.volume > 0) & (amount > 0)
vwap = np.full(market.shape, np.nan, dtype=np.float32)
np.divide(amount, shares, out=vwap, where=valid)
return _matrix_relative(market.close, vwap)
if name == "vol_trend_5_60":
volume_valid = close_valid & np.isfinite(market.volume)
fast = valid_rolling_mean(market.volume, volume_valid, 5)
slow = valid_rolling_mean(market.volume, volume_valid, 60)
return _matrix_relative(fast, slow)
if name in {"limit_up_count_20d", "limit_up_count_60d"}:
window = 20 if name == "limit_up_count_20d" else 60
consecutive = market.field("consecutive_limit_ups")
hits = np.where(np.isfinite(consecutive) & (consecutive > 0), np.float32(1.0), np.float32(0.0))
hits = hits.astype(np.float32)
return valid_rolling_sum(hits, close_valid, window)
raise ValueError(f"unsupported matrix feature: {name}")
def _matrix_rolling_skew(values: np.ndarray, window: int) -> np.ndarray:
valid = np.isfinite(values)
first = valid_rolling_mean(values, valid, window)
second = valid_rolling_mean(np.square(values, dtype=np.float32), valid, window)
third = valid_rolling_mean(
(values * values * values).astype(np.float32), valid, window
)
variance = second - np.square(first, dtype=np.float32)
central_third = (
third
- np.float32(3.0) * first * second
+ np.float32(2.0) * np.power(first, 3)
)
out = _matrix_ratio(central_third, np.sqrt(np.power(variance, 3)))
out[np.isfinite(variance) & (variance <= 0)] = np.nan
return out
def _matrix_rolling_corr(
left: np.ndarray, right: np.ndarray, valid_mask: np.ndarray, window: int
) -> np.ndarray:
valid = valid_mask & np.isfinite(left) & np.isfinite(right)
product = (left * right).astype(np.float32)
mean_left = valid_rolling_mean(left, valid, window)
mean_right = valid_rolling_mean(right, valid, window)
mean_product = valid_rolling_mean(product, valid, window)
mean_left_sq = valid_rolling_mean(
np.square(left, dtype=np.float32), valid, window
)
mean_right_sq = valid_rolling_mean(
np.square(right, dtype=np.float32), valid, window
)
covariance = mean_product - mean_left * mean_right
variance_left = mean_left_sq - np.square(mean_left, dtype=np.float32)
variance_right = mean_right_sq - np.square(mean_right, dtype=np.float32)
denominator = np.sqrt(variance_left * variance_right)
out = _matrix_ratio(covariance, denominator)
degenerate = (
np.isfinite(variance_left)
& np.isfinite(variance_right)
& ((variance_left <= 0) | (variance_right <= 0))
)
out[degenerate] = np.nan
return out
def _matrix_ema(values: np.ndarray, valid: np.ndarray, period: int) -> np.ndarray:
return valid_ewm_adjust_false(values, valid, alpha=2.0 / (period + 1.0))