mirror of
https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git
synced 2026-09-12 17:54:15 +08:00
feat(v0.2): 市场阶段与主线识别 + 因子挖掘全链路 + 数据层完善
- 市场环境: 新增情绪周期6阶段(冰点/启动/主升/高潮/退潮/修复, 连板梯队驱动, EMA平滑+2日确认+弱档否决, 平均段长9.7天)与概念/行业主线排名(涨停梯队聚合, 可配置宽基/风格标签过滤); 市场环境页重构, regime 透明加列, 与5档state并存 - 挖掘: 因子与策略挖掘全链路(API/worker/进程锁/候选库/前端工作台/文档), 周度调度默认关闭且永不自动发布 - 回测: 财务快照因子(点时口径), 批量回测预计算共享下期收益, 信号路径矩阵列依赖展开修复(consecutive_limit_ups 缺列报错) - 数据/性能: enriched 生成与预热治理, 重任务限流, 行情/K线缓存复用, 时区修复 - 测试: 后端全量 914 通过; GUI 黑盒验证截图存证 gui-test-screenshots/
This commit is contained in:
shy3130
@@ -0,0 +1,319 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import weakref
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO
|
||||
|
||||
import polars as pl
|
||||
|
||||
|
||||
class EnrichedGenerationUnavailableError(RuntimeError):
|
||||
"""The enriched dataset has no stable generation available for readers."""
|
||||
|
||||
|
||||
_WRITER_LOCKS_GUARD = threading.Lock()
|
||||
_WRITER_LOCKS: dict[tuple[str, str], threading.RLock] = {}
|
||||
_ACTIVE_PUBLICATIONS: weakref.WeakValueDictionary[str, EnrichedPublication] = (
|
||||
weakref.WeakValueDictionary()
|
||||
)
|
||||
|
||||
|
||||
def _marker_path(data_dir: Path, asset_type: str) -> Path:
|
||||
return Path(data_dir) / f".matrix_generation_{asset_type}.json"
|
||||
|
||||
|
||||
def _writer_lock(data_dir: Path, asset_type: str) -> threading.RLock:
|
||||
key = (str(Path(data_dir).resolve()), asset_type)
|
||||
with _WRITER_LOCKS_GUARD:
|
||||
return _WRITER_LOCKS.setdefault(key, threading.RLock())
|
||||
|
||||
|
||||
def _read_marker(path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise EnrichedGenerationUnavailableError(
|
||||
"enriched data generation marker is invalid"
|
||||
) from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise EnrichedGenerationUnavailableError(
|
||||
"enriched data generation marker is invalid"
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _fsync_directory(path: Path) -> None:
|
||||
if os.name == "nt":
|
||||
return
|
||||
descriptor = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _write_marker(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
||||
try:
|
||||
with temporary.open("x", encoding="utf-8", newline="\n") as stream:
|
||||
json.dump(payload, stream, separators=(",", ":"))
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
_fsync_directory(path.parent)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _unlock_file(stream: BinaryIO) -> None:
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
stream.seek(0)
|
||||
msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
return
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def _try_lock_file(stream: BinaryIO) -> None:
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
stream.seek(0)
|
||||
try:
|
||||
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
except OSError as exc:
|
||||
raise EnrichedGenerationUnavailableError(
|
||||
"another enriched publication is active"
|
||||
) from exc
|
||||
return
|
||||
import fcntl
|
||||
|
||||
try:
|
||||
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as exc:
|
||||
raise EnrichedGenerationUnavailableError(
|
||||
"another enriched publication is active"
|
||||
) from exc
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _exclusive_generation_lock(data_dir: Path, asset_type: str) -> Iterator[None]:
|
||||
lock_path = Path(data_dir) / f".matrix_generation_{asset_type}.lock"
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with (
|
||||
_writer_lock(data_dir, asset_type),
|
||||
lock_path.open("a+b") as stream,
|
||||
):
|
||||
stream.seek(0, os.SEEK_END)
|
||||
if stream.tell() == 0:
|
||||
stream.write(b"0")
|
||||
stream.flush()
|
||||
_try_lock_file(stream)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_unlock_file(stream)
|
||||
|
||||
|
||||
def _process_is_alive(pid: Any) -> bool:
|
||||
if not isinstance(pid, int) or pid <= 0:
|
||||
return False
|
||||
if pid == os.getpid():
|
||||
return True
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except (OSError, PermissionError):
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def _ready_payload(generation: str) -> dict[str, Any]:
|
||||
return {
|
||||
"state": "ready",
|
||||
"generation": generation,
|
||||
"updated_at_ns": time.time_ns(),
|
||||
}
|
||||
|
||||
|
||||
def get_enriched_generation(
|
||||
data_dir: Path,
|
||||
asset_type: str = "stock",
|
||||
*,
|
||||
initialize: bool = True,
|
||||
) -> str:
|
||||
path = _marker_path(data_dir, asset_type)
|
||||
payload = _read_marker(path)
|
||||
if payload is None:
|
||||
if not initialize:
|
||||
raise EnrichedGenerationUnavailableError(
|
||||
"enriched data generation marker is unavailable"
|
||||
)
|
||||
with _exclusive_generation_lock(data_dir, asset_type):
|
||||
payload = _read_marker(path)
|
||||
if payload is None:
|
||||
generation = uuid.uuid4().hex
|
||||
_write_marker(path, _ready_payload(generation))
|
||||
return generation
|
||||
state = payload.get("state", "ready")
|
||||
generation = payload.get("generation")
|
||||
if state != "ready" or not isinstance(generation, str) or not generation:
|
||||
raise EnrichedGenerationUnavailableError(
|
||||
"enriched data is being published; retry after the update finishes"
|
||||
)
|
||||
return generation
|
||||
|
||||
|
||||
def enriched_publication_incomplete(
|
||||
data_dir: Path,
|
||||
asset_type: str = "stock",
|
||||
) -> bool:
|
||||
try:
|
||||
payload = _read_marker(_marker_path(data_dir, asset_type))
|
||||
except EnrichedGenerationUnavailableError:
|
||||
return True
|
||||
if payload is None:
|
||||
return False
|
||||
return (
|
||||
payload.get("state", "ready") != "ready"
|
||||
or not isinstance(payload.get("generation"), str)
|
||||
or not payload["generation"]
|
||||
)
|
||||
|
||||
|
||||
def bump_enriched_generation(data_dir: Path, asset_type: str = "stock") -> str:
|
||||
path = _marker_path(data_dir, asset_type)
|
||||
with _exclusive_generation_lock(data_dir, asset_type):
|
||||
current = _read_marker(path)
|
||||
if current is not None and current.get("state", "ready") != "ready":
|
||||
raise EnrichedGenerationUnavailableError(
|
||||
"cannot bump an incomplete enriched publication"
|
||||
)
|
||||
generation = uuid.uuid4().hex
|
||||
_write_marker(path, _ready_payload(generation))
|
||||
return generation
|
||||
|
||||
|
||||
class EnrichedPublication:
|
||||
"""Publish one logical enriched write batch under a stable generation token."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
data_dir: Path,
|
||||
asset_type: str = "stock",
|
||||
*,
|
||||
recover: bool = False,
|
||||
) -> None:
|
||||
self.data_dir = Path(data_dir)
|
||||
self.asset_type = asset_type
|
||||
self.recover = recover
|
||||
self._publishing = False
|
||||
self._changed = False
|
||||
self._base_generation: str | None = None
|
||||
self._publication_id = uuid.uuid4().hex
|
||||
|
||||
def begin(self) -> None:
|
||||
with _exclusive_generation_lock(self.data_dir, self.asset_type):
|
||||
self._claim_or_verify()
|
||||
|
||||
def mark_changed(self) -> None:
|
||||
if not self._publishing:
|
||||
raise RuntimeError("enriched publication has not started")
|
||||
self._changed = True
|
||||
|
||||
def abandon(self) -> None:
|
||||
if not self._publishing or self._changed:
|
||||
return
|
||||
path = _marker_path(self.data_dir, self.asset_type)
|
||||
with _exclusive_generation_lock(self.data_dir, self.asset_type):
|
||||
current = _read_marker(path)
|
||||
if current is not None and current.get("publication_id") == self._publication_id:
|
||||
_write_marker(path, _ready_payload(str(self._base_generation)))
|
||||
self._publishing = False
|
||||
|
||||
def write_parquet(self, df: pl.DataFrame, out: Path) -> None:
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = out.with_name(f".{out.name}.{uuid.uuid4().hex}.tmp")
|
||||
try:
|
||||
df.write_parquet(temporary)
|
||||
with temporary.open("r+b") as stream:
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
with _exclusive_generation_lock(self.data_dir, self.asset_type):
|
||||
self._claim_or_verify()
|
||||
os.replace(temporary, out)
|
||||
_fsync_directory(out.parent)
|
||||
self._changed = True
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
def commit(self) -> str | None:
|
||||
if not self._changed:
|
||||
return None
|
||||
path = _marker_path(self.data_dir, self.asset_type)
|
||||
with _exclusive_generation_lock(self.data_dir, self.asset_type):
|
||||
current = _read_marker(path)
|
||||
if current is None or current.get("publication_id") != self._publication_id:
|
||||
raise EnrichedGenerationUnavailableError(
|
||||
"enriched publication ownership was lost"
|
||||
)
|
||||
generation = uuid.uuid4().hex
|
||||
_write_marker(path, _ready_payload(generation))
|
||||
self._publishing = False
|
||||
return generation
|
||||
|
||||
def _claim_or_verify(self) -> None:
|
||||
path = _marker_path(self.data_dir, self.asset_type)
|
||||
try:
|
||||
current = _read_marker(path)
|
||||
except EnrichedGenerationUnavailableError:
|
||||
if not self.recover:
|
||||
raise
|
||||
current = None
|
||||
if self._publishing:
|
||||
if current is None or current.get("publication_id") != self._publication_id:
|
||||
raise EnrichedGenerationUnavailableError(
|
||||
"enriched publication ownership was lost"
|
||||
)
|
||||
return
|
||||
_ACTIVE_PUBLICATIONS[self._publication_id] = self
|
||||
if current is not None and current.get("state", "ready") != "ready":
|
||||
current_id = current.get("publication_id")
|
||||
current_owner = _ACTIVE_PUBLICATIONS.get(str(current_id))
|
||||
owner_pid = current.get("owner_pid")
|
||||
if current_owner is not None or (
|
||||
owner_pid != os.getpid() and _process_is_alive(owner_pid)
|
||||
):
|
||||
raise EnrichedGenerationUnavailableError(
|
||||
"another enriched publication is active"
|
||||
)
|
||||
if not self.recover:
|
||||
raise EnrichedGenerationUnavailableError(
|
||||
"another enriched publication is incomplete"
|
||||
)
|
||||
generation = None if current is None else current.get("generation")
|
||||
if not isinstance(generation, str) or not generation:
|
||||
generation = uuid.uuid4().hex
|
||||
self._base_generation = generation
|
||||
_write_marker(path, {
|
||||
"state": "publishing",
|
||||
"generation": generation,
|
||||
"publication_id": self._publication_id,
|
||||
"owner_pid": os.getpid(),
|
||||
"updated_at_ns": time.time_ns(),
|
||||
})
|
||||
self._publishing = True
|
||||
Reference in New Issue
Block a user