mirror of
https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git
synced 2026-09-12 15:34:16 +08:00
fix(ext-pull): 定时拉取不再清空策略结果缓存
扩展表 PullScheduler 每轮成功写入都会经 invalidate_ext_caches 销毁 strategy_cache, 策略页随之下一次轮询整页空白, 且前端会话级防重入 (runAllDateRef) 不会自动补跑, 小服务器上全量重算需分钟级 → 页面长期黑屏。 例行数据刷新改为只失效扩展帧缓存与注册同步状态 (下次策略运行自然 读到新值); 手动上传/配置变更保持全清 (触发下次全量重算) 旧行为。 keep_strategy_cache 沿 fetch_and_ingest → rows_to_parquet → write_ext_parquet → invalidate_ext_caches 显式传递。
This commit is contained in:
@@ -326,18 +326,25 @@ def attach_ext_columns(
|
|||||||
return df
|
return df
|
||||||
|
|
||||||
|
|
||||||
def invalidate_ext_caches(data_dir: Path | None = None) -> None:
|
def invalidate_ext_caches(data_dir: Path | None = None, *, keep_strategy_cache: bool = False) -> None:
|
||||||
"""扩展数据/配置变更后的失效入口 (写入端自动调用)。
|
"""扩展数据/配置变更后的失效入口 (写入端自动调用)。
|
||||||
|
|
||||||
清扩展帧缓存与注册同步状态 (下次读取重新加载), 并清策略结果缓存 ——
|
清扩展帧缓存与注册同步状态 (下次读取重新加载), 并清策略结果缓存 ——
|
||||||
策略历史窗口磁盘缓存里已含旧扩展列。repo 内存 enriched 缓存由
|
策略历史窗口磁盘缓存里已含旧扩展列。repo 内存 enriched 缓存由
|
||||||
API 层 (repo.clear_cache) 补充清理。
|
API 层 (repo.clear_cache) 补充清理。
|
||||||
|
|
||||||
|
keep_strategy_cache=True: 例行数据刷新 (定时拉取) 只失效帧缓存 —— 下次
|
||||||
|
策略运行自然读到新值, 但不销毁已算好的结果。周期性清空会让策略页在两次
|
||||||
|
重算之间整页空白 (小服务器上全量重算需分钟级), 例行刷新的取舍是保留旧
|
||||||
|
结果 (页面秒加载) 而非黑屏; 手动上传/配置变更仍走全清。
|
||||||
"""
|
"""
|
||||||
global _sync_state
|
global _sync_state
|
||||||
root_key = str(_resolve_dir(data_dir))
|
root_key = str(_resolve_dir(data_dir))
|
||||||
for key in [k for k in _frame_cache if k[0] == root_key]:
|
for key in [k for k in _frame_cache if k[0] == root_key]:
|
||||||
_frame_cache.pop(key, None)
|
_frame_cache.pop(key, None)
|
||||||
_sync_state = None
|
_sync_state = None
|
||||||
|
if keep_strategy_cache:
|
||||||
|
return
|
||||||
from app.config import settings as _settings
|
from app.config import settings as _settings
|
||||||
from app.services import strategy_cache
|
from app.services import strategy_cache
|
||||||
|
|
||||||
|
|||||||
@@ -593,6 +593,8 @@ def write_ext_parquet(
|
|||||||
config: ExtConfig,
|
config: ExtConfig,
|
||||||
data_dir: Path,
|
data_dir: Path,
|
||||||
snapshot_date: date | None = None,
|
snapshot_date: date | None = None,
|
||||||
|
*,
|
||||||
|
keep_strategy_cache: bool = False,
|
||||||
) -> int:
|
) -> int:
|
||||||
"""将 DataFrame 写入扩展数据 Parquet。
|
"""将 DataFrame 写入扩展数据 Parquet。
|
||||||
|
|
||||||
@@ -645,20 +647,21 @@ def write_ext_parquet(
|
|||||||
df.write_parquet(out_path)
|
df.write_parquet(out_path)
|
||||||
logger.info("扩展表写入: %s → %s (%d 行)", config.id, out_path, len(df))
|
logger.info("扩展表写入: %s → %s (%d 行)", config.id, out_path, len(df))
|
||||||
# 扩展列已接入 enriched 帧/因子注册表: 写入后必须失效相关缓存
|
# 扩展列已接入 enriched 帧/因子注册表: 写入后必须失效相关缓存
|
||||||
_invalidate_ext_derived(data_dir)
|
_invalidate_ext_derived(data_dir, keep_strategy_cache=keep_strategy_cache)
|
||||||
return len(df)
|
return len(df)
|
||||||
|
|
||||||
|
|
||||||
def _invalidate_ext_derived(data_dir: Path) -> None:
|
def _invalidate_ext_derived(data_dir: Path, *, keep_strategy_cache: bool = False) -> None:
|
||||||
"""扩展数据/配置变更 → 扩展帧缓存 + 因子同步状态 + 策略结果缓存。
|
"""扩展数据/配置变更 → 扩展帧缓存 + 因子同步状态 + 策略结果缓存。
|
||||||
|
|
||||||
惰性导入避免与 ext_factors (反向惰性引用本模块) 构成模块级环。
|
惰性导入避免与 ext_factors (反向惰性引用本模块) 构成模块级环。
|
||||||
repo 内存 enriched 缓存由 API 层 repo.clear_cache() 补充清理。
|
repo 内存 enriched 缓存由 API 层 repo.clear_cache() 补充清理。
|
||||||
|
keep_strategy_cache 语义见 ext_factors.invalidate_ext_caches。
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
from app.factors.ext_factors import invalidate_ext_caches
|
from app.factors.ext_factors import invalidate_ext_caches
|
||||||
|
|
||||||
invalidate_ext_caches(data_dir)
|
invalidate_ext_caches(data_dir, keep_strategy_cache=keep_strategy_cache)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("扩展数据缓存失效失败: %s", e)
|
logger.warning("扩展数据缓存失效失败: %s", e)
|
||||||
|
|
||||||
@@ -736,6 +739,8 @@ def rows_to_parquet(
|
|||||||
config: ExtConfig,
|
config: ExtConfig,
|
||||||
data_dir: Path,
|
data_dir: Path,
|
||||||
snapshot_date: date | None = None,
|
snapshot_date: date | None = None,
|
||||||
|
*,
|
||||||
|
keep_strategy_cache: bool = False,
|
||||||
) -> int:
|
) -> int:
|
||||||
"""将 JSON 行列表转为 DataFrame 写入 Parquet,复用 write_ext_parquet 的存储逻辑。
|
"""将 JSON 行列表转为 DataFrame 写入 Parquet,复用 write_ext_parquet 的存储逻辑。
|
||||||
|
|
||||||
@@ -746,4 +751,7 @@ def rows_to_parquet(
|
|||||||
df = apply_config_mapping(df, config, data_dir)
|
df = apply_config_mapping(df, config, data_dir)
|
||||||
if "symbol" in df.columns:
|
if "symbol" in df.columns:
|
||||||
df = df.with_columns(pl.col("symbol").cast(pl.Utf8))
|
df = df.with_columns(pl.col("symbol").cast(pl.Utf8))
|
||||||
return write_ext_parquet(df, config, data_dir, snapshot_date=snapshot_date)
|
return write_ext_parquet(
|
||||||
|
df, config, data_dir, snapshot_date=snapshot_date,
|
||||||
|
keep_strategy_cache=keep_strategy_cache,
|
||||||
|
)
|
||||||
|
|||||||
@@ -257,10 +257,13 @@ async def fetch_and_ingest(
|
|||||||
config: ExtConfig,
|
config: ExtConfig,
|
||||||
data_dir,
|
data_dir,
|
||||||
target_date: date | None = None,
|
target_date: date | None = None,
|
||||||
|
*,
|
||||||
|
keep_strategy_cache: bool = False,
|
||||||
) -> tuple[int, str]:
|
) -> tuple[int, str]:
|
||||||
"""执行一次拉取: 请求外部 API → 解析响应 → 写入 Parquet。
|
"""执行一次拉取: 请求外部 API → 解析响应 → 写入 Parquet。
|
||||||
|
|
||||||
target_date 默认当日; 历史回补传入目标日期 (写入对应分区)。
|
target_date 默认当日; 历史回补传入目标日期 (写入对应分区)。
|
||||||
|
keep_strategy_cache=True 由定时拉取循环传入: 例行刷新不清策略结果缓存。
|
||||||
Returns:
|
Returns:
|
||||||
(rows_written, date_str)
|
(rows_written, date_str)
|
||||||
"""
|
"""
|
||||||
@@ -269,7 +272,10 @@ async def fetch_and_ingest(
|
|||||||
rows = await fetch_rows_for_date(config, day)
|
rows = await fetch_rows_for_date(config, day)
|
||||||
if not rows:
|
if not rows:
|
||||||
raise ValueError("提取到的行数为 0")
|
raise ValueError("提取到的行数为 0")
|
||||||
n = rows_to_parquet(rows, config, data_dir, snapshot_date=day)
|
n = rows_to_parquet(
|
||||||
|
rows, config, data_dir, snapshot_date=day,
|
||||||
|
keep_strategy_cache=keep_strategy_cache,
|
||||||
|
)
|
||||||
return n, day.isoformat()
|
return n, day.isoformat()
|
||||||
|
|
||||||
|
|
||||||
@@ -498,7 +504,11 @@ class PullScheduler:
|
|||||||
|
|
||||||
# 先执行一次 (启用即拉取, 让用户立刻看到生效)
|
# 先执行一次 (启用即拉取, 让用户立刻看到生效)
|
||||||
try:
|
try:
|
||||||
n, d = await fetch_and_ingest(fresh, self._data_dir)
|
# 例行定时刷新: 不清策略结果缓存 (见 invalidate_ext_caches),
|
||||||
|
# 否则策略页每轮拉取后整页空白, 直到下次全量重算完成。
|
||||||
|
n, d = await fetch_and_ingest(
|
||||||
|
fresh, self._data_dir, keep_strategy_cache=True
|
||||||
|
)
|
||||||
fresh.pull.last_run = datetime.now(timezone.utc).isoformat()
|
fresh.pull.last_run = datetime.now(timezone.utc).isoformat()
|
||||||
fresh.pull.last_status = "success"
|
fresh.pull.last_status = "success"
|
||||||
fresh.pull.last_message = f"{n} rows @ {d}"
|
fresh.pull.last_message = f"{n} rows @ {d}"
|
||||||
|
|||||||
@@ -241,6 +241,40 @@ def test_write_invalidates_frame_cache(data_dir):
|
|||||||
assert out2[COL].to_list() == [0.8]
|
assert out2[COL].to_list() == [0.8]
|
||||||
|
|
||||||
|
|
||||||
|
def test_routine_pull_keeps_strategy_cache_but_default_clears(data_dir):
|
||||||
|
"""定时拉取 (keep_strategy_cache=True) 只失效扩展帧缓存, 不销毁策略结果。
|
||||||
|
|
||||||
|
策略页依赖 strategy_cache 秒加载; 周期性拉取每轮全清会让页面在两次
|
||||||
|
全量重算之间整页空白。手动上传/配置变更 (默认路径) 保持全清旧行为。
|
||||||
|
"""
|
||||||
|
from app.services import strategy_cache
|
||||||
|
|
||||||
|
cfg = _mk_config(data_dir, mode="timeseries")
|
||||||
|
strategy_cache.write_cache(
|
||||||
|
data_dir, "2026-01-05",
|
||||||
|
{"s1": {"total": 1, "as_of": "2026-01-05", "rows": []}},
|
||||||
|
)
|
||||||
|
|
||||||
|
write_ext_parquet(
|
||||||
|
pl.DataFrame({"symbol": ["600000.SH"], "hot": [0.8]}),
|
||||||
|
cfg, data_dir, snapshot_date=date(2026, 1, 5),
|
||||||
|
keep_strategy_cache=True,
|
||||||
|
)
|
||||||
|
# 策略结果保留; 扩展帧缓存仍失效 → 新值立即可见
|
||||||
|
cached = strategy_cache.read_cache(data_dir) or {}
|
||||||
|
assert cached.get("results", {}).get("s1", {}).get("total") == 1
|
||||||
|
frame = _frame([("600000.SH", "2026-01-05", 10.0)])
|
||||||
|
out = ext_factors.attach_ext_columns(frame, include_snapshot=False, data_dir=data_dir)
|
||||||
|
assert out[COL].to_list() == [0.8]
|
||||||
|
|
||||||
|
# 默认路径 (手动写入): 全清
|
||||||
|
write_ext_parquet(
|
||||||
|
pl.DataFrame({"symbol": ["600000.SH"], "hot": [0.9]}),
|
||||||
|
cfg, data_dir, snapshot_date=date(2026, 1, 5),
|
||||||
|
)
|
||||||
|
assert strategy_cache.read_cache(data_dir) is None
|
||||||
|
|
||||||
|
|
||||||
def test_config_field_change_invalidates_sync(data_dir):
|
def test_config_field_change_invalidates_sync(data_dir):
|
||||||
_mk_config(data_dir)
|
_mk_config(data_dir)
|
||||||
ext_factors.ensure_synced(data_dir)
|
ext_factors.ensure_synced(data_dir)
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ async def test_default_landing_date_is_the_beijing_date(at_beijing, monkeypatch)
|
|||||||
seen["fetched"] = target_date
|
seen["fetched"] = target_date
|
||||||
return [{"symbol": "000001.SZ", "v": 1}]
|
return [{"symbol": "000001.SZ", "v": 1}]
|
||||||
|
|
||||||
def fake_write(rows, config, data_dir, snapshot_date):
|
def fake_write(rows, config, data_dir, snapshot_date, **kwargs):
|
||||||
seen["written"] = snapshot_date
|
seen["written"] = snapshot_date
|
||||||
return len(rows)
|
return len(rows)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user