mirror of
https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git
synced 2026-09-12 14:24:15 +08:00
merge: 同步 main (辐射本地读 + 传输压缩/网络设置) 到全量分钟分支
冲突两处均为 minute-refresh status 字段两侧各自新增, 两者全保留 (healthy + provider/provider_effective/repair_only)。
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
"""K 线 / 同步 API。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
from datetime import date, timedelta
|
||||
@@ -9,7 +11,7 @@ from zoneinfo import ZoneInfo
|
||||
from functools import lru_cache
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi import APIRouter, HTTPException, Query, Request, Response
|
||||
|
||||
from app.indicators.pipeline import compute_enriched, compute_enriched_single
|
||||
from app.market_time import cn_now, cn_today, in_continuous_session
|
||||
@@ -22,6 +24,41 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/kline", tags=["kline"])
|
||||
|
||||
|
||||
def _gzip_payload(request: Request, payload: dict, *, pref_key: str) -> dict | Response:
|
||||
"""大 JSON 响应的传输压缩: 偏好开启 + 客户端接受 gzip + 响应超阈值才压。
|
||||
|
||||
分时/日K批量各自独立偏好键 (网络设置里大开关批量、子开关单独控制)。
|
||||
level 6 实测 13MB ≈ 290ms CPU 压掉 87%; level 9 要 2.5s 不可用。
|
||||
datetime → isoformat, 与 FastAPI jsonable_encoder 输出一致
|
||||
(前端 since 增量按字符串字典序比较, 格式必须与非压缩路径相同)。
|
||||
"""
|
||||
from app.services import preferences as _prefs
|
||||
_getters = {
|
||||
"minute_batch_compress": _prefs.get_minute_batch_compress,
|
||||
"daily_batch_compress": _prefs.get_daily_batch_compress,
|
||||
}
|
||||
getter = _getters.get(pref_key)
|
||||
compress_on = False
|
||||
if getter is not None:
|
||||
try:
|
||||
compress_on = bool(getter())
|
||||
except Exception: # 偏好读取异常按不压缩返回原样
|
||||
compress_on = False
|
||||
headers = getattr(request, "headers", None) or {}
|
||||
if compress_on and "gzip" in (headers.get("accept-encoding") or ""):
|
||||
raw = json.dumps(
|
||||
payload, ensure_ascii=False, separators=(",", ":"), allow_nan=True,
|
||||
default=lambda o: o.isoformat() if hasattr(o, "isoformat") else str(o),
|
||||
).encode()
|
||||
if len(raw) > 1024:
|
||||
return Response(
|
||||
content=gzip.compress(raw, 6),
|
||||
media_type="application/json",
|
||||
headers={"Content-Encoding": "gzip", "Vary": "Accept-Encoding"},
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _minute_allowed(capset) -> bool:
|
||||
"""是否有分钟K权限 (TickFlow Pro+ 或 custom minute 源)。"""
|
||||
from app.tickflow.capabilities import Cap
|
||||
@@ -567,7 +604,8 @@ def get_daily_batch(request: Request, body: dict):
|
||||
if not sub.is_empty():
|
||||
result[sub["symbol"][0]] = sub.to_dicts()
|
||||
|
||||
return {"data": result}
|
||||
# 日K批量同为大响应端点 (千只自选 MB 级), 与分时各自独立压缩开关
|
||||
return _gzip_payload(request, {"data": result}, pref_key="daily_batch_compress")
|
||||
|
||||
|
||||
@router.post("/minute-batch")
|
||||
@@ -779,12 +817,15 @@ def get_minute_batch(request: Request, body: dict):
|
||||
}
|
||||
result = {sym: rows for sym, rows in result.items() if rows}
|
||||
|
||||
# full_minute_local: 本轮 prefer_local 生效 (本地分区由全量分钟服务供给, 股票未做补拉)
|
||||
return {
|
||||
"data": result,
|
||||
"full_minute_local": full_minute_healthy,
|
||||
"incremental": since_dt is not None,
|
||||
}
|
||||
return _gzip_payload(
|
||||
request,
|
||||
{
|
||||
"data": result,
|
||||
"full_minute_local": full_minute_healthy,
|
||||
"incremental": since_dt is not None,
|
||||
},
|
||||
pref_key="minute_batch_compress",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/minute-range")
|
||||
|
||||
@@ -428,6 +428,14 @@ class DataSourceJobTimeoutPrefs(BaseModel):
|
||||
data_source_long_job_timeout_s: int = Field(ge=60)
|
||||
|
||||
|
||||
class MinuteBatchCompressPrefs(BaseModel):
|
||||
minute_batch_compress: bool
|
||||
|
||||
|
||||
class DailyBatchCompressPrefs(BaseModel):
|
||||
daily_batch_compress: bool
|
||||
|
||||
|
||||
class DatasetFieldMapItem(BaseModel):
|
||||
source: str
|
||||
target: str
|
||||
@@ -506,6 +514,8 @@ def get_preferences() -> dict:
|
||||
"financial_data_provider": preferences.get_financial_provider(),
|
||||
"data_source_job_timeout_s": preferences.get_data_source_job_timeout_s(),
|
||||
"data_source_long_job_timeout_s": preferences.get_data_source_long_job_timeout_s(),
|
||||
"minute_batch_compress": preferences.get_minute_batch_compress(),
|
||||
"daily_batch_compress": preferences.get_daily_batch_compress(),
|
||||
**preferences.get_realtime_quote_scope(),
|
||||
"pipeline_pull_a_share": preferences.get_pipeline_pull_a_share(),
|
||||
"pipeline_pull_etf": preferences.get_pipeline_pull_etf(),
|
||||
@@ -793,6 +803,22 @@ def update_data_source_job_timeouts(req: DataSourceJobTimeoutPrefs) -> dict:
|
||||
return req.model_dump()
|
||||
|
||||
|
||||
@router.put("/preferences/minute-batch-compress")
|
||||
def update_minute_batch_compress(req: MinuteBatchCompressPrefs) -> dict:
|
||||
"""保存分时批量响应的 gzip 传输压缩开关。逐请求即时读取, 保存后立即生效。"""
|
||||
from app.services import preferences
|
||||
preferences.save({"minute_batch_compress": req.minute_batch_compress})
|
||||
return {"minute_batch_compress": preferences.get_minute_batch_compress()}
|
||||
|
||||
|
||||
@router.put("/preferences/daily-batch-compress")
|
||||
def update_daily_batch_compress(req: DailyBatchCompressPrefs) -> dict:
|
||||
"""保存日K批量响应的 gzip 传输压缩开关 (与分时独立)。逐请求即时读取。"""
|
||||
from app.services import preferences
|
||||
preferences.save({"daily_batch_compress": req.daily_batch_compress})
|
||||
return {"daily_batch_compress": preferences.get_daily_batch_compress()}
|
||||
|
||||
|
||||
@router.put("/preferences/mining-schedule")
|
||||
def update_mining_schedule(req: MiningSchedulePrefs) -> dict:
|
||||
"""一次更新周度自动 mining 配置。"""
|
||||
|
||||
@@ -358,7 +358,7 @@ class MinuteRefreshService:
|
||||
def is_healthy(self) -> bool:
|
||||
"""读侧 freshness 判定: 本地 kline_minute 分区是否正被服务持续写入。
|
||||
|
||||
条件: 偏好开启 + 轮询线程存活 + 最近一轮距现在 ≤ max(2×间隔, 30s)。
|
||||
条件: 偏好开启 + 轮询线程存活 + 最近一轮距现在 ≤ max(2*间隔, 30s)。
|
||||
健康时消费方 (自选分时等) 可信任本地分区、跳过批量补拉;
|
||||
连续失败轮不更新 last_round_at → 自动超时判不健康, 无需单独盯 last_error。
|
||||
"""
|
||||
@@ -385,6 +385,7 @@ class MinuteRefreshService:
|
||||
return {
|
||||
"enabled": enabled,
|
||||
"running": running,
|
||||
"healthy": self.is_healthy(),
|
||||
"provider": self.active_provider(),
|
||||
"provider_effective": self._resolve_custom()[1],
|
||||
"repair_only": self.repair_only(),
|
||||
|
||||
@@ -9,6 +9,7 @@ import copy
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -54,14 +55,22 @@ def load() -> dict:
|
||||
return copy.deepcopy(_cache)
|
||||
|
||||
|
||||
_SAVE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def save(updates: dict) -> dict:
|
||||
"""合并写入。返回新内容。"""
|
||||
current = load()
|
||||
current.update(updates)
|
||||
_path().write_text(
|
||||
json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8",
|
||||
)
|
||||
_invalidate_cache()
|
||||
"""合并写入。返回新内容。
|
||||
|
||||
锁内 read-modify-write: FastAPI 同步端点跑线程池, 并行 PUT 各自基于旧快照
|
||||
写盘会互相覆盖 (实测: 压缩总开关并行写分时/日K两键, 后写者把先写者覆盖)。
|
||||
"""
|
||||
with _SAVE_LOCK:
|
||||
current = load()
|
||||
current.update(updates)
|
||||
_path().write_text(
|
||||
json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8",
|
||||
)
|
||||
_invalidate_cache()
|
||||
return current
|
||||
|
||||
|
||||
@@ -238,6 +247,20 @@ def get_data_source_long_job_timeout_s() -> int:
|
||||
return max(DATA_SOURCE_JOB_TIMEOUT_MIN_S, timeout_s)
|
||||
|
||||
|
||||
def get_minute_batch_compress() -> bool:
|
||||
"""分时批量响应是否启用 gzip 传输压缩。默认开启 (公网部署传输是大头);
|
||||
本机/内网可关闭省服务端 CPU。每次请求即时读取, 开关保存后立即生效。
|
||||
"""
|
||||
raw = load().get("minute_batch_compress", True)
|
||||
return bool(raw)
|
||||
|
||||
|
||||
def get_daily_batch_compress() -> bool:
|
||||
"""日K批量响应是否启用 gzip 传输压缩 (与分时各自独立配置)。默认开启。"""
|
||||
raw = load().get("daily_batch_compress", True)
|
||||
return bool(raw)
|
||||
|
||||
|
||||
def _allowed_data_providers() -> set[str]:
|
||||
try:
|
||||
from app.data_providers import custom as custom_sources
|
||||
|
||||
@@ -1241,11 +1241,24 @@ class QuoteService:
|
||||
)
|
||||
|
||||
capset = getattr(self._app_state, "capabilities", None)
|
||||
support = intraday_monitor_support(capset)
|
||||
if not support["available"] or len(symbols) > int(support["max_symbols"]):
|
||||
return self._intraday_signal_evaluator.inject(enriched, [])
|
||||
|
||||
minute_df = fetch_intraday_monitor_batch(sorted(symbols), capset, now=now)
|
||||
# 全量分钟健康时股票读本地分区 (服务按间隔持续落盘, 与 API 同一列契约),
|
||||
# 免去每分钟 bucket 一次的全量 API 拉取; ETF 不在服务 universe 内,
|
||||
# 本地读空/异常回落原 API 路径 (含能力与上限检查)
|
||||
minute_df = pl.DataFrame()
|
||||
if asset_type == "stock":
|
||||
svc = getattr(self._app_state, "minute_refresh", None) if self._app_state else None
|
||||
if svc is not None and svc.is_healthy() and self._repo is not None:
|
||||
try:
|
||||
minute_df = self._repo.get_minute_batch(sorted(symbols), cn_today())
|
||||
except Exception as e: # 本地读异常回落 API
|
||||
logger.warning("分时信号本地读失败, 回退 API 路径: %s", e)
|
||||
minute_df = pl.DataFrame()
|
||||
if minute_df.is_empty():
|
||||
support = intraday_monitor_support(capset)
|
||||
if not support["available"] or len(symbols) > int(support["max_symbols"]):
|
||||
return self._intraday_signal_evaluator.inject(enriched, [])
|
||||
minute_df = fetch_intraday_monitor_batch(sorted(symbols), capset, now=now)
|
||||
prev_close: dict[str, float] = {}
|
||||
available_cols = set(enriched.columns)
|
||||
for row in enriched.filter(pl.col("symbol").is_in(sorted(symbols))).iter_rows(named=True):
|
||||
|
||||
@@ -11,6 +11,8 @@ from __future__ import annotations
|
||||
from types import SimpleNamespace
|
||||
from typing import ClassVar
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.services import quote_service as qs
|
||||
from app.services.index_const import CORE_INDEX_SYMBOLS
|
||||
|
||||
@@ -114,3 +116,76 @@ def test_custom_provider_index_fetch_error_is_soft(monkeypatch):
|
||||
service, captured = _service_with_provider(monkeypatch, _Boom())
|
||||
service._fetch_full_market_quotes()
|
||||
assert len(captured) == 1 and captured[0][0]["symbol"] == "600519.SH"
|
||||
|
||||
|
||||
# ---- 监控分时注入: 全量分钟健康时股票读本地分区 ----
|
||||
|
||||
|
||||
def _injection_env(monkeypatch, *, healthy, local_df, asset_type="stock", symbols=None):
|
||||
"""构造 _inject_intraday_signals 最小环境, 捕获传入 evaluator 的 minute_df。"""
|
||||
symbols = symbols or {"600519.SH"}
|
||||
service = qs.QuoteService()
|
||||
service._repo = SimpleNamespace(
|
||||
get_minute_batch=lambda syms, d: local_df,
|
||||
)
|
||||
engine = SimpleNamespace(
|
||||
intraday_signal_symbols=lambda at: set(symbols) if at == asset_type else set(),
|
||||
)
|
||||
minute_svc = SimpleNamespace(is_healthy=lambda: healthy)
|
||||
service._app_state = SimpleNamespace(minute_refresh=minute_svc)
|
||||
|
||||
import app.services.quote_service as qsm
|
||||
api_calls: list[list[str]] = []
|
||||
monkeypatch.setattr(
|
||||
qsm, "_noop", qsm.__dict__.get("_noop", None), raising=False) # 占位无操作
|
||||
from app.services.kline_sync import intraday_monitor_support
|
||||
monkeypatch.setattr(
|
||||
"app.services.quote_service.logger", qsm.logger, raising=False)
|
||||
# 打桩 API 拉取路径 (健康时不应被调)
|
||||
import app.services.kline_sync as ks
|
||||
monkeypatch.setattr(
|
||||
ks, "fetch_intraday_monitor_batch",
|
||||
lambda symbols, capset, *, now=None: (api_calls.append(list(symbols)), local_df)[1])
|
||||
monkeypatch.setattr(
|
||||
ks, "intraday_monitor_support",
|
||||
lambda capset: {"available": True, "max_symbols": 200, "source": "minute_batch"})
|
||||
|
||||
evaluator = SimpleNamespace(
|
||||
evaluate=lambda minute_df, **kw: (captured.append(minute_df), [])[1],
|
||||
inject=lambda enriched, signals: enriched,
|
||||
)
|
||||
captured: list = []
|
||||
service._intraday_signal_evaluator = evaluator
|
||||
service._intraday_signal_bucket = {}
|
||||
return service, engine, api_calls, captured
|
||||
|
||||
|
||||
def test_intraday_signals_read_local_when_healthy(monkeypatch):
|
||||
"""健康时股票读本地分区, 不触发 API 拉取。"""
|
||||
local = pl.DataFrame({
|
||||
"symbol": ["600519.SH"], "datetime": ["2026-01-15 09:31:00"],
|
||||
"open": [100.0], "high": [101.0], "low": [99.0], "close": [100.5],
|
||||
"volume": [1000.0], "amount": [100500.0],
|
||||
})
|
||||
service, engine, api_calls, captured = _injection_env(monkeypatch, healthy=True, local_df=local)
|
||||
enriched = pl.DataFrame({"symbol": ["600519.SH"], "close": [100.5]})
|
||||
service._inject_intraday_signals(enriched, engine, asset_type="stock")
|
||||
assert api_calls == [] # 未走 API
|
||||
assert captured and captured[0].height == 1 # evaluator 拿到本地数据
|
||||
|
||||
|
||||
def test_intraday_signals_fall_back_to_api_when_unhealthy(monkeypatch):
|
||||
"""不健康 (服务关/挂) 时回落原 API 拉取路径。"""
|
||||
service, engine, api_calls, captured = _injection_env(monkeypatch, healthy=False, local_df=pl.DataFrame())
|
||||
enriched = pl.DataFrame({"symbol": ["600519.SH"], "close": [100.5]})
|
||||
service._inject_intraday_signals(enriched, engine, asset_type="stock")
|
||||
assert api_calls == [["600519.SH"]] # 走了 API
|
||||
|
||||
|
||||
def test_intraday_signals_etf_never_reads_local(monkeypatch):
|
||||
"""ETF 不在全量分钟 universe: 即使健康也走 API 路径。"""
|
||||
service, engine, api_calls, captured = _injection_env(
|
||||
monkeypatch, healthy=True, local_df=pl.DataFrame(), asset_type="etf", symbols={"510300.SH"})
|
||||
enriched = pl.DataFrame({"symbol": ["510300.SH"], "close": [4.0]})
|
||||
service._inject_intraday_signals(enriched, engine, asset_type="etf")
|
||||
assert api_calls == [["510300.SH"]]
|
||||
|
||||
@@ -1011,3 +1011,173 @@ def test_minute_refresh_is_healthy_requires_recent_round(monkeypatch):
|
||||
monkeypatch.setattr(mr.preferences, "get_minute_refresh_enabled", lambda: False)
|
||||
svc._state.last_round_at = time_mod.time() - 10
|
||||
assert svc.is_healthy() is False
|
||||
|
||||
|
||||
# ---------- 测试: 分时批量传输压缩 (网络设置开关) ----------
|
||||
|
||||
|
||||
def test_minute_batch_compress_preference_default_and_toggle(monkeypatch):
|
||||
"""偏好默认开启; 关闭后 getter 立即反映 (逐请求读取, 无缓存)。"""
|
||||
from app.services import preferences as prefs
|
||||
|
||||
monkeypatch.setattr(prefs, "load", lambda: {})
|
||||
assert prefs.get_minute_batch_compress() is True
|
||||
monkeypatch.setattr(prefs, "load", lambda: {"minute_batch_compress": False})
|
||||
assert prefs.get_minute_batch_compress() is False
|
||||
|
||||
|
||||
def _compress_mock_env(monkeypatch, *, compress_on, accept="gzip, deflate"):
|
||||
"""构造 get_minute_batch 压缩路径的最小 mock 环境, 返回 (mock_request, sync_spy)。"""
|
||||
from app.api import kline as kline_api
|
||||
from app.services import preferences as prefs
|
||||
|
||||
monkeypatch.setattr(prefs, "get_minute_batch_compress", lambda: compress_on)
|
||||
|
||||
sync_spy = MagicMock(return_value=_mock_minute_df())
|
||||
monkeypatch.setattr(kline_api.kline_sync, "sync_minute_batch", sync_spy)
|
||||
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.get_etf_symbol_set.return_value = set()
|
||||
mock_repo.get_minute_batch.return_value = _mock_minute_rows("600519.SH", 100)
|
||||
|
||||
mock_capset = MagicMock()
|
||||
mock_capset.has.return_value = True
|
||||
mock_capset.limits.return_value = None
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.app.state.repo = mock_repo
|
||||
mock_request.app.state.capabilities = mock_capset
|
||||
mock_request.headers = {"accept-encoding": accept} if accept else {}
|
||||
return mock_request, sync_spy
|
||||
|
||||
|
||||
def test_get_minute_batch_gzip_response_when_enabled(monkeypatch):
|
||||
"""开关开 + 客户端接受 gzip + 响应超阈值 → 返回 gzip Response, 解压后 JSON 完整。"""
|
||||
import gzip as gzip_mod
|
||||
from app.api import kline as kline_api
|
||||
from fastapi import Response
|
||||
|
||||
mock_request, _ = _compress_mock_env(monkeypatch, compress_on=True)
|
||||
result = kline_api.get_minute_batch(
|
||||
mock_request, {"symbols": ["600519.SH"], "date": "2026-01-15"}
|
||||
)
|
||||
assert isinstance(result, Response)
|
||||
assert result.headers["content-encoding"] == "gzip"
|
||||
import json as json_mod
|
||||
payload = json_mod.loads(gzip_mod.decompress(result.body))
|
||||
assert payload["full_minute_local"] is False
|
||||
assert len(payload["data"]["600519.SH"]) > 0
|
||||
|
||||
|
||||
def test_get_minute_batch_plain_when_disabled(monkeypatch):
|
||||
"""开关关 → 恒返回普通 dict, 不做压缩。"""
|
||||
from app.api import kline as kline_api
|
||||
|
||||
mock_request, _ = _compress_mock_env(monkeypatch, compress_on=False)
|
||||
result = kline_api.get_minute_batch(
|
||||
mock_request, {"symbols": ["600519.SH"], "date": "2026-01-15"}
|
||||
)
|
||||
assert isinstance(result, dict) and "600519.SH" in result["data"]
|
||||
|
||||
|
||||
def test_get_minute_batch_plain_without_accept_encoding(monkeypatch):
|
||||
"""开关开但客户端未声明 gzip (如裸 curl) → 尊重协商, 原样返回。"""
|
||||
from app.api import kline as kline_api
|
||||
|
||||
mock_request, _ = _compress_mock_env(monkeypatch, compress_on=True, accept=None)
|
||||
result = kline_api.get_minute_batch(
|
||||
mock_request, {"symbols": ["600519.SH"], "date": "2026-01-15"}
|
||||
)
|
||||
assert isinstance(result, dict)
|
||||
|
||||
|
||||
# ---------- 测试: 日K批量传输压缩 (与分时独立开关) ----------
|
||||
|
||||
|
||||
def _daily_mock_env(monkeypatch, *, compress_on, accept="gzip, deflate"):
|
||||
"""构造 get_daily_batch 压缩路径的最小 mock。"""
|
||||
from app.api import kline as kline_api
|
||||
from app.services import preferences as prefs
|
||||
|
||||
monkeypatch.setattr(prefs, "get_daily_batch_compress", lambda: compress_on)
|
||||
|
||||
# 20 根日K (date 列), 足以过 1KB 阈值
|
||||
n = 20
|
||||
daily_df = pl.DataFrame({
|
||||
"symbol": ["600519.SH"] * n,
|
||||
"date": [date(2026, 1, 1) + timedelta(days=i) for i in range(n)],
|
||||
"open": [100.0] * n, "high": [101.0] * n,
|
||||
"low": [99.0] * n, "close": [100.5] * n, "volume": [1000.0] * n,
|
||||
})
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.resolve_asset_type.return_value = "stock"
|
||||
mock_repo.get_daily_batch.return_value = daily_df
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.app.state.repo = mock_repo
|
||||
mock_request.headers = {"accept-encoding": accept} if accept else {}
|
||||
return mock_request
|
||||
|
||||
|
||||
def test_daily_batch_gzip_when_enabled(monkeypatch):
|
||||
"""日K压缩开 + 接受 gzip → 压缩 Response, 解压 JSON 完整。"""
|
||||
import gzip as gzip_mod
|
||||
import json as json_mod
|
||||
from app.api import kline as kline_api
|
||||
from fastapi import Response
|
||||
|
||||
req = _daily_mock_env(monkeypatch, compress_on=True)
|
||||
result = kline_api.get_daily_batch(req, {"symbols": ["600519.SH"], "days": 20})
|
||||
assert isinstance(result, Response)
|
||||
assert result.headers["content-encoding"] == "gzip"
|
||||
payload = json_mod.loads(gzip_mod.decompress(result.body))
|
||||
assert len(payload["data"]["600519.SH"]) == 20
|
||||
|
||||
|
||||
def test_daily_batch_plain_when_disabled(monkeypatch):
|
||||
from app.api import kline as kline_api
|
||||
|
||||
req = _daily_mock_env(monkeypatch, compress_on=False)
|
||||
result = kline_api.get_daily_batch(req, {"symbols": ["600519.SH"], "days": 20})
|
||||
assert isinstance(result, dict) and "600519.SH" in result["data"]
|
||||
|
||||
|
||||
def test_daily_batch_independent_from_minute_switch(monkeypatch):
|
||||
"""日K与分时独立: 分时关、日K开 → 日K仍压缩 (helper 按 pref_key 走各自 getter)。"""
|
||||
import gzip as gzip_mod
|
||||
from app.api import kline as kline_api
|
||||
from app.services import preferences as prefs
|
||||
from fastapi import Response
|
||||
|
||||
monkeypatch.setattr(prefs, "get_minute_batch_compress", lambda: False)
|
||||
req = _daily_mock_env(monkeypatch, compress_on=True)
|
||||
result = kline_api.get_daily_batch(req, {"symbols": ["600519.SH"], "days": 20})
|
||||
assert isinstance(result, Response) and gzip_mod.decompress(result.body)
|
||||
|
||||
|
||||
def test_preferences_parallel_saves_do_not_lose_each_other(tmp_path, monkeypatch):
|
||||
"""回归: 并行 save 不同键不得互相覆盖 (压缩总开关并行 PUT 两键的竞态)。
|
||||
|
||||
save 是 read-modify-write, 无锁时两线程同时基于旧快照写盘,
|
||||
后写者会把先写者的更新覆盖掉。
|
||||
"""
|
||||
import threading
|
||||
from app.services import preferences as prefs
|
||||
|
||||
monkeypatch.setattr(prefs, "_path", lambda: tmp_path / "preferences.json")
|
||||
prefs._invalidate_cache()
|
||||
prefs.save({"minute_batch_compress": True})
|
||||
|
||||
barrier = threading.Barrier(2)
|
||||
|
||||
def write_key(key: str) -> None:
|
||||
barrier.wait() # 尽量同时进入 save
|
||||
prefs.save({key: False})
|
||||
|
||||
t1 = threading.Thread(target=write_key, args=("minute_batch_compress",))
|
||||
t2 = threading.Thread(target=write_key, args=("daily_batch_compress",))
|
||||
t1.start(); t2.start(); t1.join(); t2.join()
|
||||
|
||||
final = prefs.load()
|
||||
assert final["minute_batch_compress"] is False
|
||||
assert final["daily_batch_compress"] is False
|
||||
|
||||
@@ -131,7 +131,7 @@ export function ActiveJobCard({ job }: { job: PipelineJob }) {
|
||||
{job.error.includes('超时自动取消') && (
|
||||
<div className="mt-1 text-[10px] text-muted">
|
||||
判定依据是「无进度」而非总时长, 任务只要仍在推进就不会被中断;
|
||||
若网络环境较慢可在 设置 → 超时设置 中调大停滞阈值。
|
||||
若网络环境较慢可在 设置 → 网络设置 中调大停滞阈值。
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -1614,6 +1614,8 @@ export interface Preferences {
|
||||
financial_data_provider?: string
|
||||
data_source_job_timeout_s: number
|
||||
data_source_long_job_timeout_s: number
|
||||
minute_batch_compress: boolean
|
||||
daily_batch_compress: boolean
|
||||
realtime_pull_stock?: boolean
|
||||
realtime_pull_etf?: boolean
|
||||
pipeline_pull_a_share: boolean
|
||||
@@ -1788,8 +1790,23 @@ export const api = {
|
||||
data_source_job_timeout_s: dataSourceJobTimeoutS,
|
||||
data_source_long_job_timeout_s: dataSourceLongJobTimeoutS,
|
||||
}),
|
||||
|
||||
},
|
||||
),
|
||||
|
||||
/** 分时批量响应 gzip 压缩开关 (网络设置) — 逐请求即时生效 */
|
||||
updateMinuteBatchCompress: (enabled: boolean) =>
|
||||
request<Pick<Preferences, 'minute_batch_compress'>>('/api/settings/preferences/minute-batch-compress', {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify({ minute_batch_compress: enabled }),
|
||||
}),
|
||||
|
||||
/** 日K批量响应 gzip 压缩开关 (与分时独立) — 逐请求即时生效 */
|
||||
updateDailyBatchCompress: (enabled: boolean) =>
|
||||
request<Pick<Preferences, 'daily_batch_compress'>>('/api/settings/preferences/daily-batch-compress', {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify({ daily_batch_compress: enabled }),
|
||||
}),
|
||||
updateMinuteSync: (enabled: boolean, days: number, segmentDays?: number) =>
|
||||
request<Preferences>('/api/settings/preferences/minute-sync', {
|
||||
method: 'PUT',
|
||||
@@ -1806,6 +1823,8 @@ export const api = {
|
||||
available: boolean
|
||||
enabled?: boolean
|
||||
running?: boolean
|
||||
/** 读侧 freshness: 本地分区正被服务持续写入 (前端据此切本地读/解除截断) */
|
||||
healthy?: boolean
|
||||
provider?: string
|
||||
provider_effective?: string
|
||||
repair_only?: boolean
|
||||
|
||||
@@ -438,6 +438,14 @@ export function Screener() {
|
||||
)
|
||||
// 分时图依赖分钟K批量数据 (kline.minute.batch), 无数据时开了列也不拉
|
||||
const caps = useCapabilities()
|
||||
// 全量分钟服务健康 (freshness 契约): 健康时本地分区按配置间隔持续落盘,
|
||||
// 分时读本地不受批量上限约束 → 不截断 + prefer_local; 与监控设置页共享缓存
|
||||
const refreshStatus = useQuery({
|
||||
queryKey: ['minute-refresh-status'],
|
||||
queryFn: api.minuteRefreshStatus,
|
||||
refetchInterval: 15000,
|
||||
})
|
||||
const fullMinuteHealthy = !!refreshStatus.data?.healthy
|
||||
const hasMinuteBatch = !!caps.data?.capabilities?.['kline.minute.batch']
|
||||
const intradayVisible = !!intradayColumn && hasMinuteBatch && intradayChartVisible
|
||||
|
||||
@@ -456,11 +464,11 @@ export function Screener() {
|
||||
() => displayRows.map((r: any) => r.symbol),
|
||||
[displayRows],
|
||||
)
|
||||
const intradayTruncated = intradayVisible && allIntradaySymbols.length > minuteBatchCap
|
||||
// 截断到 batch 上限, 一次请求 = 一次数据源调用
|
||||
// 拉模型 (走批量接口) 才截断到 batch 上限; 全量分钟健康时读本地分区无上限
|
||||
const intradayTruncated = intradayVisible && !fullMinuteHealthy && allIntradaySymbols.length > minuteBatchCap
|
||||
const intradaySymbols = useMemo(
|
||||
() => intradayTruncated ? allIntradaySymbols.slice(0, minuteBatchCap) : allIntradaySymbols,
|
||||
[allIntradaySymbols, intradayTruncated, minuteBatchCap],
|
||||
[allIntradaySymbols, intradayTruncated, minuteBatchCap, fullMinuteHealthy],
|
||||
)
|
||||
const intradayRequestSymbols = useMemo(
|
||||
() => [...new Set(intradaySymbols)].sort(),
|
||||
@@ -471,7 +479,7 @@ export function Screener() {
|
||||
const minuteBatch = useQuery({
|
||||
queryKey: QK.minuteBatch(intradaySymbolsKey),
|
||||
// 增量轮询: 读缓存以最后一根为 since 只拉新增, 本地合并为完整序列
|
||||
queryFn: () => fetchMinuteBatchIncremental(qc, QK.minuteBatch(intradaySymbolsKey), intradayRequestSymbols),
|
||||
queryFn: () => fetchMinuteBatchIncremental(qc, QK.minuteBatch(intradaySymbolsKey), intradayRequestSymbols, fullMinuteHealthy),
|
||||
enabled: intradayVisible && intradayRequestSymbols.length > 0,
|
||||
staleTime: 10_000,
|
||||
placeholderData: previousData => previousData,
|
||||
|
||||
@@ -36,7 +36,7 @@ const TABS: readonly TabDef[] = [
|
||||
{ key: 'monitoring', label: '实时监控', icon: Radio, panel: SettingsMonitoringPanel },
|
||||
{ key: 'ext-pages', label: '扩展页面', icon: BarChart3, panel: SettingsExtPagesPanel },
|
||||
{ key: 'signals', label: '信号库', icon: Zap, panel: SettingsCustomSignalsPanel },
|
||||
{ key: 'timeout', label: '超时设置', icon: Clock3, panel: SettingsTimeoutPanel },
|
||||
{ key: 'timeout', label: '网络设置', icon: Clock3, panel: SettingsTimeoutPanel },
|
||||
{ key: 'menus', label: '菜单设置', icon: SlidersHorizontal, panel: SettingsMenuSettingsPanel },
|
||||
{ key: 'system', label: '系统设置', icon: Settings2, panel: SettingsSystemPanel },
|
||||
]
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* 数据任务超时配置卡片 — 从 DataSources 抽出, 放在系统设置页。
|
||||
* 网络设置面板内容 — 任务停滞超时配置 + 分时批量传输压缩开关。
|
||||
* 菜单/Tab 名为「网络设置」(Settings.tsx), 卡片内超时区块标题保持「超时设置」。
|
||||
*/
|
||||
import { useState } from 'react'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
@@ -54,6 +55,47 @@ export function JobTimeoutCard() {
|
||||
const timeoutValuesChanged = regularTimeout !== currentRegularTimeout
|
||||
|| longTimeout !== currentLongTimeout
|
||||
|
||||
const minuteBatchCompress = prefs.data?.minute_batch_compress ?? true
|
||||
const dailyBatchCompress = prefs.data?.daily_batch_compress ?? true
|
||||
// 总开关显示: 任一子开即亮, 全关才灭; 点击 = 全开/全关 (批量写两个子项)
|
||||
const compressAnyOn = minuteBatchCompress || dailyBatchCompress
|
||||
const toggleCompress = useMutation({
|
||||
mutationFn: (enabled: boolean) => api.updateMinuteBatchCompress(enabled),
|
||||
onSuccess: (saved) => {
|
||||
qc.setQueryData<Preferences>(QK.preferences, current => (
|
||||
current ? { ...current, ...saved } : current
|
||||
))
|
||||
toast(saved.minute_batch_compress ? '分时压缩已开启' : '分时压缩已关闭', 'success')
|
||||
},
|
||||
onError: (e: Error) => toast(`保存失败: ${e.message}`, 'error'),
|
||||
})
|
||||
const toggleDailyCompress = useMutation({
|
||||
mutationFn: (enabled: boolean) => api.updateDailyBatchCompress(enabled),
|
||||
onSuccess: (saved) => {
|
||||
qc.setQueryData<Preferences>(QK.preferences, current => (
|
||||
current ? { ...current, ...saved } : current
|
||||
))
|
||||
toast(saved.daily_batch_compress ? '日K压缩已开启' : '日K压缩已关闭', 'success')
|
||||
},
|
||||
onError: (e: Error) => toast(`保存失败: ${e.message}`, 'error'),
|
||||
})
|
||||
const toggleAllCompress = useMutation({
|
||||
mutationFn: async (enabled: boolean) => {
|
||||
const [a, b] = await Promise.all([
|
||||
api.updateMinuteBatchCompress(enabled),
|
||||
api.updateDailyBatchCompress(enabled),
|
||||
])
|
||||
return { ...a, ...b }
|
||||
},
|
||||
onSuccess: (saved) => {
|
||||
qc.setQueryData<Preferences>(QK.preferences, current => (
|
||||
current ? { ...current, ...saved } : current
|
||||
))
|
||||
toast(saved.minute_batch_compress ? '传输压缩已全部开启' : '传输压缩已全部关闭', 'success')
|
||||
},
|
||||
onError: (e: Error) => toast(`保存失败: ${e.message}`, 'error'),
|
||||
})
|
||||
|
||||
const saveJobTimeouts = useMutation({
|
||||
mutationFn: () => api.updateDataSourceJobTimeouts(regularTimeout, longTimeout),
|
||||
onSuccess: (saved) => {
|
||||
@@ -152,6 +194,74 @@ export function JobTimeoutCard() {
|
||||
<span className="block text-[10px] text-muted/60 mt-1.5">默认 30 分钟无进度,最小 1 分钟</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div className="mt-3 pt-3 border-t border-border space-y-3">
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<div className="min-w-0">
|
||||
<span className="block text-xs font-medium text-foreground">数据传输压缩</span>
|
||||
<span className="block text-[10px] text-muted mt-0.5 leading-relaxed">
|
||||
大数据接口(分时、日K)启用 gzip 压缩,响应可缩至约 1/8,公网访问明显更快;本机或内网可关闭以节省服务端 CPU。任一子项开启时总开关为开,点击总开关一键全开/全关,子项可单独微调,立即生效。
|
||||
</span>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => toggleAllCompress.mutate(!compressAnyOn)}
|
||||
disabled={toggleAllCompress.isPending}
|
||||
className={`shrink-0 relative h-5 w-9 rounded-full transition-colors disabled:opacity-40 ${
|
||||
compressAnyOn ? 'bg-accent' : 'bg-elevated'
|
||||
}`}
|
||||
title={compressAnyOn ? '全部关闭' : '全部开启'}
|
||||
>
|
||||
<span className={`absolute top-0.5 h-4 w-4 rounded-full bg-white shadow transition-all ${
|
||||
compressAnyOn ? 'left-[1.125rem]' : 'left-0.5'
|
||||
}`} />
|
||||
</button>
|
||||
</div>
|
||||
<div className="ml-1 space-y-2 border-l-2 border-border/60 pl-3">
|
||||
<CompressToggleRow
|
||||
label="分时数据压缩"
|
||||
desc="分时批量接口(自选/策略分时图,千只标的 MB 级响应)"
|
||||
enabled={minuteBatchCompress}
|
||||
pending={toggleCompress.isPending}
|
||||
onToggle={() => toggleCompress.mutate(!minuteBatchCompress)}
|
||||
/>
|
||||
<CompressToggleRow
|
||||
label="日K数据压缩"
|
||||
desc="日K批量接口(自选/策略日K列,千只标的 MB 级响应)"
|
||||
enabled={dailyBatchCompress}
|
||||
pending={toggleDailyCompress.isPending}
|
||||
onToggle={() => toggleDailyCompress.mutate(!dailyBatchCompress)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
function CompressToggleRow({ label, desc, enabled, pending, onToggle }: {
|
||||
label: string
|
||||
desc: string
|
||||
enabled: boolean
|
||||
pending: boolean
|
||||
onToggle: () => void
|
||||
}) {
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<div className="min-w-0">
|
||||
<span className="block text-[11px] font-medium text-secondary">{label}</span>
|
||||
<span className="block text-[10px] text-muted/80 mt-0.5 leading-relaxed">{desc}</span>
|
||||
</div>
|
||||
<button
|
||||
onClick={onToggle}
|
||||
disabled={pending}
|
||||
className={`shrink-0 relative h-4 w-7 rounded-full transition-colors disabled:opacity-40 ${
|
||||
enabled ? 'bg-accent' : 'bg-elevated'
|
||||
}`}
|
||||
title={enabled ? '点击关闭' : '点击开启'}
|
||||
>
|
||||
<span className={`absolute top-0.5 h-3 w-3 rounded-full bg-white shadow transition-all ${
|
||||
enabled ? 'left-[0.875rem]' : 'left-0.5'
|
||||
}`} />
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user