diff --git a/backend/app/api/kline.py b/backend/app/api/kline.py index 68f2404..333bec1 100644 --- a/backend/app/api/kline.py +++ b/backend/app/api/kline.py @@ -33,16 +33,38 @@ def search_instruments( request: Request, q: str = Query("", min_length=0, max_length=50, description="搜索关键词"), limit: int = Query(20, ge=1, le=50), + asset_types: str = Query("stock", description="逗号分隔的资产类型: stock,etf"), ): - """模糊搜索标的 (代码 / 名称)。从内存 instruments 缓存中查。""" - repo = request.app.state.repo - df = repo.get_instruments() - if df.is_empty() or not q.strip(): + """模糊搜索标的 (代码 / 名称)。从内存 instruments 缓存中查。 + + 默认只搜股票, 保持既有调用方行为不变; 自选等场景传 asset_types=stock,etf + 可一并搜出 ETF, 结果附带 asset_type 字段供前端区分。 + """ + if not q.strip(): return {"results": []} - keyword = q.strip().upper() + repo = request.app.state.repo import polars as pl + types = [t.strip() for t in asset_types.split(",") if t.strip()] + parts: list[pl.DataFrame] = [] + for t in types: + df_t = repo.get_instruments_asset(t) + if df_t.is_empty() or "symbol" not in df_t.columns: + continue + # dtype 全部归一到 Utf8: 股票/ETF 两份缓存来源不同 (ETF 含 legacy 合并), 防 concat SchemaError + parts.append(df_t.with_columns([ + pl.col("symbol").cast(pl.Utf8).alias("symbol"), + (pl.col("name").cast(pl.Utf8) if "name" in df_t.columns else pl.lit("")).alias("name"), + (pl.col("code").cast(pl.Utf8) if "code" in df_t.columns else pl.lit("")).alias("code"), + pl.lit(t).alias("asset_type"), + ]).select(["symbol", "name", "code", "asset_type"])) + if not parts: + return {"results": []} + df = pl.concat(parts, how="vertical") + + keyword = q.strip().upper() + # code/symbol 前缀优先,再 name 包含匹配 prefix_mask = ( pl.col("code").str.starts_with(keyword) @@ -64,23 +86,17 @@ def search_instruments( prefix_symbols = set(prefix_hits["symbol"].to_list()) if not prefix_hits.is_empty() else set() contain_hits = df.filter(contains_mask & ~pl.col("symbol").is_in(prefix_symbols)).head(remaining) matched = pl.concat([prefix_hits, contain_hits]) if not prefix_hits.is_empty() else contain_hits - rows = matched.select(["symbol", "name", "code"]).to_dicts() + rows = matched.select(["symbol", "name", "code", "asset_type"]).to_dicts() return {"results": rows} @router.post("/instruments/names") def instruments_names(request: Request, symbols: list[str]): - """批量查股票名称。传入 symbol 列表, 返回 {symbol: name}。""" + """批量查标的名称 (股票 + ETF)。传入 symbol 列表, 返回 {symbol: name}。""" if not symbols: return {"names": {}} repo = request.app.state.repo - df = repo.get_instruments() - if df.is_empty(): - return {"names": {}} - import polars as pl - matched = df.filter(pl.col("symbol").is_in(symbols)).select(["symbol", "name"]) - names = {row["symbol"]: row["name"] for row in matched.iter_rows(named=True)} - return {"names": names} + return {"names": repo.get_name_map(symbols)} def _get_stock_info(repo, symbol: str) -> dict: @@ -101,6 +117,21 @@ def _get_stock_info(repo, symbol: str) -> dict: } +def _get_asset_info(repo, symbol: str, asset_type: str) -> dict: + """非股票标的 (ETF / 指数) 的名称信息 — 从对应 instruments 缓存查, 无股本概念。""" + import polars as pl + try: + df = repo.get_instruments_asset(asset_type) + if df.is_empty() or "symbol" not in df.columns or "name" not in df.columns: + return {} + hit = df.filter(pl.col("symbol") == symbol).head(1) + if hit.is_empty(): + return {} + return {"name": hit["name"][0]} + except Exception: + return {} + + @router.get("/daily") def get_daily( request: Request, @@ -126,11 +157,12 @@ def get_daily( else: start = end - timedelta(days=days) - stock_info = _get_stock_info(repo, symbol) + asset_type = repo.resolve_asset_type(symbol) + stock_info = _get_stock_info(repo, symbol) if asset_type == "stock" else _get_asset_info(repo, symbol, asset_type) stock_name = stock_info.get("name") - # 从 enriched 表读取 (已含前复权 OHLCV + 技术指标 + 信号) - df = repo.get_daily(symbol, start, end) + # 从 enriched 表读取 (已含前复权 OHLCV + 技术指标 + 信号); ETF/指数走独立存储 + df = repo.get_daily_asset(asset_type, symbol, start, end) if df.is_empty(): try: @@ -151,14 +183,14 @@ def get_daily( enriched = compute_enriched(raw, factors=factors) rows = enriched.tail(days).to_dicts() # 即使 live 模式也尝试追加实时蜡烛 - rows = _maybe_inject_live_candle(request, symbol, rows) + rows = _maybe_inject_live_candle(request, symbol, rows, asset_type) resp = {"symbol": symbol, "name": stock_name, "stock_info": stock_info, "rows": rows, "source": "live"} return _attach_ext(resp, repo, symbol, ext_columns) rows = df.to_dicts() # 追加/覆盖今日实时蜡烛 - rows = _maybe_inject_live_candle(request, symbol, rows) + rows = _maybe_inject_live_candle(request, symbol, rows, asset_type) resp = {"symbol": symbol, "name": stock_name, "stock_info": stock_info, "rows": rows, "source": "enriched"} return _attach_ext(resp, repo, symbol, ext_columns) @@ -229,13 +261,21 @@ def _attach_ext(resp: dict, repo, symbol: str, ext_columns: Optional[str]) -> di return resp -def _maybe_inject_live_candle(request: Request, symbol: str, rows: list[dict]) -> list[dict]: - """如果 QuoteService 有实时 enriched 数据, 用实时数据生成今日蜡烛并追加/覆盖。""" - qs = getattr(request.app.state, "quote_service", None) - if not qs: - return rows +def _maybe_inject_live_candle(request: Request, symbol: str, rows: list[dict], asset_type: str = "stock") -> list[dict]: + """如果有当日实时 enriched 数据, 用实时数据生成今日蜡烛并追加/覆盖。 - df_today, enriched_date = qs.get_enriched_today() + stock 走 QuoteService 的股票实时缓存; etf 走 ETF enriched 缓存 (开启实时 ETF + 拉取时为盘中数据, 否则为磁盘最新日, 由下方"非今日不注入"守卫自然跳过)。 + """ + if asset_type == "stock": + qs = getattr(request.app.state, "quote_service", None) + if not qs: + return rows + df_today, enriched_date = qs.get_enriched_today() + elif asset_type == "etf": + df_today, enriched_date = request.app.state.repo.get_enriched_latest_asset("etf") + else: + return rows if df_today.is_empty(): return rows @@ -377,8 +417,17 @@ def get_minute_batch(request: Request, body: dict): if recent_date is not None: trade_date = recent_date - # Step 1: 本地优先 — 一次 scan 读全部 symbol 当日分钟K - df_local = repo.get_minute_batch(symbols, trade_date) + # Step 1: 本地优先 — 一次 scan 读全部 symbol 当日分钟K (股票 / ETF 分钟数据分开存储) + etf_set = repo.get_etf_symbol_set() + stock_syms = [s for s in symbols if s not in etf_set] + etf_syms = [s for s in symbols if s in etf_set] + df_local = repo.get_minute_batch(stock_syms, trade_date) + if etf_syms: + df_etf = repo.get_minute_batch(etf_syms, trade_date, asset_type="etf") + if df_local.is_empty(): + df_local = df_etf + elif not df_etf.is_empty(): + df_local = pl.concat([df_local, df_etf], how="diagonal_relaxed") # 期望条数 (盘中按当前时刻估算, 盘后 240) now = datetime.now() @@ -442,11 +491,12 @@ def get_minute( - 本地无数据或不完整 → 从 TickFlow 实时拉取返回(不写入) """ repo = request.app.state.repo - stock_info = _get_stock_info(repo, symbol) + asset_type = repo.resolve_asset_type(symbol) + stock_info = _get_stock_info(repo, symbol) if asset_type == "stock" else _get_asset_info(repo, symbol, asset_type) stock_name = stock_info.get("name") if trade_date is None: - trade_date = repo.latest_minute_date(symbol) + trade_date = repo.latest_minute_date(symbol, asset_type=asset_type) if trade_date is None: # 本地无任何分钟K,尝试从 TickFlow 拉取当天 trade_date = date.today() @@ -456,7 +506,7 @@ def get_minute( "date": str(trade_date), "rows": df.to_dicts(), "source": "live", } - df = repo.get_minute(symbol, trade_date) + df = repo.get_minute(symbol, trade_date, asset_type=asset_type) # 完整交易日应有 240 条分钟K;如果是今天(盘中),期望条数按已交易分钟估算 expected = 240 diff --git a/backend/app/api/stock_analysis.py b/backend/app/api/stock_analysis.py index 8926fc1..0eb4b58 100644 --- a/backend/app/api/stock_analysis.py +++ b/backend/app/api/stock_analysis.py @@ -121,7 +121,8 @@ def get_levels( repo = request.app.state.repo end = date.today() start = end - timedelta(days=days * 2) - df = repo.get_daily(symbol, start, end) + # 按资产类型分流: ETF/指数走独立 enriched 存储, 股票保持原路径 + df = repo.get_daily_asset(repo.resolve_asset_type(symbol), symbol, start, end) if df.is_empty(): return {"levels": {"sr": [], "pivot": [], "extreme": [], "boll": [], "keltner_s": [], "keltner_m": [], "keltner_l": [], diff --git a/backend/app/api/watchlist.py b/backend/app/api/watchlist.py index 81fc34d..213a530 100644 --- a/backend/app/api/watchlist.py +++ b/backend/app/api/watchlist.py @@ -31,10 +31,10 @@ def _with_names(rows: list[dict], request: Request) -> list[dict]: if not rows: return rows try: - df_i = request.app.state.repo.get_instruments() - if df_i.is_empty() or "symbol" not in df_i.columns or "name" not in df_i.columns: + # 股票 + ETF 名称统一由 repo.get_name_map 解析, 自选列表可混合持有 + name_by_symbol = request.app.state.repo.get_name_map([r.get("symbol") for r in rows]) + if not name_by_symbol: return rows - name_by_symbol = dict(df_i.select(["symbol", "name"]).iter_rows()) return [{**row, "name": name_by_symbol.get(row.get("symbol"))} for row in rows] except Exception as e: # noqa: BLE001 logger.debug("attach watchlist names failed: %s", e) @@ -119,20 +119,42 @@ def watchlist_enriched( if not symbols: return {"rows": [], "as_of": None, "elapsed_ms": 0} + # 按资产拆分自选 symbol; ETF enriched 是独立缓存, 仅自选真的含 ETF 才去加载 + # (避免无 ETF 用户在缓存冷启动时触发 ETF 全量懒加载) + etf_set = repo.get_etf_symbol_set() + stock_symbols = [s for s in symbols if s not in etf_set] + etf_symbols = [s for s in symbols if s in etf_set] + df_e, cache_date = repo.get_enriched_latest() - if df_e.is_empty(): + # 保持原契约: 自选含股票但股票 enriched 未就绪 (预热中) → 返回"未就绪"而非部分结果 + if stock_symbols and df_e.is_empty(): return {"rows": [], "as_of": None, "elapsed_ms": 0} - # 按 symbol 过滤 - df = df_e.filter(pl.col("symbol").is_in(symbols)) - if df.is_empty(): - return {"rows": [], "as_of": str(cache_date) if cache_date else None, "elapsed_ms": 0} + df = df_e.filter(pl.col("symbol").is_in(stock_symbols)) if stock_symbols else pl.DataFrame() - # JOIN instruments 取 name + float_shares + # ETF 行合并; 缺失列 (换手率/涨跌停信号等) 为 null + etf_date = None + if etf_symbols: + df_etf_all, etf_date = repo.get_enriched_latest_asset("etf") + if not df_etf_all.is_empty(): + df_etf = df_etf_all.filter(pl.col("symbol").is_in(etf_symbols)) + if not df_etf.is_empty(): + df = df_etf if df.is_empty() else pl.concat([df, df_etf], how="diagonal_relaxed") + + # as_of 取两类缓存中较旧者, 避免把旧的 ETF 行标成股票缓存日期 + dates = [d for d in (cache_date if stock_symbols else None, etf_date) if d is not None] + as_of = min(dates) if dates else None + if df.is_empty(): + return {"rows": [], "as_of": str(as_of) if as_of else None, "elapsed_ms": 0} + + # JOIN float_shares (仅股票有) + 名称 (股票/ETF 统一走 get_name_map) df_i = repo.get_instruments() - if not df_i.is_empty() and "name" in df_i.columns: - inst_cols = [c for c in ["symbol", "name", "float_shares"] if c in df_i.columns] - df = df.join(df_i.select(inst_cols), on="symbol", how="left") + if not df_i.is_empty() and "float_shares" in df_i.columns: + df = df.join(df_i.select(["symbol", "float_shares"]), on="symbol", how="left") + name_map = repo.get_name_map(df["symbol"].to_list()) + df = df.with_columns( + pl.col("symbol").replace_strict(name_map, default=None, return_dtype=pl.Utf8).alias("name") + ) # 选择内置需要的列 keep = [c for c in _WATCHLIST_COLS + ["name", "float_shares"] if c in df.columns] @@ -204,7 +226,7 @@ def watchlist_enriched( rows = df.to_dicts() elapsed = (time.perf_counter() - t0) * 1000 - return {"rows": rows, "as_of": str(cache_date) if cache_date else None, "elapsed_ms": elapsed} + return {"rows": rows, "as_of": str(as_of) if as_of else None, "elapsed_ms": elapsed} def _parse_ext_columns(ext_columns: str) -> list[tuple[str, str]]: diff --git a/backend/app/services/stock_analyzer.py b/backend/app/services/stock_analyzer.py index c312459..db45c78 100644 --- a/backend/app/services/stock_analyzer.py +++ b/backend/app/services/stock_analyzer.py @@ -45,7 +45,8 @@ def _load_kline(repo, symbol: str) -> pl.DataFrame: end = date.today() start = end - timedelta(days=_KLINE_WINDOW * 2) # 多取一些保证交易日够 - df = repo.get_daily(symbol, start, end) + # 按资产类型分流: ETF/指数走独立 enriched 存储 (无财务数据, 提示词已有兜底) + df = repo.get_daily_asset(repo.resolve_asset_type(symbol), symbol, start, end) if df.is_empty(): return df return df.tail(_KLINE_WINDOW) diff --git a/backend/app/tickflow/repository.py b/backend/app/tickflow/repository.py index 90efae3..857fdf7 100644 --- a/backend/app/tickflow/repository.py +++ b/backend/app/tickflow/repository.py @@ -301,6 +301,9 @@ class KlineRepository: self._etf_live_agg_cache: pl.DataFrame | None = None self._etf_live_agg_cache_date: date | None = None self._etf_instruments_cache: pl.DataFrame | None = None + # symbol 集合 memo (随对应 instruments 缓存失效): 供每请求资产分流用 + self._index_symbol_set_cache: set[str] | None = None + self._etf_symbol_set_cache: set[str] | None = None # ---- enriched 后台预热 ---- # 启动时 compute_indicators (107万行, 低配机 50s+) 移出 lifespan 关键路径, @@ -362,6 +365,11 @@ class KlineRepository: self._refresh_etf_instruments() logger.info("cache refresh step done: ETF instruments (%.2fs)", time.perf_counter() - step) + # ETF enriched 只失效不重建: 下次访问时按新数据懒加载, + # 避免自选无 ETF 的用户在管道后白付全量重算成本 + self._etf_enriched_cache = None + self._etf_enriched_cache_date = None + if background: logger.info("cache refresh: enriched 推后台线程预热") self._start_enriched_warmup() @@ -438,6 +446,8 @@ class KlineRepository: self._etf_live_agg_cache = None self._etf_live_agg_cache_date = None self._etf_instruments_cache = None + self._index_symbol_set_cache = None + self._etf_symbol_set_cache = None def _refresh_enriched(self) -> None: """从 parquet 加载 enriched 最新日到内存 + 构建聚合表。 @@ -854,6 +864,7 @@ class KlineRepository: df = pl.scan_parquet(self._index_inst_glob).collect() if not df.is_empty(): self._index_instruments_cache = df + self._index_symbol_set_cache = None logger.info("index instruments 缓存已加载: %d 只", len(df)) except Exception as e: # noqa: BLE001 logger.debug("index instruments 缓存刷新跳过: %s", e) @@ -878,6 +889,7 @@ class KlineRepository: if parts: df_all = pl.concat(parts, how="diagonal_relaxed").unique(subset=["symbol"], keep="last").sort("symbol") self._etf_instruments_cache = df_all + self._etf_symbol_set_cache = None logger.info("ETF instruments 缓存已加载: %d 只", len(df_all)) def get_enriched_latest(self) -> tuple[pl.DataFrame, date | None]: @@ -1035,11 +1047,50 @@ class KlineRepository: return pl.DataFrame() def get_index_symbol_set(self) -> set[str]: - """返回已缓存指数 symbol 集合。""" - df = self.get_index_instruments() - if df.is_empty() or "symbol" not in df.columns: - return set() - return set(df["symbol"].cast(pl.Utf8).to_list()) + """返回已缓存指数 symbol 集合 (memo, 随 instruments 缓存失效)。""" + if self._index_symbol_set_cache is None: + df = self.get_index_instruments() + if df.is_empty() or "symbol" not in df.columns: + return set() + self._index_symbol_set_cache = set(df["symbol"].cast(pl.Utf8).to_list()) + return self._index_symbol_set_cache + + def get_etf_symbol_set(self) -> set[str]: + """返回已缓存 ETF symbol 集合 (memo, 随 instruments 缓存失效)。""" + if self._etf_symbol_set_cache is None: + df = self.get_etf_instruments() + if df.is_empty() or "symbol" not in df.columns: + return set() + self._etf_symbol_set_cache = set(df["symbol"].cast(pl.Utf8).to_list()) + return self._etf_symbol_set_cache + + def resolve_asset_type(self, symbol: str) -> str: + """按 symbol 判定资产类型: etf / index / stock(默认)。 + + 供 API 层对单标的查询做资产分流 (get_daily_asset 等)。 + ETF/指数集合为 memo, 每请求查询成本可忽略。 + """ + if symbol in self.get_etf_symbol_set(): + return "etf" + if symbol in self.get_index_symbol_set(): + return "index" + return "stock" + + def get_name_map(self, symbols: list[str] | None = None) -> dict[str, str]: + """返回 {symbol: name} 映射, 合并股票 + ETF instruments (股票优先去重)。 + + 自选列表/名称批查等场景的统一名称解析入口, 避免各调用方自行合并两份缓存。 + symbols 非 None 时只返回命中的条目。 + """ + name_map: dict[str, str] = {} + for df in (self.get_instruments(), self.get_etf_instruments()): + if df.is_empty() or "symbol" not in df.columns or "name" not in df.columns: + continue + if symbols is not None: + df = df.filter(pl.col("symbol").is_in(symbols)) + for symbol, name in df.select(["symbol", "name"]).iter_rows(): + name_map.setdefault(symbol, name) + return name_map def enriched_latest_date(self) -> date | None: """返回缓存中的 enriched 最新日期。""" @@ -1162,14 +1213,19 @@ class KlineRepository: return self.get_etf_daily(symbol, start, end, columns) return pl.DataFrame() + def _minute_glob_for(self, asset_type: str) -> str: + """按资产类型选择分钟K parquet glob。ETF 分钟数据独立存储于 kline_etf_minute。""" + return self._etf_minute_glob if asset_type == "etf" else self._minute_glob + def get_minute( self, symbol: str, trade_date: date, + asset_type: str = "stock", ) -> pl.DataFrame: """分钟K查询 — Polars scan_parquet + predicate pushdown。""" try: - return pl.scan_parquet(self._minute_glob).filter( + return pl.scan_parquet(self._minute_glob_for(asset_type)).filter( (pl.col("symbol") == symbol) & (pl.col("datetime").dt.date() == trade_date) ).sort("datetime").collect() @@ -1181,6 +1237,7 @@ class KlineRepository: self, symbols: list[str], trade_date: date, + asset_type: str = "stock", ) -> pl.DataFrame: """批量分钟K查询 — 多 symbol 一次 scan_parquet。 @@ -1190,7 +1247,7 @@ class KlineRepository: if not symbols: return pl.DataFrame() try: - return pl.scan_parquet(self._minute_glob).filter( + return pl.scan_parquet(self._minute_glob_for(asset_type)).filter( pl.col("symbol").is_in(symbols) & (pl.col("datetime").dt.date() == trade_date) ).sort(["symbol", "datetime"]).collect() @@ -1353,11 +1410,12 @@ class KlineRepository: # DuckDB 查询 (冷路径: 统计/元数据/自定义SQL) # ================================================================ - def latest_minute_date(self, symbol: str) -> date | None: + def latest_minute_date(self, symbol: str, asset_type: str = "stock") -> date | None: + table = "kline_etf_minute" if asset_type == "etf" else "kline_minute" try: with self._lock: row = self.db.execute( - "SELECT max(CAST(datetime AS DATE)) FROM kline_minute WHERE symbol = ?", + f"SELECT max(CAST(datetime AS DATE)) FROM {table} WHERE symbol = ?", [symbol], ).fetchone() if row and row[0]: diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index b6bceee..00a2565 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1097,9 +1097,9 @@ export const api = { method: 'POST', body: JSON.stringify({ symbols, date }), }), - instrumentSearch: (q: string, limit = 20) => - request<{ results: { symbol: string; name: string; code: string }[] }>( - `/api/kline/instruments/search?q=${encodeURIComponent(q)}&limit=${limit}`, + instrumentSearch: (q: string, limit = 20, assetTypes?: string) => + request<{ results: { symbol: string; name: string; code: string; asset_type?: string }[] }>( + `/api/kline/instruments/search?q=${encodeURIComponent(q)}&limit=${limit}${assetTypes ? `&asset_types=${encodeURIComponent(assetTypes)}` : ''}`, ), /** 批量查股票名称 (传入 symbol 列表, 返回 {symbol: name}) */ diff --git a/frontend/src/lib/queryKeys.ts b/frontend/src/lib/queryKeys.ts index 6766ef6..ab7788c 100644 --- a/frontend/src/lib/queryKeys.ts +++ b/frontend/src/lib/queryKeys.ts @@ -29,7 +29,7 @@ export const QK = { // 不用 watchlist- 前缀: 避免被 SSE quotes_updated 高频失效(expert 1s/pro 2s) // 导致每次都拉 TickFlow 触限流。分时图用固定 refetchInterval 刷新即可。 minuteBatch: (symbols: string) => ['minute-batch', symbols] as const, - instrumentSearch: (q: string) => ['instrument-search', q] as const, + instrumentSearch: (q: string, assetTypes?: string) => ['instrument-search', q, assetTypes ?? 'stock'] as const, // Screener screener: ['screener'] as const, diff --git a/frontend/src/pages/Watchlist.tsx b/frontend/src/pages/Watchlist.tsx index 52c3f02..b432601 100644 --- a/frontend/src/pages/Watchlist.tsx +++ b/frontend/src/pages/Watchlist.tsx @@ -191,8 +191,8 @@ function StockSearchBox({ const [activeIdx, setActiveIdx] = useState(-1) const search = useQuery({ - queryKey: QK.instrumentSearch(query), - queryFn: () => api.instrumentSearch(query), + queryKey: QK.instrumentSearch(query, 'stock,etf'), + queryFn: () => api.instrumentSearch(query, 20, 'stock,etf'), enabled: query.trim().length > 0, staleTime: 30_000, }) @@ -273,6 +273,9 @@ function StockSearchBox({ > {r.symbol} {r.name} + {r.asset_type === 'etf' && ( + ETF + )}