mirror of
https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git
synced 2026-09-12 14:24:15 +08:00
feat: 数据源插件化架构 + stock-sdk 首个插件
将可选数据源改为插件化架构: 插件代码放 backend/app/plugins/<name>/, 用户在设置页点击安装依赖, 主仓库不背运行时依赖(nodejs 等)。 架构: - loader.py 新增 _load_builtin_plugins() 扫描 plugins/ 目录, 通过 plugin.yaml 清单动态发现并注册插件(委托自检 + 优雅降级) - runtime 字段支持 node/python, 安装/卸载分别用 npm/pip - 现有 tickflow + YAML 自定义源逻辑 100% 保留, 插件是叠加层 stock-sdk 插件 (plugins/stocksdk/): - 原始实现来自 @forrany 的 PR #57, 迁移到插件化架构, 署名保留 - Node 桥接 (bridge.py/mjs) + Python provider, 覆盖日K/除权/分钟/实时 - 设置页主列表直接显示: 未装灰显+安装按钮, 装完可切换/卸载 配套: - instrument_sync: 通用增强, 任何 provider 都能提供标的维表 - install/uninstall: uv 优先回退 pip, 容错坏 uv.toml + 国内镜像 - 10 例单测全过, tsc 零错误
This commit is contained in:
@@ -413,10 +413,11 @@ def get_preferences() -> dict:
|
||||
|
||||
@router.get("/data-sources")
|
||||
def list_data_sources() -> dict:
|
||||
"""列出已加载的自定义数据源。"""
|
||||
"""列出已加载的数据源 (内置 / 插件 / 用户自定义)。"""
|
||||
from app.data_providers import custom as custom_sources
|
||||
return {
|
||||
"builtin": [{"name": "tickflow", "display_name": "TickFlow", "datasets": ["daily", "adj_factor", "realtime"]}],
|
||||
"builtin": [{"name": "tickflow", "display_name": "TickFlow", "datasets": ["daily", "adj_factor", "realtime", "minute"]}],
|
||||
"plugins": custom_sources.list_plugins(),
|
||||
"custom": custom_sources.list_sources(),
|
||||
"errors": custom_sources.errors(),
|
||||
"config_dir": str(custom_sources.data_sources_dir()),
|
||||
@@ -431,6 +432,52 @@ def reload_data_sources() -> dict:
|
||||
return list_data_sources()
|
||||
|
||||
|
||||
@router.post("/plugins/{name}/install")
|
||||
def install_plugin(name: str) -> dict:
|
||||
"""安装指定插件的依赖 (npm install / pip install), 完成后重新扫描。
|
||||
|
||||
根据 plugin.yaml 的 runtime 字段决定安装方式。安装可能耗时较长 (网络下载),
|
||||
客户端需设较长超时。
|
||||
"""
|
||||
from app.data_providers import custom as custom_sources
|
||||
if not custom_sources.is_builtin(name):
|
||||
raise HTTPException(status_code=404, detail=f"插件 '{name}' 不存在")
|
||||
ok, message = custom_sources.install_plugin(name)
|
||||
# 无论成功失败都重新扫描, 刷新插件状态 (安装可能部分成功)
|
||||
custom_sources.load_all()
|
||||
result = list_data_sources()
|
||||
result["install_ok"] = ok
|
||||
result["install_message"] = message
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/plugins/{name}/install")
|
||||
def uninstall_plugin(name: str) -> dict:
|
||||
"""卸载指定插件的依赖 (删除 node_modules / pip uninstall), 完成后重新扫描。
|
||||
|
||||
如果该插件当前正被使用, 自动回退到 tickflow。
|
||||
"""
|
||||
from app.data_providers import custom as custom_sources
|
||||
from app.services import preferences
|
||||
if not custom_sources.is_builtin(name):
|
||||
raise HTTPException(status_code=404, detail=f"插件 '{name}' 不存在")
|
||||
ok, message = custom_sources.uninstall_plugin(name)
|
||||
# 卸载后若该插件正被使用, 回退 tickflow
|
||||
for getter, key, default in [
|
||||
(preferences.get_daily_data_provider, "daily_data_provider", "tickflow"),
|
||||
(preferences.get_minute_data_provider, "minute_data_provider", "tickflow"),
|
||||
(preferences.get_realtime_data_provider, "realtime_data_provider", "tickflow"),
|
||||
(preferences.get_financial_provider, "financial_data_provider", "tickflow"),
|
||||
]:
|
||||
if getter() == name:
|
||||
preferences.save({key: default})
|
||||
custom_sources.load_all()
|
||||
result = list_data_sources()
|
||||
result["uninstall_ok"] = ok
|
||||
result["uninstall_message"] = message
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/data-sources/{name}")
|
||||
def get_data_source(name: str) -> dict:
|
||||
"""读取一个自定义数据源的完整配置(用于前端编辑回填)。"""
|
||||
|
||||
@@ -5,12 +5,16 @@ from app.data_providers.custom.loader import (
|
||||
errors,
|
||||
get_config_dict,
|
||||
get_provider,
|
||||
install_plugin,
|
||||
is_builtin,
|
||||
is_custom_provider,
|
||||
list_plugins,
|
||||
list_sources,
|
||||
load_all,
|
||||
names,
|
||||
provider_has_dataset,
|
||||
save_config,
|
||||
uninstall_plugin,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@@ -19,10 +23,14 @@ __all__ = [
|
||||
"errors",
|
||||
"get_config_dict",
|
||||
"get_provider",
|
||||
"install_plugin",
|
||||
"is_builtin",
|
||||
"is_custom_provider",
|
||||
"list_plugins",
|
||||
"list_sources",
|
||||
"load_all",
|
||||
"names",
|
||||
"provider_has_dataset",
|
||||
"save_config",
|
||||
"uninstall_plugin",
|
||||
]
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
"""Load custom data source definitions from user data files."""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
@@ -16,9 +19,18 @@ logger = logging.getLogger(__name__)
|
||||
_PROVIDERS: dict[str, GenericHTTPProvider] = {}
|
||||
_LOAD_ERRORS: list[dict] = []
|
||||
|
||||
# 内置插件状态: {name: {available, status, runtime, ...}} 供设置页独立分类展示。
|
||||
# available=False 的插件不注册进 _PROVIDERS (不可切换), 但记录状态供 UI 显示安装提示。
|
||||
_PLUGIN_STATUS: dict[str, dict] = {}
|
||||
|
||||
_NAME_RE = re.compile(r"^[a-z0-9_]+$")
|
||||
|
||||
|
||||
def plugins_dir() -> Path:
|
||||
"""内置可选插件目录 (app/plugins/, 与现有包结构一致, 开发态/容器态路径统一)。"""
|
||||
return Path(__file__).resolve().parents[2] / "plugins"
|
||||
|
||||
|
||||
def data_sources_dir() -> Path:
|
||||
return settings.data_dir / "data_sources"
|
||||
|
||||
@@ -47,8 +59,12 @@ def load_all(path: Path | None = None) -> None:
|
||||
logger.warning("custom data source load failed %s: %s", file, e)
|
||||
_LOAD_ERRORS.append({"path": str(file), "errors": [str(e)]})
|
||||
|
||||
# 内置可选插件 (plugins/ 目录)。与用户 YAML 源独立, 缺依赖只记状态不报错。
|
||||
_load_builtin_plugins()
|
||||
|
||||
|
||||
def list_sources() -> list[dict]:
|
||||
"""只列出用户自定义 (YAML) 源。内置插件 (builtin=True) 由 list_plugins 独立呈现。"""
|
||||
return [
|
||||
{
|
||||
"name": provider.name,
|
||||
@@ -57,9 +73,164 @@ def list_sources() -> list[dict]:
|
||||
"path": str(provider.config.path) if provider.config.path else None,
|
||||
}
|
||||
for provider in _PROVIDERS.values()
|
||||
if not getattr(provider, "builtin", False)
|
||||
]
|
||||
|
||||
|
||||
def list_plugins() -> list[dict]:
|
||||
"""返回所有内置插件的状态 (含已装/未装), 供设置页独立分类显示。"""
|
||||
return list(_PLUGIN_STATUS.values())
|
||||
|
||||
|
||||
def plugin_manifest(name: str) -> dict | None:
|
||||
"""读取指定插件的 plugin.yaml 清单。"""
|
||||
plugin_dir = plugins_dir() / (name or "")
|
||||
manifest_path = plugin_dir / "plugin.yaml"
|
||||
if not manifest_path.exists():
|
||||
return None
|
||||
return yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
|
||||
|
||||
|
||||
def plugin_dir_of(name: str) -> Path:
|
||||
"""返回插件目录路径。"""
|
||||
return plugins_dir() / (name or "")
|
||||
|
||||
|
||||
def install_plugin(name: str) -> tuple[bool, str]:
|
||||
"""安装指定插件的依赖。根据 runtime 执行 npm install / pip install。
|
||||
|
||||
返回 (是否成功, 消息)。成功后调用方应 reload 重新扫描。
|
||||
依赖未找到 (npm/pip 缺失) 或命令失败时返回 False。
|
||||
"""
|
||||
manifest = plugin_manifest(name)
|
||||
if manifest is None:
|
||||
return False, f"插件 '{name}' 不存在或无 plugin.yaml"
|
||||
runtime = str(manifest.get("runtime", "none")).lower()
|
||||
pdir = plugin_dir_of(name)
|
||||
if not pdir.exists():
|
||||
return False, f"插件目录不存在: {pdir}"
|
||||
|
||||
try:
|
||||
if runtime == "node":
|
||||
npm = shutil.which("npm")
|
||||
if not npm:
|
||||
return False, "未找到 npm, 请先安装 Node.js (>=18)"
|
||||
# 在插件目录执行 npm install
|
||||
result = subprocess.run(
|
||||
[npm, "install", "--omit=dev", "--no-audit", "--no-fund"],
|
||||
cwd=str(pdir),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300,
|
||||
)
|
||||
elif runtime == "python":
|
||||
# Python 型插件: 优先用 uv pip install (uv 管理的 venv 无 pip 模块),
|
||||
# 回退 python -m pip。都装进当前后端虚拟环境。
|
||||
# uv 容错: 用户全局 uv.toml 配置错误时 exit 2, 回退 --no-config 重试。
|
||||
# UV_HTTP_TIMEOUT=300: akshare 等含大包(如 mini-racer 14MB), 默认 30s 不够。
|
||||
req = pdir / "requirements.txt"
|
||||
if not req.exists():
|
||||
return False, "Python 型插件需要 requirements.txt"
|
||||
uv_bin = shutil.which("uv")
|
||||
if uv_bin:
|
||||
result = subprocess.run(
|
||||
[uv_bin, "pip", "install", "-r", str(req)],
|
||||
capture_output=True, text=True, timeout=300,
|
||||
env={**__import__("os").environ, "UV_HTTP_TIMEOUT": "300"},
|
||||
)
|
||||
# exit 2 通常是配置文件解析错误, 绕过配置重试
|
||||
# --no-config 会丢镜像, 显式传国内镜像加速 (与用户 uv.toml 意图一致)
|
||||
if result.returncode == 2:
|
||||
result = subprocess.run(
|
||||
[uv_bin, "pip", "install", "--no-config",
|
||||
"--index-url", "https://pypi.tuna.tsinghua.edu.cn/simple",
|
||||
"-r", str(req)],
|
||||
capture_output=True, text=True, timeout=300,
|
||||
env={**__import__("os").environ, "UV_HTTP_TIMEOUT": "300"},
|
||||
)
|
||||
else:
|
||||
import sys
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "-r", str(req)],
|
||||
capture_output=True, text=True, timeout=300,
|
||||
)
|
||||
else:
|
||||
return False, f"runtime={runtime} 无需安装依赖"
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "安装超时 (5分钟), 请检查网络后重试"
|
||||
except Exception as e: # noqa: BLE001
|
||||
return False, f"安装失败: {e}"
|
||||
|
||||
if result.returncode != 0:
|
||||
# 取 stderr 的第一个 error: 行(真正的错误原因), 避免把 uv 的长字段列表返回给用户
|
||||
raw = (result.stderr or result.stdout or "").strip()
|
||||
first_err = ""
|
||||
for line in raw.splitlines():
|
||||
if line.strip().startswith(("error", "Error", "Caused by")):
|
||||
first_err = line.strip()
|
||||
break
|
||||
msg = first_err or raw[-200:]
|
||||
return False, f"安装失败 (exit {result.returncode}): {msg}"
|
||||
return True, "安装成功"
|
||||
|
||||
|
||||
def uninstall_plugin(name: str) -> tuple[bool, str]:
|
||||
"""卸载指定插件的依赖。
|
||||
|
||||
node 型: 删除 node_modules 目录 (干净彻底, 下次需要重新 npm install)。
|
||||
python 型: pip uninstall (包名从 requirements.txt 推断)。
|
||||
"""
|
||||
import shutil as _shutil
|
||||
|
||||
manifest = plugin_manifest(name)
|
||||
if manifest is None:
|
||||
return False, f"插件 '{name}' 不存在或无 plugin.yaml"
|
||||
runtime = str(manifest.get("runtime", "none")).lower()
|
||||
pdir = plugin_dir_of(name)
|
||||
if not pdir.exists():
|
||||
return False, f"插件目录不存在: {pdir}"
|
||||
|
||||
if runtime == "node":
|
||||
nm = pdir / "node_modules"
|
||||
if not nm.exists():
|
||||
return True, "node_modules 不存在, 无需卸载"
|
||||
try:
|
||||
_shutil.rmtree(nm)
|
||||
return True, "已删除 node_modules"
|
||||
except Exception as e: # noqa: BLE001
|
||||
return False, f"删除 node_modules 失败: {e}"
|
||||
|
||||
if runtime == "python":
|
||||
req = pdir / "requirements.txt"
|
||||
if not req.exists():
|
||||
return False, "Python 型插件缺少 requirements.txt, 无法自动卸载"
|
||||
# 读 requirements.txt 拿包名, 逐个 pip uninstall -y
|
||||
pkgs = [l.strip().split("==")[0].split(">=")[0].strip()
|
||||
for l in req.read_text().splitlines()
|
||||
if l.strip() and not l.startswith("#")]
|
||||
if not pkgs:
|
||||
return True, "requirements.txt 无有效包名"
|
||||
uv_bin = _shutil.which("uv")
|
||||
cmd = [uv_bin, "pip", "uninstall", *pkgs] if uv_bin else None
|
||||
if cmd is None:
|
||||
import sys
|
||||
cmd = [sys.executable, "-m", "pip", "uninstall", "-y", *pkgs]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
if result.returncode != 0:
|
||||
return False, f"卸载失败: {(result.stderr or '').strip()[-300:]}"
|
||||
return True, f"已卸载 {len(pkgs)} 个包"
|
||||
except Exception as e: # noqa: BLE001
|
||||
return False, f"卸载失败: {e}"
|
||||
|
||||
return False, f"runtime={runtime} 无需卸载"
|
||||
|
||||
|
||||
def is_builtin(name: str) -> bool:
|
||||
"""判断 name 是否为内置插件 (不可被用户编辑/删除)。"""
|
||||
return (name or "").lower() in _PLUGIN_STATUS
|
||||
|
||||
|
||||
def names() -> set[str]:
|
||||
return set(_PROVIDERS)
|
||||
|
||||
@@ -91,7 +262,9 @@ def provider_has_dataset(name: str, dataset: str) -> bool:
|
||||
|
||||
|
||||
def get_config_dict(name: str) -> dict | None:
|
||||
"""读取一个已加载 custom 源的原始配置 dict(用于前端编辑回填)。"""
|
||||
"""读取一个已加载 custom 源的原始配置 dict(用于前端编辑回填)。内置插件不可编辑。"""
|
||||
if is_builtin(name):
|
||||
return None
|
||||
provider = _PROVIDERS.get((name or "").lower())
|
||||
if provider is None:
|
||||
return None
|
||||
@@ -129,6 +302,8 @@ def _config_to_dict(config: CustomSourceConfig) -> dict:
|
||||
|
||||
def save_config(name: str, config: dict) -> Path:
|
||||
"""把一份配置 dict 写成 data/data_sources/{name}.yaml, 返回写入路径。"""
|
||||
if is_builtin(name):
|
||||
raise ValueError(f"'{name}' 是内置插件, 不可编辑")
|
||||
if not _NAME_RE.match(name or ""):
|
||||
raise ValueError(f"invalid data source name: {name!r} (only lowercase a-z 0-9 _ allowed)")
|
||||
base = data_sources_dir()
|
||||
@@ -143,6 +318,8 @@ def save_config(name: str, config: dict) -> Path:
|
||||
|
||||
def delete_config(name: str) -> bool:
|
||||
"""删除 data/data_sources/{name}.yaml。返回是否真的删除了。"""
|
||||
if is_builtin(name):
|
||||
raise ValueError(f"'{name}' 是内置插件, 不可删除")
|
||||
if not _NAME_RE.match(name or ""):
|
||||
raise ValueError(f"invalid data source name: {name!r}")
|
||||
base = data_sources_dir().resolve()
|
||||
@@ -226,3 +403,97 @@ def _sanitize_dataset(ds_cfg: dict) -> dict:
|
||||
out["end_param"] = str(ds_cfg["end_param"])
|
||||
return out
|
||||
|
||||
|
||||
# ================================================================
|
||||
# 内置可选插件 (plugins/ 目录) 的发现与注册
|
||||
# ================================================================
|
||||
|
||||
def _load_builtin_plugins() -> None:
|
||||
"""扫描 plugins/ 目录下每个含 plugin.yaml 的子目录, 动态加载。
|
||||
|
||||
缺依赖时记录 "不可用" 状态, 不抛异常, 不影响主流程。
|
||||
每次调用重建 _PLUGIN_STATUS, 并把可用的插件注册进 _PROVIDERS。
|
||||
"""
|
||||
global _PLUGIN_STATUS
|
||||
_PLUGIN_STATUS = {}
|
||||
pdir = plugins_dir()
|
||||
if not pdir.exists():
|
||||
return
|
||||
for plugin_dir in sorted(pdir.iterdir()):
|
||||
if not plugin_dir.is_dir():
|
||||
continue
|
||||
manifest_path = plugin_dir / "plugin.yaml"
|
||||
if not manifest_path.exists():
|
||||
continue
|
||||
try:
|
||||
manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
|
||||
_register_one_plugin(manifest)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("插件 %s 清单解析失败: %s", plugin_dir.name, e)
|
||||
|
||||
|
||||
def _register_one_plugin(manifest: dict) -> None:
|
||||
"""注册单个插件: 委托自检 → 可用则动态 import entry 注册进 _PROVIDERS。"""
|
||||
name = manifest.get("name")
|
||||
if not name or not _NAME_RE.match(name):
|
||||
logger.warning("插件清单缺少合法 name: %r", name)
|
||||
return
|
||||
runtime = str(manifest.get("runtime", "none")).lower()
|
||||
# 委托检测: 调用插件自己的 check 函数 (node 型/python 型各自实现)
|
||||
available, reason = _call_check(manifest.get("check"))
|
||||
_PLUGIN_STATUS[name] = {
|
||||
"name": name,
|
||||
"display_name": manifest.get("display_name", name),
|
||||
"datasets": list(manifest.get("datasets", []) or []),
|
||||
"runtime": runtime,
|
||||
"available": available,
|
||||
"status": reason,
|
||||
"description": manifest.get("description", ""),
|
||||
"install_hint": manifest.get("install_hint", ""),
|
||||
}
|
||||
if not available:
|
||||
return # 依赖没装: 不注册, 但状态已记录供 UI 显示
|
||||
# 可用 → 动态加载 provider 类并实例化
|
||||
try:
|
||||
provider_cls = _load_entry(manifest["entry"])
|
||||
provider = provider_cls() if isinstance(provider_cls, type) else provider_cls
|
||||
provider.builtin = True # 标记为内置 (list_sources 过滤, 不可被用户编辑/删除)
|
||||
_PROVIDERS[name] = provider
|
||||
logger.info("内置插件 %s 已注册 (runtime=%s)", name, runtime)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# 声称可用但 import 失败 → 标记不可用, 避免启动崩溃
|
||||
_PLUGIN_STATUS[name]["available"] = False
|
||||
_PLUGIN_STATUS[name]["status"] = f"加载失败: {e}"
|
||||
logger.warning("插件 %s provider 加载失败: %s", name, e)
|
||||
|
||||
|
||||
def _call_check(check_ref: str | None) -> tuple[bool, str]:
|
||||
"""调用插件清单里指定的可用性检测函数, 返回 (是否可用, 原因)。
|
||||
|
||||
check_ref 格式 "module.path:func_name"。无 check 字段时视为可用。
|
||||
"""
|
||||
if not check_ref:
|
||||
return True, "ok"
|
||||
try:
|
||||
func = _load_entry(check_ref)
|
||||
result = func()
|
||||
# 兼容两种返回: (bool, str) 或 bool
|
||||
if isinstance(result, tuple):
|
||||
return bool(result[0]), str(result[1])
|
||||
return bool(result), "ok" if result else "不可用"
|
||||
except Exception as e: # noqa: BLE001
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def _load_entry(entry_ref: str):
|
||||
"""动态加载 'module.path:attr' 形式的引用, 返回属性对象 (类或函数)。"""
|
||||
if ":" not in entry_ref:
|
||||
raise ValueError(f"entry 格式应为 'module.path:attr', 得到: {entry_ref!r}")
|
||||
module_path, attr = entry_ref.split(":", 1)
|
||||
module = importlib.import_module(module_path)
|
||||
return getattr(module, attr)
|
||||
|
||||
|
||||
# 模块导入时即扫描一次, 保证 names()/_allowed_data_providers() 在 startup 前可用。
|
||||
_load_builtin_plugins()
|
||||
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""stock-sdk 内置数据源插件(免费行情, tickflow 的平替)。"""
|
||||
from app.plugins.stocksdk.bridge import StockSDKBridgeError, availability, run_job
|
||||
from app.plugins.stocksdk.provider import StockSDKProvider
|
||||
|
||||
PROVIDER_NAME = "stocksdk"
|
||||
|
||||
__all__ = [
|
||||
"PROVIDER_NAME",
|
||||
"StockSDKBridgeError",
|
||||
"StockSDKProvider",
|
||||
"availability",
|
||||
"run_job",
|
||||
]
|
||||
@@ -0,0 +1,292 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* stock-sdk 桥接脚本。Python 后端通过 subprocess 调用它,用真实 stock-sdk 抓数据。
|
||||
*
|
||||
* Original implementation by @forrany (PR #57), migrated to plugin architecture.
|
||||
*
|
||||
* 协议:
|
||||
* - stdin: 单行 JSON { op, symbols?, adjust?, period?, start?, end?, concurrency? }
|
||||
* - stdout: 单行 JSON
|
||||
* daily/adj/minute: { ok:true, op, rows: { [appSymbol]: Row[] } }
|
||||
* realtime/instruments: { ok:true, op, rows: Row[] }
|
||||
* ping: { ok:true, op:'ping', version }
|
||||
* 失败: { ok:false, error }
|
||||
*
|
||||
* op:
|
||||
* daily —— 日K(adjust 默认 none), 每个 symbol 一组 bars
|
||||
* adj —— 除权因子: 取 hfq 与 none 收盘价, ex_factor = close_hfq / close_none
|
||||
* minute —— 分钟K(period 默认 5)
|
||||
* realtime —— 全 A 股实时快照(batch.cn)
|
||||
* instruments —— 全 A 股标的维表(batch.cn 提取元数据)
|
||||
* ping —— 探活
|
||||
*
|
||||
* 说明: daily/adj/minute 的入参 symbols 是「app 符号」(如 600519.SH)。stock-sdk 能容错解析,
|
||||
* 返回结果里我们**回显原始 app 符号**作为 key,避免 code→符号 的歧义(指数/股票同码等)。
|
||||
* realtime/instruments 是全市场枚举,由 code + marketId 反推后缀。
|
||||
*/
|
||||
import { createRequire } from 'node:module'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { execSync } from 'node:child_process'
|
||||
import path from 'node:path'
|
||||
|
||||
/**
|
||||
* 解析 stock-sdk 入口。ESM 的 bare import 只查本地 node_modules 链,不查全局,
|
||||
* 因此这里显式在 [本地(脚本旁), 全局 npm root, NODE_PATH] 中查找后动态 import。
|
||||
* 部署时优先用脚本旁 vendored 的 node_modules/stock-sdk。
|
||||
*/
|
||||
async function loadSDK() {
|
||||
const require = createRequire(import.meta.url)
|
||||
const scriptDir = path.dirname(new URL(import.meta.url).pathname)
|
||||
// 候选 node_modules 目录(按优先级)
|
||||
const nmDirs = [path.join(scriptDir, 'node_modules')]
|
||||
for (const p of (process.env.NODE_PATH || '').split(path.delimiter).filter(Boolean)) nmDirs.push(p)
|
||||
try {
|
||||
const groot = execSync('npm root -g', { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()
|
||||
if (groot) nmDirs.push(groot)
|
||||
} catch {
|
||||
/* npm 不可用则忽略 */
|
||||
}
|
||||
let entry
|
||||
for (const nm of nmDirs) {
|
||||
try {
|
||||
entry = require.resolve(path.join(nm, 'stock-sdk'))
|
||||
break
|
||||
} catch {
|
||||
/* 试下一个 */
|
||||
}
|
||||
}
|
||||
if (!entry) {
|
||||
throw new Error(
|
||||
`无法解析 stock-sdk(已搜索: ${nmDirs.join(', ')})。请在桥接目录 npm install,或全局 npm i -g stock-sdk。`
|
||||
)
|
||||
}
|
||||
const mod = await import(pathToFileURL(entry).href)
|
||||
return mod.StockSDK || (mod.default && mod.default.StockSDK)
|
||||
}
|
||||
|
||||
const MARKET_ID_TO_SUFFIX = { '1': 'SH', '51': 'SZ', '62': 'BJ' }
|
||||
|
||||
// 下游(Python)可能提前关闭管道,忽略 EPIPE 避免噪声崩溃。
|
||||
process.stdout.on('error', (e) => {
|
||||
if (e && e.code === 'EPIPE') process.exit(0)
|
||||
})
|
||||
|
||||
function readStdin() {
|
||||
return new Promise((resolve, reject) => {
|
||||
let buf = ''
|
||||
process.stdin.setEncoding('utf8')
|
||||
process.stdin.on('data', (c) => (buf += c))
|
||||
process.stdin.on('end', () => resolve(buf))
|
||||
process.stdin.on('error', reject)
|
||||
})
|
||||
}
|
||||
|
||||
/** 简单并发池: 对 items 逐个跑 worker,最多 concurrency 个在飞。 */
|
||||
async function mapPool(items, concurrency, worker) {
|
||||
const results = new Array(items.length)
|
||||
let next = 0
|
||||
const runners = new Array(Math.min(concurrency, items.length)).fill(0).map(async () => {
|
||||
while (true) {
|
||||
const i = next++
|
||||
if (i >= items.length) return
|
||||
try {
|
||||
results[i] = await worker(items[i], i)
|
||||
} catch (e) {
|
||||
results[i] = { __error: String((e && e.message) || e) }
|
||||
}
|
||||
}
|
||||
})
|
||||
await runners.reduce((p) => p, Promise.resolve())
|
||||
await Promise.all(runners)
|
||||
return results
|
||||
}
|
||||
|
||||
/** code + marketId → app 符号(600519.SH)。反推失败则退化用 code 前缀猜测。 */
|
||||
function toAppSymbol(code, marketId) {
|
||||
const suffix = MARKET_ID_TO_SUFFIX[String(marketId)] || guessSuffix(code)
|
||||
return suffix ? `${code}.${suffix}` : String(code)
|
||||
}
|
||||
|
||||
function guessSuffix(code) {
|
||||
const c = String(code)
|
||||
if (/^(6|5|9)/.test(c)) return 'SH'
|
||||
if (/^(0|3|1|2)/.test(c)) return 'SZ'
|
||||
if (/^(4|8|92)/.test(c)) return 'BJ'
|
||||
return ''
|
||||
}
|
||||
|
||||
/** stock-sdk 的 adjust 取值是 '' | 'qfq' | 'hfq'(无 'none')。这里做兼容映射。 */
|
||||
function normAdjust(v) {
|
||||
if (v === 'hfq') return 'hfq'
|
||||
if (v === 'qfq') return 'qfq'
|
||||
return '' // none / undefined / 空 → 不复权
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
||||
|
||||
/**
|
||||
* 上游(东财)偶发冷启动/限流时对活跃标的返回空数组(非报错)。对已知应有数据的请求,
|
||||
* 空结果重试若干次以提升鲁棒性;真正无数据(退市/停牌/区间无交易)时多花几次调用可接受。
|
||||
*/
|
||||
async function fetchWithRetry(fn, { retries = 2, delayMs = 300 } = {}) {
|
||||
let last = []
|
||||
for (let i = 0; i <= retries; i++) {
|
||||
const r = await fn()
|
||||
if (Array.isArray(r) && r.length > 0) return r
|
||||
last = Array.isArray(r) ? r : []
|
||||
if (i < retries) await sleep(delayMs)
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
async function fetchDaily(sdk, sym, { adjust, period = 'daily', start, end }) {
|
||||
const opts = { period, adjust: normAdjust(adjust) }
|
||||
if (start) opts.startDate = start
|
||||
if (end) opts.endDate = end
|
||||
return fetchWithRetry(() => sdk.kline.cn(sym, opts))
|
||||
}
|
||||
|
||||
async function opDaily(sdk, job) {
|
||||
const { symbols = [], adjust = 'none', period = 'daily', start, end, concurrency = 6 } = job
|
||||
const out = {}
|
||||
const rows = await mapPool(symbols, concurrency, (sym) =>
|
||||
fetchDaily(sdk, sym, { adjust, period, start, end })
|
||||
)
|
||||
symbols.forEach((sym, i) => {
|
||||
const r = rows[i]
|
||||
out[sym] = Array.isArray(r) ? r : []
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
async function opAdj(sdk, job) {
|
||||
const { symbols = [], start, end, concurrency = 6 } = job
|
||||
const out = {}
|
||||
await mapPool(symbols, concurrency, async (sym) => {
|
||||
const [none, hfq] = await Promise.all([
|
||||
fetchDaily(sdk, sym, { adjust: 'none', start, end }),
|
||||
fetchDaily(sdk, sym, { adjust: 'hfq', start, end }),
|
||||
])
|
||||
const noneByDate = new Map()
|
||||
for (const b of none) if (b && b.close) noneByDate.set(b.date, b.close)
|
||||
const factors = []
|
||||
for (const b of hfq) {
|
||||
if (!b || !b.date) continue
|
||||
const rawClose = noneByDate.get(b.date)
|
||||
if (!rawClose || !b.close) continue
|
||||
factors.push({ symbol: sym, trade_date: b.date, ex_factor: b.close / rawClose })
|
||||
}
|
||||
out[sym] = factors
|
||||
return factors
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
async function opMinute(sdk, job) {
|
||||
const { symbols = [], period = 5, start, end, concurrency = 6 } = job
|
||||
const out = {}
|
||||
await mapPool(symbols, concurrency, async (sym) => {
|
||||
const opts = { period: String(period) }
|
||||
if (start) opts.startDate = start
|
||||
if (end) opts.endDate = end
|
||||
const bars = await fetchWithRetry(() => sdk.kline.cnMinute(sym, opts))
|
||||
out[sym] = Array.isArray(bars) ? bars : []
|
||||
return out[sym]
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
async function opRealtime(sdk, job) {
|
||||
const { concurrency = 8 } = job
|
||||
const all = await sdk.batch.cn({ concurrency })
|
||||
const rows = []
|
||||
for (const q of all || []) {
|
||||
if (!q || !q.code) continue
|
||||
rows.push({
|
||||
symbol: toAppSymbol(q.code, q.marketId),
|
||||
name: q.name,
|
||||
last_price: q.price,
|
||||
prev_close: q.prevClose,
|
||||
open: q.open,
|
||||
high: q.high,
|
||||
low: q.low,
|
||||
volume: q.volume,
|
||||
amount: q.amount,
|
||||
change_pct: q.changePercent,
|
||||
})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
async function opInstruments(sdk, job) {
|
||||
const { concurrency = 8 } = job
|
||||
const all = await sdk.batch.cn({ concurrency })
|
||||
const rows = []
|
||||
for (const q of all || []) {
|
||||
if (!q || !q.code) continue
|
||||
const suffix = MARKET_ID_TO_SUFFIX[String(q.marketId)] || guessSuffix(q.code)
|
||||
// 形状对齐 tickflow 的 Instrument(数值扩展字段放 ext),以复用 instrument_sync 的 flatten。
|
||||
rows.push({
|
||||
symbol: toAppSymbol(q.code, q.marketId),
|
||||
name: q.name,
|
||||
code: String(q.code),
|
||||
exchange: suffix,
|
||||
region: 'CN',
|
||||
type: 'stock',
|
||||
ext: {
|
||||
total_shares: q.totalShares ?? null,
|
||||
float_shares: q.circulatingShares ?? null,
|
||||
limit_up: q.limitUp ?? null,
|
||||
limit_down: q.limitDown ?? null,
|
||||
},
|
||||
})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
async function main() {
|
||||
let job
|
||||
try {
|
||||
const raw = (await readStdin()).trim()
|
||||
job = raw ? JSON.parse(raw) : {}
|
||||
} catch (e) {
|
||||
process.stdout.write(JSON.stringify({ ok: false, error: `invalid job json: ${e.message}` }))
|
||||
return
|
||||
}
|
||||
const op = job.op || 'ping'
|
||||
try {
|
||||
const StockSDK = await loadSDK()
|
||||
if (!StockSDK) throw new Error('stock-sdk 已解析但未导出 StockSDK')
|
||||
if (op === 'ping') {
|
||||
process.stdout.write(JSON.stringify({ ok: true, op: 'ping', version: StockSDK.version || 'ok' }))
|
||||
return
|
||||
}
|
||||
const sdk = new StockSDK({ retry: { maxRetries: 3, baseDelay: 400 } })
|
||||
let rows
|
||||
switch (op) {
|
||||
case 'daily':
|
||||
rows = await opDaily(sdk, job)
|
||||
break
|
||||
case 'adj':
|
||||
rows = await opAdj(sdk, job)
|
||||
break
|
||||
case 'minute':
|
||||
rows = await opMinute(sdk, job)
|
||||
break
|
||||
case 'realtime':
|
||||
rows = await opRealtime(sdk, job)
|
||||
break
|
||||
case 'instruments':
|
||||
rows = await opInstruments(sdk, job)
|
||||
break
|
||||
default:
|
||||
process.stdout.write(JSON.stringify({ ok: false, error: `unknown op: ${op}` }))
|
||||
return
|
||||
}
|
||||
process.stdout.write(JSON.stringify({ ok: true, op, rows }))
|
||||
} catch (e) {
|
||||
process.stdout.write(JSON.stringify({ ok: false, op, error: String((e && e.stack) || e) }))
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Python ↔ Node 桥接: 通过 subprocess 调用 bridge.mjs 使用真实 stock-sdk 抓数据。
|
||||
|
||||
Original implementation by @forrany (PR #57), migrated to plugin architecture.
|
||||
|
||||
后端是 Python, stock-sdk 是 Node/JS 包, 这里用 subprocess 把二者接起来:
|
||||
每次调用 spawn 一个 `node bridge.mjs`, 从 stdin 喂 JSON job, 从 stdout 读 JSON 结果。
|
||||
批内并发由 bridge.mjs 内部承担, 一次进程调用摊薄 node 启动开销。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_HERE = Path(__file__).resolve().parent
|
||||
_BRIDGE_MJS = _HERE / "bridge.mjs"
|
||||
|
||||
# 默认超时(秒)。全市场(realtime/instruments)与大批量 daily 可能较久, provider 侧按 op 调大。
|
||||
DEFAULT_TIMEOUT = 120
|
||||
|
||||
|
||||
class StockSDKBridgeError(RuntimeError):
|
||||
"""桥接调用失败(node 缺失 / stock-sdk 未安装 / 子进程异常 / 结果非法)。"""
|
||||
|
||||
|
||||
def _node_bin() -> str | None:
|
||||
"""定位 node 可执行文件: 优先环境变量 STOCK_SDK_NODE, 否则 PATH 中的 node。"""
|
||||
env = os.getenv("STOCK_SDK_NODE")
|
||||
if env:
|
||||
return env if (Path(env).exists() or shutil.which(env)) else None
|
||||
return shutil.which("node")
|
||||
|
||||
|
||||
def run_job(job: dict, timeout: int = DEFAULT_TIMEOUT) -> dict:
|
||||
"""执行一次桥接 job, 返回解析后的 dict(含 `rows`)。失败抛 StockSDKBridgeError。"""
|
||||
node = _node_bin()
|
||||
if not node:
|
||||
raise StockSDKBridgeError(
|
||||
"未找到 node 可执行文件。请安装 Node.js(>=18)或设置 STOCK_SDK_NODE 指向 node。"
|
||||
)
|
||||
if not _BRIDGE_MJS.exists():
|
||||
raise StockSDKBridgeError(f"桥接脚本缺失: {_BRIDGE_MJS}")
|
||||
|
||||
payload = json.dumps(job, ensure_ascii=False)
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[node, str(_BRIDGE_MJS)],
|
||||
input=payload,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
cwd=str(_HERE),
|
||||
)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
raise StockSDKBridgeError(f"stock-sdk 桥接超时(op={job.get('op')}, {timeout}s)") from e
|
||||
except OSError as e:
|
||||
raise StockSDKBridgeError(f"启动 node 失败: {e}") from e
|
||||
|
||||
if proc.returncode != 0:
|
||||
tail = (proc.stderr or proc.stdout or "").strip()[-800:]
|
||||
raise StockSDKBridgeError(f"stock-sdk 桥接非零退出({proc.returncode}): {tail}")
|
||||
|
||||
out = (proc.stdout or "").strip()
|
||||
if not out:
|
||||
raise StockSDKBridgeError(f"stock-sdk 桥接无输出。stderr: {(proc.stderr or '').strip()[-500:]}")
|
||||
try:
|
||||
result = json.loads(out)
|
||||
except json.JSONDecodeError as e:
|
||||
raise StockSDKBridgeError(f"stock-sdk 桥接输出非法 JSON: {out[:500]}") from e
|
||||
|
||||
if not result.get("ok"):
|
||||
raise StockSDKBridgeError(f"stock-sdk 桥接返回错误: {result.get('error')}")
|
||||
return result
|
||||
|
||||
|
||||
def availability() -> tuple[bool, str]:
|
||||
"""探活: 返回 (是否可用, 原因)。用于 UI 与日志。不抛异常。"""
|
||||
node = _node_bin()
|
||||
if not node:
|
||||
return False, "未找到 node(需 Node.js>=18 或设置 STOCK_SDK_NODE)"
|
||||
try:
|
||||
result = run_job({"op": "ping"}, timeout=20)
|
||||
return True, f"ok (stock-sdk {result.get('version', '?')})"
|
||||
except StockSDKBridgeError as e:
|
||||
return False, str(e)
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"name": "tickflow-stocksdk-bridge",
|
||||
"version": "0.1.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "tickflow-stocksdk-bridge",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"stock-sdk": "^2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/stock-sdk": {
|
||||
"version": "2.2.2",
|
||||
"resolved": "https://mirrors.tencent.com/npm/stock-sdk/-/stock-sdk-2.2.2.tgz",
|
||||
"integrity": "sha512-nYPIrdd9vm28w2dHXLWBgyXF/bnC3eXFaM7vhxeXBDQJs7imaFR3rWBQUBfhPPxMuhrunnnNDdEPLCq8Uv839Q==",
|
||||
"license": "ISC",
|
||||
"bin": {
|
||||
"stock-sdk": "dist/cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "tickflow-stocksdk-bridge",
|
||||
"private": true,
|
||||
"version": "0.1.0",
|
||||
"description": "Node bridge that exposes stock-sdk to the Python backend as a data provider.",
|
||||
"type": "module",
|
||||
"main": "bridge.mjs",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"dependencies": {
|
||||
"stock-sdk": "^2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
# stock-sdk 内置可选数据源插件
|
||||
# 基于 stock-sdk (Node.js) 的免费 A 股行情, 无需 API Key。
|
||||
# 用户需手动安装依赖后才可用: cd backend/app/plugins/stocksdk && npm install
|
||||
|
||||
name: stocksdk
|
||||
display_name: "stock-sdk(免费行情)"
|
||||
runtime: node
|
||||
entry: app.plugins.stocksdk.provider:StockSDKProvider
|
||||
check: app.plugins.stocksdk.bridge:availability
|
||||
datasets: [daily, adj_factor, minute, realtime]
|
||||
description: "基于 stock-sdk 的免费 A 股行情, 无需 API Key。日K/除权/分钟/实时全市场。需运行环境含 Node.js 18+。"
|
||||
install_hint: "cd backend/app/plugins/stocksdk && npm install"
|
||||
@@ -0,0 +1,251 @@
|
||||
"""stock-sdk 内置数据源 provider。
|
||||
|
||||
Original implementation by @forrany (PR #57), migrated to plugin architecture.
|
||||
核心抓取/归一化逻辑保留原作者实现, 仅调整 import 路径与注册方式。
|
||||
|
||||
通过 bridge.mjs 调真实 stock-sdk 抓 A 股行情, 归一化到项目内部 schema。
|
||||
方法签名对齐 custom.GenericHTTPProvider(service 分流点按这套签名调用),
|
||||
因此注入 custom loader 注册表后, 各 service 无需改动即可路由到本 provider。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.data_providers.normalizer import normalize_adj_factors, normalize_daily
|
||||
from app.plugins.stocksdk import bridge
|
||||
from app.tickflow.rate_limits import chunked
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# stock-sdk 支持的数据集(financial 不支持 → 不声明, 自动回退 tickflow)
|
||||
_DATASETS = ("daily", "adj_factor", "minute", "realtime")
|
||||
|
||||
# 每次桥接调用的符号数。桥接内部按 concurrency 并发, 分批仅为进度反馈与超时控制。
|
||||
_BATCH = 40
|
||||
_MINUTE_CANONICAL = ["symbol", "datetime", "open", "high", "low", "close", "volume", "amount"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _StockSDKConfig:
|
||||
"""轻量 config shim, 让 custom loader 的 list_sources/provider_has_dataset 能识别本 provider。"""
|
||||
|
||||
name: str = "stocksdk"
|
||||
display_name: str = "stock-sdk(免费行情)"
|
||||
datasets: dict = field(default_factory=lambda: dict.fromkeys(_DATASETS))
|
||||
path: None = None
|
||||
builtin: bool = True
|
||||
|
||||
|
||||
def _yyyymmdd(dt: datetime | None) -> str | None:
|
||||
return dt.strftime("%Y%m%d") if dt else None
|
||||
|
||||
|
||||
class StockSDKProvider:
|
||||
"""内置 stock-sdk 数据源。"""
|
||||
|
||||
name = "stocksdk"
|
||||
builtin = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.config = _StockSDKConfig()
|
||||
|
||||
def close(self) -> None: # loader.load_all 会对每个 provider 调 close
|
||||
pass
|
||||
|
||||
# ---- daily ----
|
||||
def get_daily(
|
||||
self,
|
||||
symbols: list[str],
|
||||
start_time: datetime | None,
|
||||
end_time: datetime | None,
|
||||
asset_type: str = "stock", # noqa: ARG002
|
||||
on_chunk_done=None,
|
||||
) -> pl.DataFrame:
|
||||
if not symbols:
|
||||
return pl.DataFrame()
|
||||
frames: list[pl.DataFrame] = []
|
||||
chunks = chunked(symbols, _BATCH)
|
||||
for i, chunk in enumerate(chunks):
|
||||
job = {
|
||||
"op": "daily",
|
||||
"symbols": chunk,
|
||||
"adjust": "none",
|
||||
"start": _yyyymmdd(start_time),
|
||||
"end": _yyyymmdd(end_time),
|
||||
}
|
||||
try:
|
||||
result = bridge.run_job(job, timeout=180)
|
||||
except bridge.StockSDKBridgeError as e:
|
||||
logger.warning("stock-sdk daily 拉取失败(%d symbols): %s", len(chunk), e)
|
||||
result = {"rows": {}}
|
||||
for sym, rows in (result.get("rows") or {}).items():
|
||||
if not rows:
|
||||
continue
|
||||
df = normalize_daily(rows, default_symbol=sym, source=self.name)
|
||||
if not df.is_empty():
|
||||
frames.append(df)
|
||||
if on_chunk_done:
|
||||
on_chunk_done(i + 1, len(chunks))
|
||||
return pl.concat(frames, how="diagonal_relaxed") if frames else pl.DataFrame()
|
||||
|
||||
# ---- adj_factor ----
|
||||
def get_adj_factors(
|
||||
self,
|
||||
symbols: list[str],
|
||||
start_time: datetime | None,
|
||||
end_time: datetime | None,
|
||||
asset_type: str = "stock", # noqa: ARG002
|
||||
on_chunk_done=None,
|
||||
) -> pl.DataFrame:
|
||||
if not symbols:
|
||||
return pl.DataFrame()
|
||||
frames: list[pl.DataFrame] = []
|
||||
chunks = chunked(symbols, _BATCH)
|
||||
for i, chunk in enumerate(chunks):
|
||||
job = {
|
||||
"op": "adj",
|
||||
"symbols": chunk,
|
||||
"start": _yyyymmdd(start_time),
|
||||
"end": _yyyymmdd(end_time),
|
||||
}
|
||||
try:
|
||||
result = bridge.run_job(job, timeout=240)
|
||||
except bridge.StockSDKBridgeError as e:
|
||||
logger.warning("stock-sdk adj 拉取失败(%d symbols): %s", len(chunk), e)
|
||||
result = {"rows": {}}
|
||||
flat: list[dict] = []
|
||||
for rows in (result.get("rows") or {}).values():
|
||||
flat.extend(rows or [])
|
||||
if flat:
|
||||
df = normalize_adj_factors(flat, source=self.name)
|
||||
if not df.is_empty():
|
||||
frames.append(df)
|
||||
if on_chunk_done:
|
||||
on_chunk_done(i + 1, len(chunks))
|
||||
return pl.concat(frames, how="diagonal_relaxed") if frames else pl.DataFrame()
|
||||
|
||||
# ---- minute ----
|
||||
def get_minute(
|
||||
self,
|
||||
symbols: list[str],
|
||||
start_time: datetime | None,
|
||||
end_time: datetime | None,
|
||||
asset_type: str = "stock", # noqa: ARG002
|
||||
on_chunk_done=None,
|
||||
freq: str = "5m",
|
||||
) -> pl.DataFrame:
|
||||
if not symbols:
|
||||
return pl.DataFrame()
|
||||
period = "".join(ch for ch in str(freq) if ch.isdigit()) or "5"
|
||||
frames: list[pl.DataFrame] = []
|
||||
chunks = chunked(symbols, _BATCH)
|
||||
for i, chunk in enumerate(chunks):
|
||||
job = {
|
||||
"op": "minute",
|
||||
"symbols": chunk,
|
||||
"period": period,
|
||||
"start": _yyyymmdd(start_time),
|
||||
"end": _yyyymmdd(end_time),
|
||||
}
|
||||
try:
|
||||
result = bridge.run_job(job, timeout=180)
|
||||
except bridge.StockSDKBridgeError as e:
|
||||
logger.warning("stock-sdk minute 拉取失败(%d symbols): %s", len(chunk), e)
|
||||
result = {"rows": {}}
|
||||
for sym, rows in (result.get("rows") or {}).items():
|
||||
df = self._minute_df(rows, sym)
|
||||
if not df.is_empty():
|
||||
frames.append(df)
|
||||
if on_chunk_done:
|
||||
on_chunk_done(i + 1, len(chunks))
|
||||
return pl.concat(frames, how="diagonal_relaxed") if frames else pl.DataFrame()
|
||||
|
||||
@staticmethod
|
||||
def _minute_df(rows: list[dict], symbol: str) -> pl.DataFrame:
|
||||
if not rows:
|
||||
return pl.DataFrame()
|
||||
df = pl.DataFrame(rows)
|
||||
# bridge 分钟行含 timestamp(ms, UTC 基准)。A 股分时按北京时间墙钟展示,
|
||||
# 故转 Asia/Shanghai 后去掉时区得到 naive 北京时间(如 09:35)。
|
||||
if "timestamp" in df.columns:
|
||||
df = df.with_columns(
|
||||
pl.from_epoch(pl.col("timestamp").cast(pl.Int64), time_unit="ms")
|
||||
.dt.replace_time_zone("UTC")
|
||||
.dt.convert_time_zone("Asia/Shanghai")
|
||||
.dt.replace_time_zone(None)
|
||||
.cast(pl.Datetime("us"))
|
||||
.alias("datetime")
|
||||
)
|
||||
elif "date" in df.columns:
|
||||
df = df.with_columns(
|
||||
pl.col("date").str.to_datetime("%Y-%m-%d %H:%M", strict=False).alias("datetime")
|
||||
)
|
||||
df = df.with_columns(pl.lit(symbol).alias("symbol"))
|
||||
for col in ("open", "high", "low", "close", "volume", "amount"):
|
||||
if col in df.columns:
|
||||
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
|
||||
keep = [c for c in _MINUTE_CANONICAL if c in df.columns]
|
||||
return df.select(keep) if "datetime" in keep else pl.DataFrame()
|
||||
|
||||
# ---- realtime (全市场快照) ----
|
||||
def get_realtime(self) -> list[dict]:
|
||||
try:
|
||||
result = bridge.run_job({"op": "realtime"}, timeout=120)
|
||||
except bridge.StockSDKBridgeError as e:
|
||||
logger.warning("stock-sdk realtime 拉取失败: %s", e)
|
||||
return []
|
||||
return result.get("rows") or []
|
||||
|
||||
# ---- instruments (标的维表) ----
|
||||
def get_instruments(self, asset_type: str = "stock") -> list[dict]:
|
||||
"""返回 tickflow Instrument 形状的行(symbol/name/code/exchange/region/type + ext),
|
||||
|
||||
供 instrument_sync._flatten_instruments 复用同一 flatten 路径, 列结构与 tickflow 一致。
|
||||
当前覆盖 A 股股票。
|
||||
"""
|
||||
if asset_type != "stock":
|
||||
return []
|
||||
try:
|
||||
result = bridge.run_job({"op": "instruments"}, timeout=120)
|
||||
except bridge.StockSDKBridgeError as e:
|
||||
logger.warning("stock-sdk instruments 拉取失败: %s", e)
|
||||
return []
|
||||
return result.get("rows") or []
|
||||
|
||||
# ---- 测试(设置页试拉) ----
|
||||
def test_dataset(self, dataset: str, symbols: list[str] | None = None) -> dict:
|
||||
symbols = symbols or ["600519.SH"]
|
||||
if dataset == "daily":
|
||||
df = self.get_daily(symbols, None, None)
|
||||
return _preview("daily", df)
|
||||
if dataset == "adj_factor":
|
||||
df = self.get_adj_factors(symbols, None, None)
|
||||
return _preview("adj_factor", df)
|
||||
if dataset == "minute":
|
||||
df = self.get_minute(symbols, None, None)
|
||||
return _preview("minute", df)
|
||||
if dataset == "realtime":
|
||||
rows = self.get_realtime()
|
||||
head = rows[:5]
|
||||
return {
|
||||
"provider": self.name,
|
||||
"dataset": "realtime",
|
||||
"rows": len(rows),
|
||||
"columns": list(head[0].keys()) if head else [],
|
||||
"preview": head,
|
||||
}
|
||||
raise ValueError(f"stock-sdk 不支持数据集: {dataset}")
|
||||
|
||||
|
||||
def _preview(dataset: str, df: pl.DataFrame) -> dict:
|
||||
return {
|
||||
"provider": "stocksdk",
|
||||
"dataset": dataset,
|
||||
"rows": df.height,
|
||||
"columns": df.columns,
|
||||
"preview": df.head(5).to_dicts() if not df.is_empty() else [],
|
||||
}
|
||||
@@ -43,22 +43,52 @@ def _flatten_instruments(items: list[dict]) -> list[dict]:
|
||||
return rows
|
||||
|
||||
|
||||
def _fetch_instruments_via_provider() -> list[dict] | None:
|
||||
"""若当前日K数据源不是 tickflow 且该 provider 提供 get_instruments, 用它拉标的维表。
|
||||
|
||||
返回 flatten 行列表; 未命中(仍应走 tickflow)时返回 None。
|
||||
标的维表跟随日K数据源(二者天然耦合, 无独立偏好项)。
|
||||
"""
|
||||
from app.services import preferences
|
||||
|
||||
provider_name = preferences.get_daily_data_provider()
|
||||
if provider_name == "tickflow":
|
||||
return None
|
||||
from app.data_providers import custom as custom_sources
|
||||
|
||||
if not custom_sources.is_custom_provider(provider_name):
|
||||
return None
|
||||
provider = custom_sources.get_provider(provider_name)
|
||||
if not hasattr(provider, "get_instruments"):
|
||||
return None
|
||||
try:
|
||||
items = provider.get_instruments("stock") or []
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("provider %s get_instruments 失败: %s", provider_name, e)
|
||||
return None
|
||||
rows = _flatten_instruments(items)
|
||||
logger.info("instruments via %s: %d stocks", provider_name, len(rows))
|
||||
return rows
|
||||
|
||||
|
||||
def sync_instruments(data_dir: Path) -> int:
|
||||
"""全量同步标的维表 → data/instruments/instruments.parquet。
|
||||
|
||||
返回写入的行数。
|
||||
"""
|
||||
tf = get_client()
|
||||
all_rows: list[dict] = []
|
||||
|
||||
for ex in _EXCHANGES:
|
||||
try:
|
||||
items = tf.exchanges.get_instruments(ex, instrument_type="stock")
|
||||
if items:
|
||||
all_rows.extend(_flatten_instruments(items))
|
||||
logger.info("instruments %s: %d stocks", ex, len(items))
|
||||
except Exception as e:
|
||||
logger.warning("get_instruments(%s) failed: %s", ex, e)
|
||||
all_rows = _fetch_instruments_via_provider()
|
||||
if all_rows is None:
|
||||
# 未命中非 tickflow provider → 走 tickflow 直连
|
||||
tf = get_client()
|
||||
all_rows = []
|
||||
for ex in _EXCHANGES:
|
||||
try:
|
||||
items = tf.exchanges.get_instruments(ex, instrument_type="stock")
|
||||
if items:
|
||||
all_rows.extend(_flatten_instruments(items))
|
||||
logger.info("instruments %s: %d stocks", ex, len(items))
|
||||
except Exception as e:
|
||||
logger.warning("get_instruments(%s) failed: %s", ex, e)
|
||||
|
||||
if not all_rows:
|
||||
return 0
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
"""StockSDKProvider 归一化与桥接契约测试。
|
||||
|
||||
不依赖真实 node / 网络: mock bridge.run_job 返回样例 payload, 只验证 Python 侧的
|
||||
归一化、除权因子合成对齐、符号回显、空结果处理与注册接线。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.plugins.stocksdk import provider as sp
|
||||
from app.plugins.stocksdk.provider import StockSDKProvider
|
||||
|
||||
|
||||
def _patch_run_job(monkeypatch, mapping):
|
||||
"""mapping: op -> payload dict(将作为 run_job 返回值)。"""
|
||||
|
||||
def fake(job, timeout=None): # noqa: ARG001
|
||||
return mapping[job["op"]]
|
||||
|
||||
monkeypatch.setattr(sp.bridge, "run_job", fake)
|
||||
|
||||
|
||||
def test_get_daily_normalizes_and_echoes_symbol(monkeypatch):
|
||||
_patch_run_job(monkeypatch, {
|
||||
"daily": {"ok": True, "op": "daily", "rows": {
|
||||
"600519.SH": [
|
||||
{"date": "2026-01-05", "open": 1385.0, "high": 1431.9, "low": 1385.0,
|
||||
"close": 1426.0, "volume": 70949, "amount": 1.0e10, "code": "600519"},
|
||||
{"date": "2026-01-06", "open": 1432.5, "high": 1437.0, "low": 1416.5,
|
||||
"close": 1428.0, "volume": 39586, "amount": 5.6e9, "code": "600519"},
|
||||
],
|
||||
}},
|
||||
})
|
||||
df = StockSDKProvider().get_daily(["600519.SH"], dt.datetime(2026, 1, 1), dt.datetime(2026, 1, 15))
|
||||
assert df.columns == ["symbol", "date", "open", "high", "low", "close", "volume", "amount"]
|
||||
assert df.height == 2
|
||||
assert df["symbol"].unique().to_list() == ["600519.SH"]
|
||||
assert df.schema["date"] == pl.Date
|
||||
assert df.schema["close"] == pl.Float64
|
||||
|
||||
|
||||
def test_get_adj_factors_from_bridge_ratio(monkeypatch):
|
||||
# 桥接内部已算好 ex_factor = close_hfq/close_none, 这里验证 Python 侧归一化。
|
||||
_patch_run_job(monkeypatch, {
|
||||
"adj": {"ok": True, "op": "adj", "rows": {
|
||||
"600519.SH": [
|
||||
{"symbol": "600519.SH", "trade_date": "2020-01-02", "ex_factor": 5.29},
|
||||
{"symbol": "600519.SH", "trade_date": "2020-01-03", "ex_factor": 5.30},
|
||||
],
|
||||
}},
|
||||
})
|
||||
df = StockSDKProvider().get_adj_factors(["600519.SH"], None, None)
|
||||
assert df.columns == ["symbol", "trade_date", "ex_factor"]
|
||||
assert df.height == 2
|
||||
assert df.schema["trade_date"] == pl.Date
|
||||
assert abs(df["ex_factor"][0] - 5.29) < 1e-9
|
||||
|
||||
|
||||
def test_get_minute_datetime_is_beijing_wall_clock(monkeypatch):
|
||||
# timestamp 1779327300000 = 2026-05-21 01:35 UTC = 09:35 Asia/Shanghai
|
||||
_patch_run_job(monkeypatch, {
|
||||
"minute": {"ok": True, "op": "minute", "rows": {
|
||||
"600519.SH": [
|
||||
{"date": "2026-05-21 09:35", "open": 1284.9, "high": 1289.1, "low": 1283.9,
|
||||
"close": 1286.7, "volume": 2740, "amount": 3.6e8, "timestamp": 1779327300000},
|
||||
],
|
||||
}},
|
||||
})
|
||||
df = StockSDKProvider().get_minute(["600519.SH"], None, None)
|
||||
assert set(df.columns) == {"symbol", "datetime", "open", "high", "low", "close", "volume", "amount"}
|
||||
assert df.height == 1
|
||||
ts = df["datetime"][0]
|
||||
assert (ts.hour, ts.minute) == (9, 35)
|
||||
assert df["symbol"][0] == "600519.SH"
|
||||
|
||||
|
||||
def test_get_realtime_passthrough(monkeypatch):
|
||||
rows = [{"symbol": "600519.SH", "name": "贵州茅台", "last_price": 1200.0,
|
||||
"prev_close": 1194.0, "open": 1186.0, "high": 1203.0, "low": 1180.0, "volume": 16325}]
|
||||
_patch_run_job(monkeypatch, {"realtime": {"ok": True, "op": "realtime", "rows": rows}})
|
||||
out = StockSDKProvider().get_realtime()
|
||||
assert out == rows
|
||||
required = {"symbol", "last_price", "prev_close", "open", "high", "low", "volume"}
|
||||
assert required <= set(out[0].keys())
|
||||
|
||||
|
||||
def test_get_instruments_flatten_compatible(monkeypatch):
|
||||
rows = [{"symbol": "600519.SH", "name": "贵州茅台", "code": "600519", "exchange": "SH",
|
||||
"region": "CN", "type": "stock", "total_shares": 1, "float_shares": 1,
|
||||
"limit_up": 1.0, "limit_down": 1.0}]
|
||||
_patch_run_job(monkeypatch, {"instruments": {"ok": True, "op": "instruments", "rows": rows}})
|
||||
out = StockSDKProvider().get_instruments("stock")
|
||||
assert out[0]["symbol"] == "600519.SH"
|
||||
assert out[0]["exchange"] == "SH"
|
||||
# 非 stock 资产暂不覆盖
|
||||
assert StockSDKProvider().get_instruments("etf") == []
|
||||
|
||||
|
||||
def test_empty_symbols_returns_empty():
|
||||
p = StockSDKProvider()
|
||||
assert p.get_daily([], None, None).is_empty()
|
||||
assert p.get_adj_factors([], None, None).is_empty()
|
||||
assert p.get_minute([], None, None).is_empty()
|
||||
|
||||
|
||||
def test_bridge_error_degrades_to_empty(monkeypatch):
|
||||
def boom(job, timeout=None): # noqa: ARG001
|
||||
raise sp.bridge.StockSDKBridgeError("node missing")
|
||||
|
||||
monkeypatch.setattr(sp.bridge, "run_job", boom)
|
||||
assert StockSDKProvider().get_daily(["600519.SH"], None, None).is_empty()
|
||||
assert StockSDKProvider().get_realtime() == []
|
||||
assert StockSDKProvider().get_instruments("stock") == []
|
||||
|
||||
|
||||
def test_plugin_discovered_in_loader():
|
||||
"""插件被发现并记录状态 (即使依赖没装, 不可用)。"""
|
||||
from app.data_providers import custom as cs
|
||||
|
||||
plugins = {p["name"]: p for p in cs.list_plugins()}
|
||||
assert "stocksdk" in plugins
|
||||
assert plugins["stocksdk"]["runtime"] == "node"
|
||||
assert "daily" in plugins["stocksdk"]["datasets"]
|
||||
assert "realtime" in plugins["stocksdk"]["datasets"]
|
||||
assert "financial" not in plugins["stocksdk"]["datasets"]
|
||||
assert cs.is_builtin("stocksdk")
|
||||
# 内置源不出现在用户自定义源列表
|
||||
assert "stocksdk" not in [s["name"] for s in cs.list_sources()]
|
||||
|
||||
|
||||
def test_plugin_registered_when_available(monkeypatch):
|
||||
"""依赖可用时, 插件注册进 _PROVIDERS 并可路由。"""
|
||||
from app.data_providers import custom as cs
|
||||
from app.data_providers.custom import loader as L
|
||||
|
||||
# mock availability 返回 (True, "ok")
|
||||
monkeypatch.setattr(L, "_call_check", lambda ref: (True, "ok"))
|
||||
monkeypatch.setattr(L, "_load_entry", _load_stocksdk_entry)
|
||||
L._load_builtin_plugins()
|
||||
|
||||
assert "stocksdk" in cs.names()
|
||||
assert cs.is_custom_provider("stocksdk")
|
||||
assert cs.provider_has_dataset("stocksdk", "daily")
|
||||
assert cs.provider_has_dataset("stocksdk", "realtime")
|
||||
assert not cs.provider_has_dataset("stocksdk", "financial")
|
||||
|
||||
|
||||
def _load_stocksdk_entry(entry_ref: str):
|
||||
"""测试用: 无条件加载 stocksdk provider 类 (跳过 check)。"""
|
||||
if "StockSDKProvider" in entry_ref:
|
||||
from app.plugins.stocksdk.provider import StockSDKProvider
|
||||
return StockSDKProvider
|
||||
if "availability" in entry_ref:
|
||||
from app.plugins.stocksdk.bridge import availability
|
||||
return availability
|
||||
raise ValueError(f"unknown entry: {entry_ref}")
|
||||
|
||||
|
||||
def test_builtin_not_editable():
|
||||
from app.data_providers import custom as cs
|
||||
|
||||
assert cs.get_config_dict("stocksdk") is None
|
||||
for fn in (lambda: cs.save_config("stocksdk", {}), lambda: cs.delete_config("stocksdk")):
|
||||
try:
|
||||
fn()
|
||||
raise AssertionError("expected ValueError for builtin")
|
||||
except ValueError:
|
||||
pass
|
||||
Generated
+1834
-1834
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,118 @@
|
||||
# 数据源插件开发指南
|
||||
|
||||
数据源插件是可选的行情数据来源(stock-sdk、akshare 等),作为独立模块放在
|
||||
`backend/app/plugins/` 下。用户**手动安装依赖**后才可用;不安装完全不影响主功能。
|
||||
|
||||
## 快速上手
|
||||
|
||||
一个插件 = 一个目录 + 一个 `plugin.yaml` 清单:
|
||||
|
||||
```
|
||||
backend/app/plugins/<your_plugin>/
|
||||
├── plugin.yaml # 清单(必需)
|
||||
├── provider.py # Provider 实现(必需)
|
||||
├── ... # 桥接/依赖文件(按需)
|
||||
```
|
||||
|
||||
### plugin.yaml 字段
|
||||
|
||||
```yaml
|
||||
name: my_source # 唯一标识, 只允许 [a-z0-9_], 也是 provider name
|
||||
display_name: "我的数据源" # 设置页显示名
|
||||
runtime: python # 运行时类型: node | python | none
|
||||
entry: app.plugins.my_source.provider:MyProvider # provider 类的导入路径
|
||||
check: app.plugins.my_source.bridge:availability # 可用性检测函数(可选)
|
||||
datasets: [daily, adj_factor, minute, realtime] # 支持的数据集
|
||||
description: "数据源描述"
|
||||
install_hint: "pip install xxx" # 未装依赖时显示的安装提示
|
||||
```
|
||||
|
||||
### runtime 字段说明
|
||||
|
||||
| runtime | 含义 | 典型场景 |
|
||||
|---|---|---|
|
||||
| `python` | 纯 Python 依赖, `pip install` | akshare、tushare |
|
||||
| `node` | 需要 Node.js 运行时, `npm install` | stock-sdk |
|
||||
| `none` | 无额外依赖 | 纯 HTTP API 源 |
|
||||
|
||||
`runtime` 字段当前仅用于 UI 展示, 实际依赖检测由 `check` 函数负责。
|
||||
|
||||
### check 函数
|
||||
|
||||
插件自己负责检测依赖是否已安装。后端启动时会调用此函数:
|
||||
|
||||
```python
|
||||
# app/plugins/my_source/bridge.py
|
||||
def availability() -> tuple[bool, str]:
|
||||
"""返回 (是否可用, 原因)。不抛异常。"""
|
||||
try:
|
||||
import akshare # noqa: F401
|
||||
return True, "ok"
|
||||
except ImportError:
|
||||
return False, "未安装 akshare, 运行: pip install akshare"
|
||||
```
|
||||
|
||||
- **可用** → 插件注册进路由表, 设置页可切换
|
||||
- **不可用** → 设置页显示插件卡片但灰显, 展示 `install_hint`
|
||||
|
||||
## Provider 接口契约
|
||||
|
||||
Provider 是一个普通 Python 类(无需继承基类), 实现以下方法签名。方法签名对齐
|
||||
`GenericHTTPProvider`, 这样 services 层(kline_sync / quote_service 等)的路由逻辑
|
||||
零改动即可路由到插件。
|
||||
|
||||
```python
|
||||
class MyProvider:
|
||||
name = "my_source"
|
||||
builtin = True # 标记为内置(不可被用户编辑/删除)
|
||||
|
||||
def __init__(self):
|
||||
self.config = MyConfig() # 需有 .datasets 属性(dict, key 是数据集名)
|
||||
|
||||
def close(self) -> None:
|
||||
"""清理资源(load_all 重建注册表时会调)。"""
|
||||
|
||||
def get_daily(self, symbols, start_time, end_time, asset_type="stock", on_chunk_done=None) -> pl.DataFrame:
|
||||
"""日K: 返回 schema [symbol, date, open, high, low, close, volume, amount]"""
|
||||
|
||||
def get_adj_factors(self, symbols, start_time, end_time, asset_type="stock", on_chunk_done=None) -> pl.DataFrame:
|
||||
"""除权因子: 返回 schema [symbol, trade_date, ex_factor]"""
|
||||
|
||||
def get_minute(self, symbols, start_time, end_time, asset_type="stock", on_chunk_done=None, freq="1m") -> pl.DataFrame:
|
||||
"""分钟K: 返回 schema [symbol, datetime, open, high, low, close, volume, amount]"""
|
||||
|
||||
def get_realtime(self) -> list[dict]:
|
||||
"""全市场实时快照: 返回 list[dict], 每行含 symbol/last_price/prev_close/open/high/low/volume"""
|
||||
|
||||
def get_instruments(self, asset_type="stock") -> list[dict]:
|
||||
"""标的维表(可选): 返回 tickflow Instrument 形状的行, 供 instrument_sync 复用 flatten"""
|
||||
```
|
||||
|
||||
### config.datasets 的作用
|
||||
|
||||
`provider_has_dataset(name, dataset)` 通过 `dataset in provider.config.datasets` 判断。
|
||||
这是 services 层路由的关键: 用户在设置页选了插件, 但某数据集未声明时, 该数据集
|
||||
自动回退 TickFlow。
|
||||
|
||||
```python
|
||||
class MyConfig:
|
||||
datasets = {"daily": ..., "realtime": ...} # key 是数据集名, value 任意
|
||||
```
|
||||
|
||||
## 现有插件参考
|
||||
|
||||
- **`backend/app/plugins/stocksdk/`** — Node 型插件, 通过 subprocess 桥接调用 stock-sdk
|
||||
- `bridge.py` — Python↔Node 桥接 + availability 检测
|
||||
- `bridge.mjs` — Node 端(并发池、重试、SDK 解析)
|
||||
- `provider.py` — Provider 实现(归一化、分批、错误降级)
|
||||
|
||||
## 路由机制(无需关心, 仅参考)
|
||||
|
||||
后端启动时, `loader.py` 的 `_load_builtin_plugins()` 扫描 `plugins/` 目录:
|
||||
1. 读每个子目录的 `plugin.yaml`
|
||||
2. 调 `check` 函数检测可用性
|
||||
3. 可用 → 动态 import `entry` 指向的 Provider 类 → 注册进 `_PROVIDERS`
|
||||
4. 不可用 → 记录状态, 设置页显示但不可切换
|
||||
|
||||
注册后, 插件和用户 YAML 自定义源走**完全相同的路由路径**(services 层的
|
||||
`provider_has_dataset` / `get_provider` 调用), 无需额外集成代码。
|
||||
@@ -693,6 +693,18 @@ export interface DataSourceItem {
|
||||
path?: string | null
|
||||
}
|
||||
|
||||
/** 内置可选插件数据源 (plugins/ 目录, 需手动装依赖) */
|
||||
export interface PluginDataSourceItem {
|
||||
name: string
|
||||
display_name: string
|
||||
datasets: string[]
|
||||
runtime: string // node | python | none
|
||||
available: boolean // 依赖是否已安装
|
||||
status: string // 可用性原因 (供 UI 显示)
|
||||
description: string
|
||||
install_hint: string // 未装依赖时显示的安装命令
|
||||
}
|
||||
|
||||
export interface DataSourceLoadError {
|
||||
name?: string
|
||||
path: string
|
||||
@@ -701,6 +713,7 @@ export interface DataSourceLoadError {
|
||||
|
||||
export interface DataSourcesResponse {
|
||||
builtin: DataSourceItem[]
|
||||
plugins: PluginDataSourceItem[]
|
||||
custom: DataSourceItem[]
|
||||
errors: DataSourceLoadError[]
|
||||
config_dir: string
|
||||
@@ -857,6 +870,20 @@ export const api = {
|
||||
deleteDataSource: (name: string) =>
|
||||
request<DataSourcesResponse>(`/api/settings/data-sources/${encodeURIComponent(name)}`, { method: 'DELETE' }),
|
||||
reloadDataSources: () => request<DataSourcesResponse>('/api/settings/data-sources/reload', { method: 'POST' }),
|
||||
installPlugin: (name: string) => {
|
||||
// npm install 可能耗时较长, 用 6 分钟超时
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => controller.abort(), 360_000)
|
||||
return request<DataSourcesResponse & { install_ok: boolean; install_message: string }>(
|
||||
`/api/settings/plugins/${encodeURIComponent(name)}/install`,
|
||||
{ method: 'POST', signal: controller.signal },
|
||||
).finally(() => clearTimeout(timer))
|
||||
},
|
||||
uninstallPlugin: (name: string) =>
|
||||
request<DataSourcesResponse & { uninstall_ok: boolean; uninstall_message: string }>(
|
||||
`/api/settings/plugins/${encodeURIComponent(name)}/install`,
|
||||
{ method: 'DELETE' },
|
||||
),
|
||||
testDataSource: (provider: string, dataset: string, symbols?: string[]) =>
|
||||
request<DataSourceTestResult>('/api/settings/data-sources/test', {
|
||||
method: 'POST',
|
||||
|
||||
@@ -2,7 +2,7 @@ import { useState } from 'react'
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { motion, AnimatePresence } from 'framer-motion'
|
||||
import { Check, Database, Plus, RefreshCw, Zap, FileWarning } from 'lucide-react'
|
||||
import { api, type DataSourceItem } from '@/lib/api'
|
||||
import { api, type DataSourceItem, type PluginDataSourceItem } from '@/lib/api'
|
||||
import { QK } from '@/lib/queryKeys'
|
||||
import { usePreferences } from '@/lib/useSharedQueries'
|
||||
import { toast } from '@/components/Toast'
|
||||
@@ -70,14 +70,50 @@ export function SettingsDataSourcesPanel() {
|
||||
onSuccess: (_data, name) => setSelected(name),
|
||||
})
|
||||
|
||||
const installMut = useMutation({
|
||||
mutationFn: (name: string) => api.installPlugin(name),
|
||||
onSuccess: (data) => {
|
||||
qc.invalidateQueries({ queryKey: QK.dataSources })
|
||||
if (data.install_ok) {
|
||||
toast('插件依赖安装成功', 'success')
|
||||
} else {
|
||||
toast(data.install_message || '安装失败', 'error')
|
||||
}
|
||||
},
|
||||
onError: (e: Error) => toast(`安装失败: ${e.message}`, 'error'),
|
||||
})
|
||||
|
||||
const uninstallMut = useMutation({
|
||||
mutationFn: (name: string) => api.uninstallPlugin(name),
|
||||
onSuccess: (data) => {
|
||||
qc.invalidateQueries({ queryKey: QK.dataSources })
|
||||
qc.invalidateQueries({ queryKey: QK.preferences })
|
||||
if (data.uninstall_ok) {
|
||||
toast(data.uninstall_message || '已卸载', 'success')
|
||||
} else {
|
||||
toast(data.uninstall_message || '卸载失败', 'error')
|
||||
}
|
||||
},
|
||||
onError: (e: Error) => toast(`卸载失败: ${e.message}`, 'error'),
|
||||
})
|
||||
|
||||
const builtin: DataSourceItem[] = sources.data?.builtin ?? []
|
||||
const pluginList: PluginDataSourceItem[] = sources.data?.plugins ?? []
|
||||
const customList: DataSourceItem[] = sources.data?.custom ?? []
|
||||
const errors = sources.data?.errors ?? []
|
||||
const activeName = prefs.data?.daily_data_provider || 'tickflow'
|
||||
|
||||
// 顶部数据源选择列表 (内置 + 自定义 + 新增)
|
||||
// 插件 name → 状态 (供卡片渲染时判断 available/installing 等)
|
||||
const pluginMap = new Map(pluginList.map(p => [p.name, p]))
|
||||
const pluginNames = new Set(pluginList.map(p => p.name))
|
||||
|
||||
// 顶部数据源选择列表 (内置 + 所有插件 + 自定义 + 新增)
|
||||
const pluginItems: DataSourceItem[] = pluginList.map(p => ({
|
||||
name: p.name, display_name: p.display_name, datasets: p.datasets,
|
||||
}))
|
||||
const allItems = [
|
||||
...builtin,
|
||||
...pluginItems,
|
||||
...customList,
|
||||
]
|
||||
|
||||
@@ -91,7 +127,10 @@ export function SettingsDataSourcesPanel() {
|
||||
<div className="flex items-center gap-2.5">
|
||||
<Database className="h-4 w-4 text-secondary" />
|
||||
<h2 className="text-sm font-medium text-foreground">数据源</h2>
|
||||
<span className="text-[10px] text-muted/40 font-mono truncate max-w-[280px]" title={sources.data?.config_dir}>
|
||||
<span
|
||||
className="text-[10px] text-muted/40 font-mono truncate hidden lg:inline max-w-[480px]"
|
||||
title={sources.data?.config_dir}
|
||||
>
|
||||
{sources.data?.config_dir}
|
||||
</span>
|
||||
</div>
|
||||
@@ -119,33 +158,84 @@ export function SettingsDataSourcesPanel() {
|
||||
{allItems.map(item => {
|
||||
const isActive = activeName === item.name
|
||||
const isSelected = selected === item.name
|
||||
const plugin = pluginMap.get(item.name)
|
||||
const pluginUnavailable = plugin && !plugin.available
|
||||
const installing = installMut.isPending && installMut.variables === item.name
|
||||
const uninstalling = uninstallMut.isPending && uninstallMut.variables === item.name
|
||||
return (
|
||||
<div
|
||||
key={item.name}
|
||||
onClick={() => {
|
||||
if (pluginUnavailable) return // 未安装的插件不可选中
|
||||
setSelected(item.name)
|
||||
if (item.name !== 'tickflow') {
|
||||
// 只有用户自定义源 (YAML) 才进编辑器; tickflow 和插件不可编辑
|
||||
if (customList.some(c => c.name === item.name)) {
|
||||
editExisting.mutate(item.name)
|
||||
}
|
||||
}}
|
||||
className={`relative cursor-pointer text-left rounded-lg border px-3.5 py-3 transition-all ${
|
||||
isSelected
|
||||
? 'border-accent/50 bg-accent/5 ring-1 ring-accent/20'
|
||||
: 'border-border/60 bg-elevated/20 hover:bg-elevated/40'
|
||||
className={`relative text-left rounded-lg border px-3.5 py-3 transition-all ${
|
||||
pluginUnavailable
|
||||
? 'border-border/40 bg-elevated/10 opacity-70'
|
||||
: isSelected
|
||||
? 'border-accent/50 bg-accent/5 ring-1 ring-accent/20 cursor-pointer'
|
||||
: 'border-border/60 bg-elevated/20 hover:bg-elevated/40 cursor-pointer'
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<span className={`h-1.5 w-1.5 rounded-full shrink-0 ${isActive ? 'bg-accent' : 'bg-transparent border border-muted/40'}`} />
|
||||
<span className={`h-1.5 w-1.5 rounded-full shrink-0 ${
|
||||
pluginUnavailable ? 'bg-muted/30' : isActive ? 'bg-accent' : 'bg-transparent border border-muted/40'
|
||||
}`} />
|
||||
<span className={`text-sm truncate flex-1 ${isActive ? 'font-medium text-foreground' : 'text-secondary'}`}>
|
||||
{item.display_name}
|
||||
</span>
|
||||
{item.name === 'tickflow' && (
|
||||
<span className="text-[9px] text-muted/50 uppercase tracking-wider shrink-0">内置</span>
|
||||
)}
|
||||
{isActive ? (
|
||||
{pluginNames.has(item.name) && (
|
||||
<span className="text-[9px] text-muted/50 uppercase tracking-wider shrink-0">插件</span>
|
||||
)}
|
||||
{/* 右侧操作区: 插件未安装→安装按钮; 已激活→使用中; 否则→使用/卸载 */}
|
||||
{pluginUnavailable ? (
|
||||
installing ? (
|
||||
<span className="inline-flex items-center gap-1 text-[9px] text-accent shrink-0">
|
||||
<RefreshCw className="h-2.5 w-2.5 animate-spin" /> 安装中...
|
||||
</span>
|
||||
) : (
|
||||
<button
|
||||
onClick={(e) => { e.stopPropagation(); installMut.mutate(item.name) }}
|
||||
disabled={installMut.isPending}
|
||||
className="shrink-0 inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[10px] font-medium bg-accent/10 text-accent hover:bg-accent/20 transition-colors disabled:opacity-50"
|
||||
>
|
||||
<Zap className="h-2.5 w-2.5" /> 安装
|
||||
</button>
|
||||
)
|
||||
) : isActive ? (
|
||||
<span className="inline-flex items-center gap-0.5 text-[9px] text-accent shrink-0">
|
||||
<Check className="h-2.5 w-2.5" /> 使用中
|
||||
</span>
|
||||
) : plugin ? (
|
||||
/* 已安装插件: 使用 + 卸载 */
|
||||
<div className="flex items-center gap-1 shrink-0">
|
||||
<button
|
||||
onClick={(e) => { e.stopPropagation(); switchProvider.mutate(item.name) }}
|
||||
disabled={switchProvider.isPending}
|
||||
className="rounded px-1.5 py-0.5 text-[10px] font-medium bg-accent/10 text-accent hover:bg-accent/20 transition-colors disabled:opacity-50"
|
||||
>
|
||||
使用
|
||||
</button>
|
||||
{uninstalling ? (
|
||||
<RefreshCw className="h-2.5 w-2.5 animate-spin text-muted" />
|
||||
) : (
|
||||
<button
|
||||
onClick={(e) => { e.stopPropagation(); uninstallMut.mutate(item.name) }}
|
||||
disabled={uninstallMut.isPending}
|
||||
className="text-[10px] text-muted/50 hover:text-danger transition-colors disabled:opacity-40"
|
||||
title="卸载依赖"
|
||||
>
|
||||
卸载
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<button
|
||||
onClick={(e) => { e.stopPropagation(); switchProvider.mutate(item.name) }}
|
||||
@@ -168,6 +258,10 @@ export function SettingsDataSourcesPanel() {
|
||||
{item.name === 'tickflow' && (
|
||||
<div className="text-[10px] text-muted/60 ml-3.5">日K · 除权 · 实时 · 分钟K</div>
|
||||
)}
|
||||
{/* 未安装插件显示安装命令提示 */}
|
||||
{pluginUnavailable && plugin?.install_hint && (
|
||||
<div className="ml-3.5 mt-1 text-[10px] text-muted/40 font-mono truncate">{plugin.install_hint}</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
@@ -223,7 +317,7 @@ export function SettingsDataSourcesPanel() {
|
||||
onSwitch={() => switchProvider.mutate('tickflow')}
|
||||
switching={switchProvider.isPending}
|
||||
/>
|
||||
) : (
|
||||
) : selected === '__new__' || customList.some(c => c.name === selected) ? (
|
||||
<DataSourceEditor
|
||||
key={selected}
|
||||
initial={null}
|
||||
@@ -241,7 +335,15 @@ export function SettingsDataSourcesPanel() {
|
||||
onActivate={(name) => switchProvider.mutate(name)}
|
||||
onDelete={selected !== '__new__' && selectedCustom ? () => setConfirmDelete(selected) : undefined}
|
||||
/>
|
||||
)}
|
||||
) : pluginList.find(x => x.name === selected) ? (
|
||||
/* 选中插件: 显示只读详情, 不进编辑器 */
|
||||
<PluginDetail
|
||||
plugin={pluginList.find(x => x.name === selected)!}
|
||||
isActive={activeName === selected}
|
||||
onSwitch={() => switchProvider.mutate(selected)}
|
||||
switching={switchProvider.isPending}
|
||||
/>
|
||||
) : null}
|
||||
</motion.div>
|
||||
</AnimatePresence>
|
||||
|
||||
@@ -279,6 +381,45 @@ export function SettingsDataSourcesPanel() {
|
||||
)
|
||||
}
|
||||
|
||||
function PluginDetail({ plugin, isActive, onSwitch, switching }: {
|
||||
plugin: PluginDataSourceItem
|
||||
isActive: boolean
|
||||
onSwitch: () => void
|
||||
switching: boolean
|
||||
}) {
|
||||
return (
|
||||
<section className="rounded-card border border-border bg-surface p-6">
|
||||
<div className="flex items-start gap-4 mb-5">
|
||||
<div className="h-11 w-11 rounded-xl bg-accent/10 flex items-center justify-center shrink-0">
|
||||
<Zap className="h-5 w-5 text-accent" />
|
||||
</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<h3 className="text-base font-semibold text-foreground">{plugin.display_name}</h3>
|
||||
<span className="text-[10px] text-muted/50 uppercase tracking-wider">插件 · {plugin.runtime}</span>
|
||||
</div>
|
||||
{plugin.description && <p className="text-xs text-secondary leading-relaxed">{plugin.description}</p>}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
{isActive ? (
|
||||
<span className="inline-flex items-center gap-1.5 text-xs text-accent">
|
||||
<Check className="h-3.5 w-3.5" /> 当前使用中
|
||||
</span>
|
||||
) : (
|
||||
<button
|
||||
onClick={onSwitch}
|
||||
disabled={switching}
|
||||
className="px-3 py-1.5 rounded-btn bg-accent/10 text-accent hover:bg-accent/20 text-xs font-medium transition-colors disabled:opacity-50"
|
||||
>
|
||||
切换为当前数据源
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
function TickFlowDetail({ active, onSwitch, switching }: { active: boolean; onSwitch: () => void; switching: boolean }) {
|
||||
return (
|
||||
<section className="rounded-card border border-border bg-surface p-6">
|
||||
|
||||
Reference in New Issue
Block a user