feat: 数据源插件化架构 + stock-sdk 首个插件

将可选数据源改为插件化架构: 插件代码放 backend/app/plugins/<name>/,
用户在设置页点击安装依赖, 主仓库不背运行时依赖(nodejs 等)。

架构:
- loader.py 新增 _load_builtin_plugins() 扫描 plugins/ 目录, 通过
  plugin.yaml 清单动态发现并注册插件(委托自检 + 优雅降级)
- runtime 字段支持 node/python, 安装/卸载分别用 npm/pip
- 现有 tickflow + YAML 自定义源逻辑 100% 保留, 插件是叠加层

stock-sdk 插件 (plugins/stocksdk/):
- 原始实现来自 @forrany 的 PR #57, 迁移到插件化架构, 署名保留
- Node 桥接 (bridge.py/mjs) + Python provider, 覆盖日K/除权/分钟/实时
- 设置页主列表直接显示: 未装灰显+安装按钮, 装完可切换/卸载

配套:
- instrument_sync: 通用增强, 任何 provider 都能提供标的维表
- install/uninstall: uv 优先回退 pip, 容错坏 uv.toml + 国内镜像
- 10 例单测全过, tsc 零错误
This commit is contained in:
shy3130
2026-07-06 17:00:56 +08:00
parent cc697e8ea4
commit 966d94ab03
16 changed files with 3374 additions and 1860 deletions
+49 -2
View File
@@ -413,10 +413,11 @@ def get_preferences() -> dict:
@router.get("/data-sources")
def list_data_sources() -> dict:
"""列出已加载的自定义数据源。"""
"""列出已加载的数据源 (内置 / 插件 / 用户自定义)"""
from app.data_providers import custom as custom_sources
return {
"builtin": [{"name": "tickflow", "display_name": "TickFlow", "datasets": ["daily", "adj_factor", "realtime"]}],
"builtin": [{"name": "tickflow", "display_name": "TickFlow", "datasets": ["daily", "adj_factor", "realtime", "minute"]}],
"plugins": custom_sources.list_plugins(),
"custom": custom_sources.list_sources(),
"errors": custom_sources.errors(),
"config_dir": str(custom_sources.data_sources_dir()),
@@ -431,6 +432,52 @@ def reload_data_sources() -> dict:
return list_data_sources()
@router.post("/plugins/{name}/install")
def install_plugin(name: str) -> dict:
"""安装指定插件的依赖 (npm install / pip install), 完成后重新扫描。
根据 plugin.yaml 的 runtime 字段决定安装方式。安装可能耗时较长 (网络下载),
客户端需设较长超时。
"""
from app.data_providers import custom as custom_sources
if not custom_sources.is_builtin(name):
raise HTTPException(status_code=404, detail=f"插件 '{name}' 不存在")
ok, message = custom_sources.install_plugin(name)
# 无论成功失败都重新扫描, 刷新插件状态 (安装可能部分成功)
custom_sources.load_all()
result = list_data_sources()
result["install_ok"] = ok
result["install_message"] = message
return result
@router.delete("/plugins/{name}/install")
def uninstall_plugin(name: str) -> dict:
"""卸载指定插件的依赖 (删除 node_modules / pip uninstall), 完成后重新扫描。
如果该插件当前正被使用, 自动回退到 tickflow。
"""
from app.data_providers import custom as custom_sources
from app.services import preferences
if not custom_sources.is_builtin(name):
raise HTTPException(status_code=404, detail=f"插件 '{name}' 不存在")
ok, message = custom_sources.uninstall_plugin(name)
# 卸载后若该插件正被使用, 回退 tickflow
for getter, key, default in [
(preferences.get_daily_data_provider, "daily_data_provider", "tickflow"),
(preferences.get_minute_data_provider, "minute_data_provider", "tickflow"),
(preferences.get_realtime_data_provider, "realtime_data_provider", "tickflow"),
(preferences.get_financial_provider, "financial_data_provider", "tickflow"),
]:
if getter() == name:
preferences.save({key: default})
custom_sources.load_all()
result = list_data_sources()
result["uninstall_ok"] = ok
result["uninstall_message"] = message
return result
@router.get("/data-sources/{name}")
def get_data_source(name: str) -> dict:
"""读取一个自定义数据源的完整配置(用于前端编辑回填)。"""
@@ -5,12 +5,16 @@ from app.data_providers.custom.loader import (
errors,
get_config_dict,
get_provider,
install_plugin,
is_builtin,
is_custom_provider,
list_plugins,
list_sources,
load_all,
names,
provider_has_dataset,
save_config,
uninstall_plugin,
)
__all__ = [
@@ -19,10 +23,14 @@ __all__ = [
"errors",
"get_config_dict",
"get_provider",
"install_plugin",
"is_builtin",
"is_custom_provider",
"list_plugins",
"list_sources",
"load_all",
"names",
"provider_has_dataset",
"save_config",
"uninstall_plugin",
]
+272 -1
View File
@@ -1,8 +1,11 @@
"""Load custom data source definitions from user data files."""
from __future__ import annotations
import importlib
import logging
import re
import shutil
import subprocess
from pathlib import Path
import yaml
@@ -16,9 +19,18 @@ logger = logging.getLogger(__name__)
_PROVIDERS: dict[str, GenericHTTPProvider] = {}
_LOAD_ERRORS: list[dict] = []
# 内置插件状态: {name: {available, status, runtime, ...}} 供设置页独立分类展示。
# available=False 的插件不注册进 _PROVIDERS (不可切换), 但记录状态供 UI 显示安装提示。
_PLUGIN_STATUS: dict[str, dict] = {}
_NAME_RE = re.compile(r"^[a-z0-9_]+$")
def plugins_dir() -> Path:
"""内置可选插件目录 (app/plugins/, 与现有包结构一致, 开发态/容器态路径统一)。"""
return Path(__file__).resolve().parents[2] / "plugins"
def data_sources_dir() -> Path:
return settings.data_dir / "data_sources"
@@ -47,8 +59,12 @@ def load_all(path: Path | None = None) -> None:
logger.warning("custom data source load failed %s: %s", file, e)
_LOAD_ERRORS.append({"path": str(file), "errors": [str(e)]})
# 内置可选插件 (plugins/ 目录)。与用户 YAML 源独立, 缺依赖只记状态不报错。
_load_builtin_plugins()
def list_sources() -> list[dict]:
"""只列出用户自定义 (YAML) 源。内置插件 (builtin=True) 由 list_plugins 独立呈现。"""
return [
{
"name": provider.name,
@@ -57,9 +73,164 @@ def list_sources() -> list[dict]:
"path": str(provider.config.path) if provider.config.path else None,
}
for provider in _PROVIDERS.values()
if not getattr(provider, "builtin", False)
]
def list_plugins() -> list[dict]:
"""返回所有内置插件的状态 (含已装/未装), 供设置页独立分类显示。"""
return list(_PLUGIN_STATUS.values())
def plugin_manifest(name: str) -> dict | None:
"""读取指定插件的 plugin.yaml 清单。"""
plugin_dir = plugins_dir() / (name or "")
manifest_path = plugin_dir / "plugin.yaml"
if not manifest_path.exists():
return None
return yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
def plugin_dir_of(name: str) -> Path:
"""返回插件目录路径。"""
return plugins_dir() / (name or "")
def install_plugin(name: str) -> tuple[bool, str]:
"""安装指定插件的依赖。根据 runtime 执行 npm install / pip install。
返回 (是否成功, 消息)。成功后调用方应 reload 重新扫描。
依赖未找到 (npm/pip 缺失) 或命令失败时返回 False。
"""
manifest = plugin_manifest(name)
if manifest is None:
return False, f"插件 '{name}' 不存在或无 plugin.yaml"
runtime = str(manifest.get("runtime", "none")).lower()
pdir = plugin_dir_of(name)
if not pdir.exists():
return False, f"插件目录不存在: {pdir}"
try:
if runtime == "node":
npm = shutil.which("npm")
if not npm:
return False, "未找到 npm, 请先安装 Node.js (>=18)"
# 在插件目录执行 npm install
result = subprocess.run(
[npm, "install", "--omit=dev", "--no-audit", "--no-fund"],
cwd=str(pdir),
capture_output=True,
text=True,
timeout=300,
)
elif runtime == "python":
# Python 型插件: 优先用 uv pip install (uv 管理的 venv 无 pip 模块),
# 回退 python -m pip。都装进当前后端虚拟环境。
# uv 容错: 用户全局 uv.toml 配置错误时 exit 2, 回退 --no-config 重试。
# UV_HTTP_TIMEOUT=300: akshare 等含大包(如 mini-racer 14MB), 默认 30s 不够。
req = pdir / "requirements.txt"
if not req.exists():
return False, "Python 型插件需要 requirements.txt"
uv_bin = shutil.which("uv")
if uv_bin:
result = subprocess.run(
[uv_bin, "pip", "install", "-r", str(req)],
capture_output=True, text=True, timeout=300,
env={**__import__("os").environ, "UV_HTTP_TIMEOUT": "300"},
)
# exit 2 通常是配置文件解析错误, 绕过配置重试
# --no-config 会丢镜像, 显式传国内镜像加速 (与用户 uv.toml 意图一致)
if result.returncode == 2:
result = subprocess.run(
[uv_bin, "pip", "install", "--no-config",
"--index-url", "https://pypi.tuna.tsinghua.edu.cn/simple",
"-r", str(req)],
capture_output=True, text=True, timeout=300,
env={**__import__("os").environ, "UV_HTTP_TIMEOUT": "300"},
)
else:
import sys
result = subprocess.run(
[sys.executable, "-m", "pip", "install", "-r", str(req)],
capture_output=True, text=True, timeout=300,
)
else:
return False, f"runtime={runtime} 无需安装依赖"
except subprocess.TimeoutExpired:
return False, "安装超时 (5分钟), 请检查网络后重试"
except Exception as e: # noqa: BLE001
return False, f"安装失败: {e}"
if result.returncode != 0:
# 取 stderr 的第一个 error: 行(真正的错误原因), 避免把 uv 的长字段列表返回给用户
raw = (result.stderr or result.stdout or "").strip()
first_err = ""
for line in raw.splitlines():
if line.strip().startswith(("error", "Error", "Caused by")):
first_err = line.strip()
break
msg = first_err or raw[-200:]
return False, f"安装失败 (exit {result.returncode}): {msg}"
return True, "安装成功"
def uninstall_plugin(name: str) -> tuple[bool, str]:
"""卸载指定插件的依赖。
node 型: 删除 node_modules 目录 (干净彻底, 下次需要重新 npm install)。
python 型: pip uninstall (包名从 requirements.txt 推断)。
"""
import shutil as _shutil
manifest = plugin_manifest(name)
if manifest is None:
return False, f"插件 '{name}' 不存在或无 plugin.yaml"
runtime = str(manifest.get("runtime", "none")).lower()
pdir = plugin_dir_of(name)
if not pdir.exists():
return False, f"插件目录不存在: {pdir}"
if runtime == "node":
nm = pdir / "node_modules"
if not nm.exists():
return True, "node_modules 不存在, 无需卸载"
try:
_shutil.rmtree(nm)
return True, "已删除 node_modules"
except Exception as e: # noqa: BLE001
return False, f"删除 node_modules 失败: {e}"
if runtime == "python":
req = pdir / "requirements.txt"
if not req.exists():
return False, "Python 型插件缺少 requirements.txt, 无法自动卸载"
# 读 requirements.txt 拿包名, 逐个 pip uninstall -y
pkgs = [l.strip().split("==")[0].split(">=")[0].strip()
for l in req.read_text().splitlines()
if l.strip() and not l.startswith("#")]
if not pkgs:
return True, "requirements.txt 无有效包名"
uv_bin = _shutil.which("uv")
cmd = [uv_bin, "pip", "uninstall", *pkgs] if uv_bin else None
if cmd is None:
import sys
cmd = [sys.executable, "-m", "pip", "uninstall", "-y", *pkgs]
try:
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
if result.returncode != 0:
return False, f"卸载失败: {(result.stderr or '').strip()[-300:]}"
return True, f"已卸载 {len(pkgs)} 个包"
except Exception as e: # noqa: BLE001
return False, f"卸载失败: {e}"
return False, f"runtime={runtime} 无需卸载"
def is_builtin(name: str) -> bool:
"""判断 name 是否为内置插件 (不可被用户编辑/删除)。"""
return (name or "").lower() in _PLUGIN_STATUS
def names() -> set[str]:
return set(_PROVIDERS)
@@ -91,7 +262,9 @@ def provider_has_dataset(name: str, dataset: str) -> bool:
def get_config_dict(name: str) -> dict | None:
"""读取一个已加载 custom 源的原始配置 dict(用于前端编辑回填)。"""
"""读取一个已加载 custom 源的原始配置 dict(用于前端编辑回填)。内置插件不可编辑。"""
if is_builtin(name):
return None
provider = _PROVIDERS.get((name or "").lower())
if provider is None:
return None
@@ -129,6 +302,8 @@ def _config_to_dict(config: CustomSourceConfig) -> dict:
def save_config(name: str, config: dict) -> Path:
"""把一份配置 dict 写成 data/data_sources/{name}.yaml, 返回写入路径。"""
if is_builtin(name):
raise ValueError(f"'{name}' 是内置插件, 不可编辑")
if not _NAME_RE.match(name or ""):
raise ValueError(f"invalid data source name: {name!r} (only lowercase a-z 0-9 _ allowed)")
base = data_sources_dir()
@@ -143,6 +318,8 @@ def save_config(name: str, config: dict) -> Path:
def delete_config(name: str) -> bool:
"""删除 data/data_sources/{name}.yaml。返回是否真的删除了。"""
if is_builtin(name):
raise ValueError(f"'{name}' 是内置插件, 不可删除")
if not _NAME_RE.match(name or ""):
raise ValueError(f"invalid data source name: {name!r}")
base = data_sources_dir().resolve()
@@ -226,3 +403,97 @@ def _sanitize_dataset(ds_cfg: dict) -> dict:
out["end_param"] = str(ds_cfg["end_param"])
return out
# ================================================================
# 内置可选插件 (plugins/ 目录) 的发现与注册
# ================================================================
def _load_builtin_plugins() -> None:
"""扫描 plugins/ 目录下每个含 plugin.yaml 的子目录, 动态加载。
缺依赖时记录 "不可用" 状态, 不抛异常, 不影响主流程。
每次调用重建 _PLUGIN_STATUS, 并把可用的插件注册进 _PROVIDERS。
"""
global _PLUGIN_STATUS
_PLUGIN_STATUS = {}
pdir = plugins_dir()
if not pdir.exists():
return
for plugin_dir in sorted(pdir.iterdir()):
if not plugin_dir.is_dir():
continue
manifest_path = plugin_dir / "plugin.yaml"
if not manifest_path.exists():
continue
try:
manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
_register_one_plugin(manifest)
except Exception as e: # noqa: BLE001
logger.warning("插件 %s 清单解析失败: %s", plugin_dir.name, e)
def _register_one_plugin(manifest: dict) -> None:
"""注册单个插件: 委托自检 → 可用则动态 import entry 注册进 _PROVIDERS。"""
name = manifest.get("name")
if not name or not _NAME_RE.match(name):
logger.warning("插件清单缺少合法 name: %r", name)
return
runtime = str(manifest.get("runtime", "none")).lower()
# 委托检测: 调用插件自己的 check 函数 (node 型/python 型各自实现)
available, reason = _call_check(manifest.get("check"))
_PLUGIN_STATUS[name] = {
"name": name,
"display_name": manifest.get("display_name", name),
"datasets": list(manifest.get("datasets", []) or []),
"runtime": runtime,
"available": available,
"status": reason,
"description": manifest.get("description", ""),
"install_hint": manifest.get("install_hint", ""),
}
if not available:
return # 依赖没装: 不注册, 但状态已记录供 UI 显示
# 可用 → 动态加载 provider 类并实例化
try:
provider_cls = _load_entry(manifest["entry"])
provider = provider_cls() if isinstance(provider_cls, type) else provider_cls
provider.builtin = True # 标记为内置 (list_sources 过滤, 不可被用户编辑/删除)
_PROVIDERS[name] = provider
logger.info("内置插件 %s 已注册 (runtime=%s)", name, runtime)
except Exception as e: # noqa: BLE001
# 声称可用但 import 失败 → 标记不可用, 避免启动崩溃
_PLUGIN_STATUS[name]["available"] = False
_PLUGIN_STATUS[name]["status"] = f"加载失败: {e}"
logger.warning("插件 %s provider 加载失败: %s", name, e)
def _call_check(check_ref: str | None) -> tuple[bool, str]:
"""调用插件清单里指定的可用性检测函数, 返回 (是否可用, 原因)。
check_ref 格式 "module.path:func_name"。无 check 字段时视为可用。
"""
if not check_ref:
return True, "ok"
try:
func = _load_entry(check_ref)
result = func()
# 兼容两种返回: (bool, str) 或 bool
if isinstance(result, tuple):
return bool(result[0]), str(result[1])
return bool(result), "ok" if result else "不可用"
except Exception as e: # noqa: BLE001
return False, str(e)
def _load_entry(entry_ref: str):
"""动态加载 'module.path:attr' 形式的引用, 返回属性对象 (类或函数)。"""
if ":" not in entry_ref:
raise ValueError(f"entry 格式应为 'module.path:attr', 得到: {entry_ref!r}")
module_path, attr = entry_ref.split(":", 1)
module = importlib.import_module(module_path)
return getattr(module, attr)
# 模块导入时即扫描一次, 保证 names()/_allowed_data_providers() 在 startup 前可用。
_load_builtin_plugins()
+13
View File
@@ -0,0 +1,13 @@
"""stock-sdk 内置数据源插件(免费行情, tickflow 的平替)。"""
from app.plugins.stocksdk.bridge import StockSDKBridgeError, availability, run_job
from app.plugins.stocksdk.provider import StockSDKProvider
PROVIDER_NAME = "stocksdk"
__all__ = [
"PROVIDER_NAME",
"StockSDKBridgeError",
"StockSDKProvider",
"availability",
"run_job",
]
+292
View File
@@ -0,0 +1,292 @@
#!/usr/bin/env node
/**
* stock-sdk 桥接脚本。Python 后端通过 subprocess 调用它,用真实 stock-sdk 抓数据。
*
* Original implementation by @forrany (PR #57), migrated to plugin architecture.
*
* 协议:
* - stdin: 单行 JSON { op, symbols?, adjust?, period?, start?, end?, concurrency? }
* - stdout: 单行 JSON
* daily/adj/minute: { ok:true, op, rows: { [appSymbol]: Row[] } }
* realtime/instruments: { ok:true, op, rows: Row[] }
* ping: { ok:true, op:'ping', version }
* 失败: { ok:false, error }
*
* op:
* daily —— 日K(adjust 默认 none), 每个 symbol 一组 bars
* adj —— 除权因子: 取 hfq 与 none 收盘价, ex_factor = close_hfq / close_none
* minute —— 分钟K(period 默认 5)
* realtime —— 全 A 股实时快照(batch.cn)
* instruments —— 全 A 股标的维表(batch.cn 提取元数据)
* ping —— 探活
*
* 说明: daily/adj/minute 的入参 symbols 是「app 符号」(如 600519.SH)。stock-sdk 能容错解析,
* 返回结果里我们**回显原始 app 符号**作为 key,避免 code→符号 的歧义(指数/股票同码等)。
* realtime/instruments 是全市场枚举,由 code + marketId 反推后缀。
*/
import { createRequire } from 'node:module'
import { pathToFileURL } from 'node:url'
import { execSync } from 'node:child_process'
import path from 'node:path'
/**
* 解析 stock-sdk 入口。ESM 的 bare import 只查本地 node_modules 链,不查全局,
* 因此这里显式在 [本地(脚本旁), 全局 npm root, NODE_PATH] 中查找后动态 import。
* 部署时优先用脚本旁 vendored 的 node_modules/stock-sdk。
*/
async function loadSDK() {
const require = createRequire(import.meta.url)
const scriptDir = path.dirname(new URL(import.meta.url).pathname)
// 候选 node_modules 目录(按优先级)
const nmDirs = [path.join(scriptDir, 'node_modules')]
for (const p of (process.env.NODE_PATH || '').split(path.delimiter).filter(Boolean)) nmDirs.push(p)
try {
const groot = execSync('npm root -g', { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()
if (groot) nmDirs.push(groot)
} catch {
/* npm 不可用则忽略 */
}
let entry
for (const nm of nmDirs) {
try {
entry = require.resolve(path.join(nm, 'stock-sdk'))
break
} catch {
/* 试下一个 */
}
}
if (!entry) {
throw new Error(
`无法解析 stock-sdk(已搜索: ${nmDirs.join(', ')})。请在桥接目录 npm install,或全局 npm i -g stock-sdk。`
)
}
const mod = await import(pathToFileURL(entry).href)
return mod.StockSDK || (mod.default && mod.default.StockSDK)
}
const MARKET_ID_TO_SUFFIX = { '1': 'SH', '51': 'SZ', '62': 'BJ' }
// 下游(Python)可能提前关闭管道,忽略 EPIPE 避免噪声崩溃。
process.stdout.on('error', (e) => {
if (e && e.code === 'EPIPE') process.exit(0)
})
function readStdin() {
return new Promise((resolve, reject) => {
let buf = ''
process.stdin.setEncoding('utf8')
process.stdin.on('data', (c) => (buf += c))
process.stdin.on('end', () => resolve(buf))
process.stdin.on('error', reject)
})
}
/** 简单并发池: 对 items 逐个跑 worker,最多 concurrency 个在飞。 */
async function mapPool(items, concurrency, worker) {
const results = new Array(items.length)
let next = 0
const runners = new Array(Math.min(concurrency, items.length)).fill(0).map(async () => {
while (true) {
const i = next++
if (i >= items.length) return
try {
results[i] = await worker(items[i], i)
} catch (e) {
results[i] = { __error: String((e && e.message) || e) }
}
}
})
await runners.reduce((p) => p, Promise.resolve())
await Promise.all(runners)
return results
}
/** code + marketId → app 符号(600519.SH)。反推失败则退化用 code 前缀猜测。 */
function toAppSymbol(code, marketId) {
const suffix = MARKET_ID_TO_SUFFIX[String(marketId)] || guessSuffix(code)
return suffix ? `${code}.${suffix}` : String(code)
}
function guessSuffix(code) {
const c = String(code)
if (/^(6|5|9)/.test(c)) return 'SH'
if (/^(0|3|1|2)/.test(c)) return 'SZ'
if (/^(4|8|92)/.test(c)) return 'BJ'
return ''
}
/** stock-sdk 的 adjust 取值是 '' | 'qfq' | 'hfq'(无 'none')。这里做兼容映射。 */
function normAdjust(v) {
if (v === 'hfq') return 'hfq'
if (v === 'qfq') return 'qfq'
return '' // none / undefined / 空 → 不复权
}
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
/**
* 上游(东财)偶发冷启动/限流时对活跃标的返回空数组(非报错)。对已知应有数据的请求,
* 空结果重试若干次以提升鲁棒性;真正无数据(退市/停牌/区间无交易)时多花几次调用可接受。
*/
async function fetchWithRetry(fn, { retries = 2, delayMs = 300 } = {}) {
let last = []
for (let i = 0; i <= retries; i++) {
const r = await fn()
if (Array.isArray(r) && r.length > 0) return r
last = Array.isArray(r) ? r : []
if (i < retries) await sleep(delayMs)
}
return last
}
async function fetchDaily(sdk, sym, { adjust, period = 'daily', start, end }) {
const opts = { period, adjust: normAdjust(adjust) }
if (start) opts.startDate = start
if (end) opts.endDate = end
return fetchWithRetry(() => sdk.kline.cn(sym, opts))
}
async function opDaily(sdk, job) {
const { symbols = [], adjust = 'none', period = 'daily', start, end, concurrency = 6 } = job
const out = {}
const rows = await mapPool(symbols, concurrency, (sym) =>
fetchDaily(sdk, sym, { adjust, period, start, end })
)
symbols.forEach((sym, i) => {
const r = rows[i]
out[sym] = Array.isArray(r) ? r : []
})
return out
}
async function opAdj(sdk, job) {
const { symbols = [], start, end, concurrency = 6 } = job
const out = {}
await mapPool(symbols, concurrency, async (sym) => {
const [none, hfq] = await Promise.all([
fetchDaily(sdk, sym, { adjust: 'none', start, end }),
fetchDaily(sdk, sym, { adjust: 'hfq', start, end }),
])
const noneByDate = new Map()
for (const b of none) if (b && b.close) noneByDate.set(b.date, b.close)
const factors = []
for (const b of hfq) {
if (!b || !b.date) continue
const rawClose = noneByDate.get(b.date)
if (!rawClose || !b.close) continue
factors.push({ symbol: sym, trade_date: b.date, ex_factor: b.close / rawClose })
}
out[sym] = factors
return factors
})
return out
}
async function opMinute(sdk, job) {
const { symbols = [], period = 5, start, end, concurrency = 6 } = job
const out = {}
await mapPool(symbols, concurrency, async (sym) => {
const opts = { period: String(period) }
if (start) opts.startDate = start
if (end) opts.endDate = end
const bars = await fetchWithRetry(() => sdk.kline.cnMinute(sym, opts))
out[sym] = Array.isArray(bars) ? bars : []
return out[sym]
})
return out
}
async function opRealtime(sdk, job) {
const { concurrency = 8 } = job
const all = await sdk.batch.cn({ concurrency })
const rows = []
for (const q of all || []) {
if (!q || !q.code) continue
rows.push({
symbol: toAppSymbol(q.code, q.marketId),
name: q.name,
last_price: q.price,
prev_close: q.prevClose,
open: q.open,
high: q.high,
low: q.low,
volume: q.volume,
amount: q.amount,
change_pct: q.changePercent,
})
}
return rows
}
async function opInstruments(sdk, job) {
const { concurrency = 8 } = job
const all = await sdk.batch.cn({ concurrency })
const rows = []
for (const q of all || []) {
if (!q || !q.code) continue
const suffix = MARKET_ID_TO_SUFFIX[String(q.marketId)] || guessSuffix(q.code)
// 形状对齐 tickflow 的 Instrument(数值扩展字段放 ext),以复用 instrument_sync 的 flatten。
rows.push({
symbol: toAppSymbol(q.code, q.marketId),
name: q.name,
code: String(q.code),
exchange: suffix,
region: 'CN',
type: 'stock',
ext: {
total_shares: q.totalShares ?? null,
float_shares: q.circulatingShares ?? null,
limit_up: q.limitUp ?? null,
limit_down: q.limitDown ?? null,
},
})
}
return rows
}
async function main() {
let job
try {
const raw = (await readStdin()).trim()
job = raw ? JSON.parse(raw) : {}
} catch (e) {
process.stdout.write(JSON.stringify({ ok: false, error: `invalid job json: ${e.message}` }))
return
}
const op = job.op || 'ping'
try {
const StockSDK = await loadSDK()
if (!StockSDK) throw new Error('stock-sdk 已解析但未导出 StockSDK')
if (op === 'ping') {
process.stdout.write(JSON.stringify({ ok: true, op: 'ping', version: StockSDK.version || 'ok' }))
return
}
const sdk = new StockSDK({ retry: { maxRetries: 3, baseDelay: 400 } })
let rows
switch (op) {
case 'daily':
rows = await opDaily(sdk, job)
break
case 'adj':
rows = await opAdj(sdk, job)
break
case 'minute':
rows = await opMinute(sdk, job)
break
case 'realtime':
rows = await opRealtime(sdk, job)
break
case 'instruments':
rows = await opInstruments(sdk, job)
break
default:
process.stdout.write(JSON.stringify({ ok: false, error: `unknown op: ${op}` }))
return
}
process.stdout.write(JSON.stringify({ ok: true, op, rows }))
} catch (e) {
process.stdout.write(JSON.stringify({ ok: false, op, error: String((e && e.stack) || e) }))
}
}
main()
+90
View File
@@ -0,0 +1,90 @@
"""Python ↔ Node 桥接: 通过 subprocess 调用 bridge.mjs 使用真实 stock-sdk 抓数据。
Original implementation by @forrany (PR #57), migrated to plugin architecture.
后端是 Python, stock-sdk 是 Node/JS 包, 这里用 subprocess 把二者接起来:
每次调用 spawn 一个 `node bridge.mjs`, 从 stdin 喂 JSON job, 从 stdout 读 JSON 结果。
批内并发由 bridge.mjs 内部承担, 一次进程调用摊薄 node 启动开销。
"""
from __future__ import annotations
import json
import logging
import os
import shutil
import subprocess
from pathlib import Path
logger = logging.getLogger(__name__)
_HERE = Path(__file__).resolve().parent
_BRIDGE_MJS = _HERE / "bridge.mjs"
# 默认超时(秒)。全市场(realtime/instruments)与大批量 daily 可能较久, provider 侧按 op 调大。
DEFAULT_TIMEOUT = 120
class StockSDKBridgeError(RuntimeError):
"""桥接调用失败(node 缺失 / stock-sdk 未安装 / 子进程异常 / 结果非法)。"""
def _node_bin() -> str | None:
"""定位 node 可执行文件: 优先环境变量 STOCK_SDK_NODE, 否则 PATH 中的 node。"""
env = os.getenv("STOCK_SDK_NODE")
if env:
return env if (Path(env).exists() or shutil.which(env)) else None
return shutil.which("node")
def run_job(job: dict, timeout: int = DEFAULT_TIMEOUT) -> dict:
"""执行一次桥接 job, 返回解析后的 dict(含 `rows`)。失败抛 StockSDKBridgeError。"""
node = _node_bin()
if not node:
raise StockSDKBridgeError(
"未找到 node 可执行文件。请安装 Node.js(>=18)或设置 STOCK_SDK_NODE 指向 node。"
)
if not _BRIDGE_MJS.exists():
raise StockSDKBridgeError(f"桥接脚本缺失: {_BRIDGE_MJS}")
payload = json.dumps(job, ensure_ascii=False)
try:
proc = subprocess.run(
[node, str(_BRIDGE_MJS)],
input=payload,
capture_output=True,
text=True,
timeout=timeout,
cwd=str(_HERE),
)
except subprocess.TimeoutExpired as e:
raise StockSDKBridgeError(f"stock-sdk 桥接超时(op={job.get('op')}, {timeout}s)") from e
except OSError as e:
raise StockSDKBridgeError(f"启动 node 失败: {e}") from e
if proc.returncode != 0:
tail = (proc.stderr or proc.stdout or "").strip()[-800:]
raise StockSDKBridgeError(f"stock-sdk 桥接非零退出({proc.returncode}): {tail}")
out = (proc.stdout or "").strip()
if not out:
raise StockSDKBridgeError(f"stock-sdk 桥接无输出。stderr: {(proc.stderr or '').strip()[-500:]}")
try:
result = json.loads(out)
except json.JSONDecodeError as e:
raise StockSDKBridgeError(f"stock-sdk 桥接输出非法 JSON: {out[:500]}") from e
if not result.get("ok"):
raise StockSDKBridgeError(f"stock-sdk 桥接返回错误: {result.get('error')}")
return result
def availability() -> tuple[bool, str]:
"""探活: 返回 (是否可用, 原因)。用于 UI 与日志。不抛异常。"""
node = _node_bin()
if not node:
return False, "未找到 node(需 Node.js>=18 或设置 STOCK_SDK_NODE)"
try:
result = run_job({"op": "ping"}, timeout=20)
return True, f"ok (stock-sdk {result.get('version', '?')})"
except StockSDKBridgeError as e:
return False, str(e)
+30
View File
@@ -0,0 +1,30 @@
{
"name": "tickflow-stocksdk-bridge",
"version": "0.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "tickflow-stocksdk-bridge",
"version": "0.1.0",
"dependencies": {
"stock-sdk": "^2"
},
"engines": {
"node": ">=18"
}
},
"node_modules/stock-sdk": {
"version": "2.2.2",
"resolved": "https://mirrors.tencent.com/npm/stock-sdk/-/stock-sdk-2.2.2.tgz",
"integrity": "sha512-nYPIrdd9vm28w2dHXLWBgyXF/bnC3eXFaM7vhxeXBDQJs7imaFR3rWBQUBfhPPxMuhrunnnNDdEPLCq8Uv839Q==",
"license": "ISC",
"bin": {
"stock-sdk": "dist/cli.js"
},
"engines": {
"node": ">=18.0.0"
}
}
}
}
+14
View File
@@ -0,0 +1,14 @@
{
"name": "tickflow-stocksdk-bridge",
"private": true,
"version": "0.1.0",
"description": "Node bridge that exposes stock-sdk to the Python backend as a data provider.",
"type": "module",
"main": "bridge.mjs",
"engines": {
"node": ">=18"
},
"dependencies": {
"stock-sdk": "^2"
}
}
+12
View File
@@ -0,0 +1,12 @@
# stock-sdk 内置可选数据源插件
# 基于 stock-sdk (Node.js) 的免费 A 股行情, 无需 API Key。
# 用户需手动安装依赖后才可用: cd backend/app/plugins/stocksdk && npm install
name: stocksdk
display_name: "stock-sdk(免费行情)"
runtime: node
entry: app.plugins.stocksdk.provider:StockSDKProvider
check: app.plugins.stocksdk.bridge:availability
datasets: [daily, adj_factor, minute, realtime]
description: "基于 stock-sdk 的免费 A 股行情, 无需 API Key。日K/除权/分钟/实时全市场。需运行环境含 Node.js 18+。"
install_hint: "cd backend/app/plugins/stocksdk && npm install"
+251
View File
@@ -0,0 +1,251 @@
"""stock-sdk 内置数据源 provider。
Original implementation by @forrany (PR #57), migrated to plugin architecture.
核心抓取/归一化逻辑保留原作者实现, 仅调整 import 路径与注册方式。
通过 bridge.mjs 调真实 stock-sdk 抓 A 股行情, 归一化到项目内部 schema。
方法签名对齐 custom.GenericHTTPProvider(service 分流点按这套签名调用),
因此注入 custom loader 注册表后, 各 service 无需改动即可路由到本 provider。
"""
from __future__ import annotations
import logging
from dataclasses import dataclass, field
from datetime import datetime
import polars as pl
from app.data_providers.normalizer import normalize_adj_factors, normalize_daily
from app.plugins.stocksdk import bridge
from app.tickflow.rate_limits import chunked
logger = logging.getLogger(__name__)
# stock-sdk 支持的数据集(financial 不支持 → 不声明, 自动回退 tickflow)
_DATASETS = ("daily", "adj_factor", "minute", "realtime")
# 每次桥接调用的符号数。桥接内部按 concurrency 并发, 分批仅为进度反馈与超时控制。
_BATCH = 40
_MINUTE_CANONICAL = ["symbol", "datetime", "open", "high", "low", "close", "volume", "amount"]
@dataclass
class _StockSDKConfig:
"""轻量 config shim, 让 custom loader 的 list_sources/provider_has_dataset 能识别本 provider。"""
name: str = "stocksdk"
display_name: str = "stock-sdk(免费行情)"
datasets: dict = field(default_factory=lambda: dict.fromkeys(_DATASETS))
path: None = None
builtin: bool = True
def _yyyymmdd(dt: datetime | None) -> str | None:
return dt.strftime("%Y%m%d") if dt else None
class StockSDKProvider:
"""内置 stock-sdk 数据源。"""
name = "stocksdk"
builtin = True
def __init__(self) -> None:
self.config = _StockSDKConfig()
def close(self) -> None: # loader.load_all 会对每个 provider 调 close
pass
# ---- daily ----
def get_daily(
self,
symbols: list[str],
start_time: datetime | None,
end_time: datetime | None,
asset_type: str = "stock", # noqa: ARG002
on_chunk_done=None,
) -> pl.DataFrame:
if not symbols:
return pl.DataFrame()
frames: list[pl.DataFrame] = []
chunks = chunked(symbols, _BATCH)
for i, chunk in enumerate(chunks):
job = {
"op": "daily",
"symbols": chunk,
"adjust": "none",
"start": _yyyymmdd(start_time),
"end": _yyyymmdd(end_time),
}
try:
result = bridge.run_job(job, timeout=180)
except bridge.StockSDKBridgeError as e:
logger.warning("stock-sdk daily 拉取失败(%d symbols): %s", len(chunk), e)
result = {"rows": {}}
for sym, rows in (result.get("rows") or {}).items():
if not rows:
continue
df = normalize_daily(rows, default_symbol=sym, source=self.name)
if not df.is_empty():
frames.append(df)
if on_chunk_done:
on_chunk_done(i + 1, len(chunks))
return pl.concat(frames, how="diagonal_relaxed") if frames else pl.DataFrame()
# ---- adj_factor ----
def get_adj_factors(
self,
symbols: list[str],
start_time: datetime | None,
end_time: datetime | None,
asset_type: str = "stock", # noqa: ARG002
on_chunk_done=None,
) -> pl.DataFrame:
if not symbols:
return pl.DataFrame()
frames: list[pl.DataFrame] = []
chunks = chunked(symbols, _BATCH)
for i, chunk in enumerate(chunks):
job = {
"op": "adj",
"symbols": chunk,
"start": _yyyymmdd(start_time),
"end": _yyyymmdd(end_time),
}
try:
result = bridge.run_job(job, timeout=240)
except bridge.StockSDKBridgeError as e:
logger.warning("stock-sdk adj 拉取失败(%d symbols): %s", len(chunk), e)
result = {"rows": {}}
flat: list[dict] = []
for rows in (result.get("rows") or {}).values():
flat.extend(rows or [])
if flat:
df = normalize_adj_factors(flat, source=self.name)
if not df.is_empty():
frames.append(df)
if on_chunk_done:
on_chunk_done(i + 1, len(chunks))
return pl.concat(frames, how="diagonal_relaxed") if frames else pl.DataFrame()
# ---- minute ----
def get_minute(
self,
symbols: list[str],
start_time: datetime | None,
end_time: datetime | None,
asset_type: str = "stock", # noqa: ARG002
on_chunk_done=None,
freq: str = "5m",
) -> pl.DataFrame:
if not symbols:
return pl.DataFrame()
period = "".join(ch for ch in str(freq) if ch.isdigit()) or "5"
frames: list[pl.DataFrame] = []
chunks = chunked(symbols, _BATCH)
for i, chunk in enumerate(chunks):
job = {
"op": "minute",
"symbols": chunk,
"period": period,
"start": _yyyymmdd(start_time),
"end": _yyyymmdd(end_time),
}
try:
result = bridge.run_job(job, timeout=180)
except bridge.StockSDKBridgeError as e:
logger.warning("stock-sdk minute 拉取失败(%d symbols): %s", len(chunk), e)
result = {"rows": {}}
for sym, rows in (result.get("rows") or {}).items():
df = self._minute_df(rows, sym)
if not df.is_empty():
frames.append(df)
if on_chunk_done:
on_chunk_done(i + 1, len(chunks))
return pl.concat(frames, how="diagonal_relaxed") if frames else pl.DataFrame()
@staticmethod
def _minute_df(rows: list[dict], symbol: str) -> pl.DataFrame:
if not rows:
return pl.DataFrame()
df = pl.DataFrame(rows)
# bridge 分钟行含 timestamp(ms, UTC 基准)。A 股分时按北京时间墙钟展示,
# 故转 Asia/Shanghai 后去掉时区得到 naive 北京时间(如 09:35)。
if "timestamp" in df.columns:
df = df.with_columns(
pl.from_epoch(pl.col("timestamp").cast(pl.Int64), time_unit="ms")
.dt.replace_time_zone("UTC")
.dt.convert_time_zone("Asia/Shanghai")
.dt.replace_time_zone(None)
.cast(pl.Datetime("us"))
.alias("datetime")
)
elif "date" in df.columns:
df = df.with_columns(
pl.col("date").str.to_datetime("%Y-%m-%d %H:%M", strict=False).alias("datetime")
)
df = df.with_columns(pl.lit(symbol).alias("symbol"))
for col in ("open", "high", "low", "close", "volume", "amount"):
if col in df.columns:
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
keep = [c for c in _MINUTE_CANONICAL if c in df.columns]
return df.select(keep) if "datetime" in keep else pl.DataFrame()
# ---- realtime (全市场快照) ----
def get_realtime(self) -> list[dict]:
try:
result = bridge.run_job({"op": "realtime"}, timeout=120)
except bridge.StockSDKBridgeError as e:
logger.warning("stock-sdk realtime 拉取失败: %s", e)
return []
return result.get("rows") or []
# ---- instruments (标的维表) ----
def get_instruments(self, asset_type: str = "stock") -> list[dict]:
"""返回 tickflow Instrument 形状的行(symbol/name/code/exchange/region/type + ext),
供 instrument_sync._flatten_instruments 复用同一 flatten 路径, 列结构与 tickflow 一致。
当前覆盖 A 股股票。
"""
if asset_type != "stock":
return []
try:
result = bridge.run_job({"op": "instruments"}, timeout=120)
except bridge.StockSDKBridgeError as e:
logger.warning("stock-sdk instruments 拉取失败: %s", e)
return []
return result.get("rows") or []
# ---- 测试(设置页试拉) ----
def test_dataset(self, dataset: str, symbols: list[str] | None = None) -> dict:
symbols = symbols or ["600519.SH"]
if dataset == "daily":
df = self.get_daily(symbols, None, None)
return _preview("daily", df)
if dataset == "adj_factor":
df = self.get_adj_factors(symbols, None, None)
return _preview("adj_factor", df)
if dataset == "minute":
df = self.get_minute(symbols, None, None)
return _preview("minute", df)
if dataset == "realtime":
rows = self.get_realtime()
head = rows[:5]
return {
"provider": self.name,
"dataset": "realtime",
"rows": len(rows),
"columns": list(head[0].keys()) if head else [],
"preview": head,
}
raise ValueError(f"stock-sdk 不支持数据集: {dataset}")
def _preview(dataset: str, df: pl.DataFrame) -> dict:
return {
"provider": "stocksdk",
"dataset": dataset,
"rows": df.height,
"columns": df.columns,
"preview": df.head(5).to_dicts() if not df.is_empty() else [],
}
+32 -2
View File
@@ -43,14 +43,44 @@ def _flatten_instruments(items: list[dict]) -> list[dict]:
return rows
def _fetch_instruments_via_provider() -> list[dict] | None:
"""若当前日K数据源不是 tickflow 且该 provider 提供 get_instruments, 用它拉标的维表。
返回 flatten 行列表; 未命中(仍应走 tickflow)时返回 None。
标的维表跟随日K数据源(二者天然耦合, 无独立偏好项)。
"""
from app.services import preferences
provider_name = preferences.get_daily_data_provider()
if provider_name == "tickflow":
return None
from app.data_providers import custom as custom_sources
if not custom_sources.is_custom_provider(provider_name):
return None
provider = custom_sources.get_provider(provider_name)
if not hasattr(provider, "get_instruments"):
return None
try:
items = provider.get_instruments("stock") or []
except Exception as e: # noqa: BLE001
logger.warning("provider %s get_instruments 失败: %s", provider_name, e)
return None
rows = _flatten_instruments(items)
logger.info("instruments via %s: %d stocks", provider_name, len(rows))
return rows
def sync_instruments(data_dir: Path) -> int:
"""全量同步标的维表 → data/instruments/instruments.parquet。
返回写入的行数。
"""
all_rows = _fetch_instruments_via_provider()
if all_rows is None:
# 未命中非 tickflow provider → 走 tickflow 直连
tf = get_client()
all_rows: list[dict] = []
all_rows = []
for ex in _EXCHANGES:
try:
items = tf.exchanges.get_instruments(ex, instrument_type="stock")
+170
View File
@@ -0,0 +1,170 @@
"""StockSDKProvider 归一化与桥接契约测试。
不依赖真实 node / 网络: mock bridge.run_job 返回样例 payload, 只验证 Python 侧的
归一化、除权因子合成对齐、符号回显、空结果处理与注册接线。
"""
from __future__ import annotations
import datetime as dt
import polars as pl
from app.plugins.stocksdk import provider as sp
from app.plugins.stocksdk.provider import StockSDKProvider
def _patch_run_job(monkeypatch, mapping):
"""mapping: op -> payload dict(将作为 run_job 返回值)。"""
def fake(job, timeout=None): # noqa: ARG001
return mapping[job["op"]]
monkeypatch.setattr(sp.bridge, "run_job", fake)
def test_get_daily_normalizes_and_echoes_symbol(monkeypatch):
_patch_run_job(monkeypatch, {
"daily": {"ok": True, "op": "daily", "rows": {
"600519.SH": [
{"date": "2026-01-05", "open": 1385.0, "high": 1431.9, "low": 1385.0,
"close": 1426.0, "volume": 70949, "amount": 1.0e10, "code": "600519"},
{"date": "2026-01-06", "open": 1432.5, "high": 1437.0, "low": 1416.5,
"close": 1428.0, "volume": 39586, "amount": 5.6e9, "code": "600519"},
],
}},
})
df = StockSDKProvider().get_daily(["600519.SH"], dt.datetime(2026, 1, 1), dt.datetime(2026, 1, 15))
assert df.columns == ["symbol", "date", "open", "high", "low", "close", "volume", "amount"]
assert df.height == 2
assert df["symbol"].unique().to_list() == ["600519.SH"]
assert df.schema["date"] == pl.Date
assert df.schema["close"] == pl.Float64
def test_get_adj_factors_from_bridge_ratio(monkeypatch):
# 桥接内部已算好 ex_factor = close_hfq/close_none, 这里验证 Python 侧归一化。
_patch_run_job(monkeypatch, {
"adj": {"ok": True, "op": "adj", "rows": {
"600519.SH": [
{"symbol": "600519.SH", "trade_date": "2020-01-02", "ex_factor": 5.29},
{"symbol": "600519.SH", "trade_date": "2020-01-03", "ex_factor": 5.30},
],
}},
})
df = StockSDKProvider().get_adj_factors(["600519.SH"], None, None)
assert df.columns == ["symbol", "trade_date", "ex_factor"]
assert df.height == 2
assert df.schema["trade_date"] == pl.Date
assert abs(df["ex_factor"][0] - 5.29) < 1e-9
def test_get_minute_datetime_is_beijing_wall_clock(monkeypatch):
# timestamp 1779327300000 = 2026-05-21 01:35 UTC = 09:35 Asia/Shanghai
_patch_run_job(monkeypatch, {
"minute": {"ok": True, "op": "minute", "rows": {
"600519.SH": [
{"date": "2026-05-21 09:35", "open": 1284.9, "high": 1289.1, "low": 1283.9,
"close": 1286.7, "volume": 2740, "amount": 3.6e8, "timestamp": 1779327300000},
],
}},
})
df = StockSDKProvider().get_minute(["600519.SH"], None, None)
assert set(df.columns) == {"symbol", "datetime", "open", "high", "low", "close", "volume", "amount"}
assert df.height == 1
ts = df["datetime"][0]
assert (ts.hour, ts.minute) == (9, 35)
assert df["symbol"][0] == "600519.SH"
def test_get_realtime_passthrough(monkeypatch):
rows = [{"symbol": "600519.SH", "name": "贵州茅台", "last_price": 1200.0,
"prev_close": 1194.0, "open": 1186.0, "high": 1203.0, "low": 1180.0, "volume": 16325}]
_patch_run_job(monkeypatch, {"realtime": {"ok": True, "op": "realtime", "rows": rows}})
out = StockSDKProvider().get_realtime()
assert out == rows
required = {"symbol", "last_price", "prev_close", "open", "high", "low", "volume"}
assert required <= set(out[0].keys())
def test_get_instruments_flatten_compatible(monkeypatch):
rows = [{"symbol": "600519.SH", "name": "贵州茅台", "code": "600519", "exchange": "SH",
"region": "CN", "type": "stock", "total_shares": 1, "float_shares": 1,
"limit_up": 1.0, "limit_down": 1.0}]
_patch_run_job(monkeypatch, {"instruments": {"ok": True, "op": "instruments", "rows": rows}})
out = StockSDKProvider().get_instruments("stock")
assert out[0]["symbol"] == "600519.SH"
assert out[0]["exchange"] == "SH"
# 非 stock 资产暂不覆盖
assert StockSDKProvider().get_instruments("etf") == []
def test_empty_symbols_returns_empty():
p = StockSDKProvider()
assert p.get_daily([], None, None).is_empty()
assert p.get_adj_factors([], None, None).is_empty()
assert p.get_minute([], None, None).is_empty()
def test_bridge_error_degrades_to_empty(monkeypatch):
def boom(job, timeout=None): # noqa: ARG001
raise sp.bridge.StockSDKBridgeError("node missing")
monkeypatch.setattr(sp.bridge, "run_job", boom)
assert StockSDKProvider().get_daily(["600519.SH"], None, None).is_empty()
assert StockSDKProvider().get_realtime() == []
assert StockSDKProvider().get_instruments("stock") == []
def test_plugin_discovered_in_loader():
"""插件被发现并记录状态 (即使依赖没装, 不可用)。"""
from app.data_providers import custom as cs
plugins = {p["name"]: p for p in cs.list_plugins()}
assert "stocksdk" in plugins
assert plugins["stocksdk"]["runtime"] == "node"
assert "daily" in plugins["stocksdk"]["datasets"]
assert "realtime" in plugins["stocksdk"]["datasets"]
assert "financial" not in plugins["stocksdk"]["datasets"]
assert cs.is_builtin("stocksdk")
# 内置源不出现在用户自定义源列表
assert "stocksdk" not in [s["name"] for s in cs.list_sources()]
def test_plugin_registered_when_available(monkeypatch):
"""依赖可用时, 插件注册进 _PROVIDERS 并可路由。"""
from app.data_providers import custom as cs
from app.data_providers.custom import loader as L
# mock availability 返回 (True, "ok")
monkeypatch.setattr(L, "_call_check", lambda ref: (True, "ok"))
monkeypatch.setattr(L, "_load_entry", _load_stocksdk_entry)
L._load_builtin_plugins()
assert "stocksdk" in cs.names()
assert cs.is_custom_provider("stocksdk")
assert cs.provider_has_dataset("stocksdk", "daily")
assert cs.provider_has_dataset("stocksdk", "realtime")
assert not cs.provider_has_dataset("stocksdk", "financial")
def _load_stocksdk_entry(entry_ref: str):
"""测试用: 无条件加载 stocksdk provider 类 (跳过 check)。"""
if "StockSDKProvider" in entry_ref:
from app.plugins.stocksdk.provider import StockSDKProvider
return StockSDKProvider
if "availability" in entry_ref:
from app.plugins.stocksdk.bridge import availability
return availability
raise ValueError(f"unknown entry: {entry_ref}")
def test_builtin_not_editable():
from app.data_providers import custom as cs
assert cs.get_config_dict("stocksdk") is None
for fn in (lambda: cs.save_config("stocksdk", {}), lambda: cs.delete_config("stocksdk")):
try:
fn()
raise AssertionError("expected ValueError for builtin")
except ValueError:
pass
+1834 -1834
View File
File diff suppressed because it is too large Load Diff
+118
View File
@@ -0,0 +1,118 @@
# 数据源插件开发指南
数据源插件是可选的行情数据来源(stock-sdk、akshare 等),作为独立模块放在
`backend/app/plugins/` 下。用户**手动安装依赖**后才可用;不安装完全不影响主功能。
## 快速上手
一个插件 = 一个目录 + 一个 `plugin.yaml` 清单:
```
backend/app/plugins/<your_plugin>/
├── plugin.yaml # 清单(必需)
├── provider.py # Provider 实现(必需)
├── ... # 桥接/依赖文件(按需)
```
### plugin.yaml 字段
```yaml
name: my_source # 唯一标识, 只允许 [a-z0-9_], 也是 provider name
display_name: "我的数据源" # 设置页显示名
runtime: python # 运行时类型: node | python | none
entry: app.plugins.my_source.provider:MyProvider # provider 类的导入路径
check: app.plugins.my_source.bridge:availability # 可用性检测函数(可选)
datasets: [daily, adj_factor, minute, realtime] # 支持的数据集
description: "数据源描述"
install_hint: "pip install xxx" # 未装依赖时显示的安装提示
```
### runtime 字段说明
| runtime | 含义 | 典型场景 |
|---|---|---|
| `python` | 纯 Python 依赖, `pip install` | akshare、tushare |
| `node` | 需要 Node.js 运行时, `npm install` | stock-sdk |
| `none` | 无额外依赖 | 纯 HTTP API 源 |
`runtime` 字段当前仅用于 UI 展示, 实际依赖检测由 `check` 函数负责。
### check 函数
插件自己负责检测依赖是否已安装。后端启动时会调用此函数:
```python
# app/plugins/my_source/bridge.py
def availability() -> tuple[bool, str]:
"""返回 (是否可用, 原因)。不抛异常。"""
try:
import akshare # noqa: F401
return True, "ok"
except ImportError:
return False, "未安装 akshare, 运行: pip install akshare"
```
- **可用** → 插件注册进路由表, 设置页可切换
- **不可用** → 设置页显示插件卡片但灰显, 展示 `install_hint`
## Provider 接口契约
Provider 是一个普通 Python 类(无需继承基类), 实现以下方法签名。方法签名对齐
`GenericHTTPProvider`, 这样 services 层(kline_sync / quote_service 等)的路由逻辑
零改动即可路由到插件。
```python
class MyProvider:
name = "my_source"
builtin = True # 标记为内置(不可被用户编辑/删除)
def __init__(self):
self.config = MyConfig() # 需有 .datasets 属性(dict, key 是数据集名)
def close(self) -> None:
"""清理资源(load_all 重建注册表时会调)。"""
def get_daily(self, symbols, start_time, end_time, asset_type="stock", on_chunk_done=None) -> pl.DataFrame:
"""日K: 返回 schema [symbol, date, open, high, low, close, volume, amount]"""
def get_adj_factors(self, symbols, start_time, end_time, asset_type="stock", on_chunk_done=None) -> pl.DataFrame:
"""除权因子: 返回 schema [symbol, trade_date, ex_factor]"""
def get_minute(self, symbols, start_time, end_time, asset_type="stock", on_chunk_done=None, freq="1m") -> pl.DataFrame:
"""分钟K: 返回 schema [symbol, datetime, open, high, low, close, volume, amount]"""
def get_realtime(self) -> list[dict]:
"""全市场实时快照: 返回 list[dict], 每行含 symbol/last_price/prev_close/open/high/low/volume"""
def get_instruments(self, asset_type="stock") -> list[dict]:
"""标的维表(可选): 返回 tickflow Instrument 形状的行, 供 instrument_sync 复用 flatten"""
```
### config.datasets 的作用
`provider_has_dataset(name, dataset)` 通过 `dataset in provider.config.datasets` 判断。
这是 services 层路由的关键: 用户在设置页选了插件, 但某数据集未声明时, 该数据集
自动回退 TickFlow。
```python
class MyConfig:
datasets = {"daily": ..., "realtime": ...} # key 是数据集名, value 任意
```
## 现有插件参考
- **`backend/app/plugins/stocksdk/`** — Node 型插件, 通过 subprocess 桥接调用 stock-sdk
- `bridge.py` — Python↔Node 桥接 + availability 检测
- `bridge.mjs` — Node 端(并发池、重试、SDK 解析)
- `provider.py` — Provider 实现(归一化、分批、错误降级)
## 路由机制(无需关心, 仅参考)
后端启动时, `loader.py``_load_builtin_plugins()` 扫描 `plugins/` 目录:
1. 读每个子目录的 `plugin.yaml`
2.`check` 函数检测可用性
3. 可用 → 动态 import `entry` 指向的 Provider 类 → 注册进 `_PROVIDERS`
4. 不可用 → 记录状态, 设置页显示但不可切换
注册后, 插件和用户 YAML 自定义源走**完全相同的路由路径**(services 层的
`provider_has_dataset` / `get_provider` 调用), 无需额外集成代码。
+27
View File
@@ -693,6 +693,18 @@ export interface DataSourceItem {
path?: string | null
}
/** 内置可选插件数据源 (plugins/ 目录, 需手动装依赖) */
export interface PluginDataSourceItem {
name: string
display_name: string
datasets: string[]
runtime: string // node | python | none
available: boolean // 依赖是否已安装
status: string // 可用性原因 (供 UI 显示)
description: string
install_hint: string // 未装依赖时显示的安装命令
}
export interface DataSourceLoadError {
name?: string
path: string
@@ -701,6 +713,7 @@ export interface DataSourceLoadError {
export interface DataSourcesResponse {
builtin: DataSourceItem[]
plugins: PluginDataSourceItem[]
custom: DataSourceItem[]
errors: DataSourceLoadError[]
config_dir: string
@@ -857,6 +870,20 @@ export const api = {
deleteDataSource: (name: string) =>
request<DataSourcesResponse>(`/api/settings/data-sources/${encodeURIComponent(name)}`, { method: 'DELETE' }),
reloadDataSources: () => request<DataSourcesResponse>('/api/settings/data-sources/reload', { method: 'POST' }),
installPlugin: (name: string) => {
// npm install 可能耗时较长, 用 6 分钟超时
const controller = new AbortController()
const timer = setTimeout(() => controller.abort(), 360_000)
return request<DataSourcesResponse & { install_ok: boolean; install_message: string }>(
`/api/settings/plugins/${encodeURIComponent(name)}/install`,
{ method: 'POST', signal: controller.signal },
).finally(() => clearTimeout(timer))
},
uninstallPlugin: (name: string) =>
request<DataSourcesResponse & { uninstall_ok: boolean; uninstall_message: string }>(
`/api/settings/plugins/${encodeURIComponent(name)}/install`,
{ method: 'DELETE' },
),
testDataSource: (provider: string, dataset: string, symbols?: string[]) =>
request<DataSourceTestResult>('/api/settings/data-sources/test', {
method: 'POST',
+153 -12
View File
@@ -2,7 +2,7 @@ import { useState } from 'react'
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
import { motion, AnimatePresence } from 'framer-motion'
import { Check, Database, Plus, RefreshCw, Zap, FileWarning } from 'lucide-react'
import { api, type DataSourceItem } from '@/lib/api'
import { api, type DataSourceItem, type PluginDataSourceItem } from '@/lib/api'
import { QK } from '@/lib/queryKeys'
import { usePreferences } from '@/lib/useSharedQueries'
import { toast } from '@/components/Toast'
@@ -70,14 +70,50 @@ export function SettingsDataSourcesPanel() {
onSuccess: (_data, name) => setSelected(name),
})
const installMut = useMutation({
mutationFn: (name: string) => api.installPlugin(name),
onSuccess: (data) => {
qc.invalidateQueries({ queryKey: QK.dataSources })
if (data.install_ok) {
toast('插件依赖安装成功', 'success')
} else {
toast(data.install_message || '安装失败', 'error')
}
},
onError: (e: Error) => toast(`安装失败: ${e.message}`, 'error'),
})
const uninstallMut = useMutation({
mutationFn: (name: string) => api.uninstallPlugin(name),
onSuccess: (data) => {
qc.invalidateQueries({ queryKey: QK.dataSources })
qc.invalidateQueries({ queryKey: QK.preferences })
if (data.uninstall_ok) {
toast(data.uninstall_message || '已卸载', 'success')
} else {
toast(data.uninstall_message || '卸载失败', 'error')
}
},
onError: (e: Error) => toast(`卸载失败: ${e.message}`, 'error'),
})
const builtin: DataSourceItem[] = sources.data?.builtin ?? []
const pluginList: PluginDataSourceItem[] = sources.data?.plugins ?? []
const customList: DataSourceItem[] = sources.data?.custom ?? []
const errors = sources.data?.errors ?? []
const activeName = prefs.data?.daily_data_provider || 'tickflow'
// 顶部数据源选择列表 (内置 + 自定义 + 新增)
// 插件 name → 状态 (供卡片渲染时判断 available/installing 等)
const pluginMap = new Map(pluginList.map(p => [p.name, p]))
const pluginNames = new Set(pluginList.map(p => p.name))
// 顶部数据源选择列表 (内置 + 所有插件 + 自定义 + 新增)
const pluginItems: DataSourceItem[] = pluginList.map(p => ({
name: p.name, display_name: p.display_name, datasets: p.datasets,
}))
const allItems = [
...builtin,
...pluginItems,
...customList,
]
@@ -91,7 +127,10 @@ export function SettingsDataSourcesPanel() {
<div className="flex items-center gap-2.5">
<Database className="h-4 w-4 text-secondary" />
<h2 className="text-sm font-medium text-foreground"></h2>
<span className="text-[10px] text-muted/40 font-mono truncate max-w-[280px]" title={sources.data?.config_dir}>
<span
className="text-[10px] text-muted/40 font-mono truncate hidden lg:inline max-w-[480px]"
title={sources.data?.config_dir}
>
{sources.data?.config_dir}
</span>
</div>
@@ -119,33 +158,84 @@ export function SettingsDataSourcesPanel() {
{allItems.map(item => {
const isActive = activeName === item.name
const isSelected = selected === item.name
const plugin = pluginMap.get(item.name)
const pluginUnavailable = plugin && !plugin.available
const installing = installMut.isPending && installMut.variables === item.name
const uninstalling = uninstallMut.isPending && uninstallMut.variables === item.name
return (
<div
key={item.name}
onClick={() => {
if (pluginUnavailable) return // 未安装的插件不可选中
setSelected(item.name)
if (item.name !== 'tickflow') {
// 只有用户自定义源 (YAML) 才进编辑器; tickflow 和插件不可编辑
if (customList.some(c => c.name === item.name)) {
editExisting.mutate(item.name)
}
}}
className={`relative cursor-pointer text-left rounded-lg border px-3.5 py-3 transition-all ${
isSelected
? 'border-accent/50 bg-accent/5 ring-1 ring-accent/20'
: 'border-border/60 bg-elevated/20 hover:bg-elevated/40'
className={`relative text-left rounded-lg border px-3.5 py-3 transition-all ${
pluginUnavailable
? 'border-border/40 bg-elevated/10 opacity-70'
: isSelected
? 'border-accent/50 bg-accent/5 ring-1 ring-accent/20 cursor-pointer'
: 'border-border/60 bg-elevated/20 hover:bg-elevated/40 cursor-pointer'
}`}
>
<div className="flex items-center gap-2 mb-1">
<span className={`h-1.5 w-1.5 rounded-full shrink-0 ${isActive ? 'bg-accent' : 'bg-transparent border border-muted/40'}`} />
<span className={`h-1.5 w-1.5 rounded-full shrink-0 ${
pluginUnavailable ? 'bg-muted/30' : isActive ? 'bg-accent' : 'bg-transparent border border-muted/40'
}`} />
<span className={`text-sm truncate flex-1 ${isActive ? 'font-medium text-foreground' : 'text-secondary'}`}>
{item.display_name}
</span>
{item.name === 'tickflow' && (
<span className="text-[9px] text-muted/50 uppercase tracking-wider shrink-0"></span>
)}
{isActive ? (
{pluginNames.has(item.name) && (
<span className="text-[9px] text-muted/50 uppercase tracking-wider shrink-0"></span>
)}
{/* 右侧操作区: 插件未安装→安装按钮; 已激活→使用中; 否则→使用/卸载 */}
{pluginUnavailable ? (
installing ? (
<span className="inline-flex items-center gap-1 text-[9px] text-accent shrink-0">
<RefreshCw className="h-2.5 w-2.5 animate-spin" /> ...
</span>
) : (
<button
onClick={(e) => { e.stopPropagation(); installMut.mutate(item.name) }}
disabled={installMut.isPending}
className="shrink-0 inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[10px] font-medium bg-accent/10 text-accent hover:bg-accent/20 transition-colors disabled:opacity-50"
>
<Zap className="h-2.5 w-2.5" />
</button>
)
) : isActive ? (
<span className="inline-flex items-center gap-0.5 text-[9px] text-accent shrink-0">
<Check className="h-2.5 w-2.5" /> 使
</span>
) : plugin ? (
/* 已安装插件: 使用 + 卸载 */
<div className="flex items-center gap-1 shrink-0">
<button
onClick={(e) => { e.stopPropagation(); switchProvider.mutate(item.name) }}
disabled={switchProvider.isPending}
className="rounded px-1.5 py-0.5 text-[10px] font-medium bg-accent/10 text-accent hover:bg-accent/20 transition-colors disabled:opacity-50"
>
使
</button>
{uninstalling ? (
<RefreshCw className="h-2.5 w-2.5 animate-spin text-muted" />
) : (
<button
onClick={(e) => { e.stopPropagation(); uninstallMut.mutate(item.name) }}
disabled={uninstallMut.isPending}
className="text-[10px] text-muted/50 hover:text-danger transition-colors disabled:opacity-40"
title="卸载依赖"
>
</button>
)}
</div>
) : (
<button
onClick={(e) => { e.stopPropagation(); switchProvider.mutate(item.name) }}
@@ -168,6 +258,10 @@ export function SettingsDataSourcesPanel() {
{item.name === 'tickflow' && (
<div className="text-[10px] text-muted/60 ml-3.5">K · · · K</div>
)}
{/* 未安装插件显示安装命令提示 */}
{pluginUnavailable && plugin?.install_hint && (
<div className="ml-3.5 mt-1 text-[10px] text-muted/40 font-mono truncate">{plugin.install_hint}</div>
)}
</div>
)
})}
@@ -223,7 +317,7 @@ export function SettingsDataSourcesPanel() {
onSwitch={() => switchProvider.mutate('tickflow')}
switching={switchProvider.isPending}
/>
) : (
) : selected === '__new__' || customList.some(c => c.name === selected) ? (
<DataSourceEditor
key={selected}
initial={null}
@@ -241,7 +335,15 @@ export function SettingsDataSourcesPanel() {
onActivate={(name) => switchProvider.mutate(name)}
onDelete={selected !== '__new__' && selectedCustom ? () => setConfirmDelete(selected) : undefined}
/>
)}
) : pluginList.find(x => x.name === selected) ? (
/* 选中插件: 显示只读详情, 不进编辑器 */
<PluginDetail
plugin={pluginList.find(x => x.name === selected)!}
isActive={activeName === selected}
onSwitch={() => switchProvider.mutate(selected)}
switching={switchProvider.isPending}
/>
) : null}
</motion.div>
</AnimatePresence>
@@ -279,6 +381,45 @@ export function SettingsDataSourcesPanel() {
)
}
function PluginDetail({ plugin, isActive, onSwitch, switching }: {
plugin: PluginDataSourceItem
isActive: boolean
onSwitch: () => void
switching: boolean
}) {
return (
<section className="rounded-card border border-border bg-surface p-6">
<div className="flex items-start gap-4 mb-5">
<div className="h-11 w-11 rounded-xl bg-accent/10 flex items-center justify-center shrink-0">
<Zap className="h-5 w-5 text-accent" />
</div>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2 mb-1">
<h3 className="text-base font-semibold text-foreground">{plugin.display_name}</h3>
<span className="text-[10px] text-muted/50 uppercase tracking-wider"> · {plugin.runtime}</span>
</div>
{plugin.description && <p className="text-xs text-secondary leading-relaxed">{plugin.description}</p>}
</div>
</div>
<div className="flex items-center gap-3">
{isActive ? (
<span className="inline-flex items-center gap-1.5 text-xs text-accent">
<Check className="h-3.5 w-3.5" /> 使
</span>
) : (
<button
onClick={onSwitch}
disabled={switching}
className="px-3 py-1.5 rounded-btn bg-accent/10 text-accent hover:bg-accent/20 text-xs font-medium transition-colors disabled:opacity-50"
>
</button>
)}
</div>
</section>
)
}
function TickFlowDetail({ active, onSwitch, switching }: { active: boolean; onSwitch: () => void; switching: boolean }) {
return (
<section className="rounded-card border border-border bg-surface p-6">