feat: warehouse sync 支持 --source auto/tdx/baostock + EXE 打包内置 baostock 兜底源 + README 使用说明

This commit is contained in:
GitHub
2026-09-04 19:40:28 +08:00
parent 01e8fc0cc1
commit 2359df8694
8 changed files with 240 additions and 7 deletions
+3 -2
View File
@@ -39,10 +39,11 @@ jobs:
npm run build
working-directory: web-ui
# 安装后端(含 [web,packaging] extrasfastapi/uvicorn + pystray/Pillow
# 安装后端(含 [web,packaging,baostock] extrasfastapi/uvicorn +
# pystray/Pillow + baostock 自动兜底数据源——EXE 内置全量灾备链)
- name: Install Python deps
run: |
pip install -e ".[web,packaging]"
pip install -e ".[web,packaging,baostock]"
pip install pyinstaller
- name: Build EXE
+10
View File
@@ -156,6 +156,16 @@ pip install -e ".[dev]"
pip install -e ".[web]"
```
### 可选:baostock 自动兜底数据源
通达信协议依赖第三方行情服务器,为了"部分兜底、总好过全挂",可选安装 baostock 作为最后一级自动回退:
```bash
pip install "easy-tdx[baostock]"
```
装完即自动生效,平时**一次都不会调用**——只有当 MAC 协议与标准协议两条 TDX 路径全部失败或返回空时才启用,响应中会带 `source: "baostock"` 字段标注数据来源。设置环境变量 `EASY_TDX_BAOSTOCK=0` 可随时关闭。边界:baostock 是收盘后更新的数据源(当日数据约 17:30 后才有),因此只兜 **日/周/月 K 线** 的历史数据(覆盖沪深,不含北交所),实时行情、分时、板块等能力仍由通达信协议提供。本地 K 线仓库同步同样支持:`easy-tdx warehouse sync --symbols SH:600519 --source auto|tdx|baostock`(默认 auto)。
## CLI 参考
`easy-tdx` 默认输出 JSON(一行一条记录),`--table` 切换表格,`--output csv` 输出 CSV。
+3 -1
View File
@@ -53,7 +53,9 @@
- Windows 10/11PyInstaller 不支持跨平台编译)
- Python 3.10+
- Node.js 20+
- 项目已 `pip install -e ".[web]"` 安装到当前环境
- 项目已 `pip install -e ".[web,packaging,baostock]"` 安装到当前环境
`baostock` 是 /bars 与 warehouse 的自动兜底数据源,EXE 需一并打进包;`packaging`
extras 提供 EXE 托盘所需的 pystray/Pillow
### 步骤
+3
View File
@@ -24,6 +24,9 @@ from PyInstaller.utils.hooks import collect_data_files, collect_submodules
hiddenimports: list[str] = []
hiddenimports += collect_submodules("uvicorn")
hiddenimports += collect_submodules("easy_tdx")
# baostock 兜底数据源(v1.33/bars 与 warehouse 的最后一级回退)在
# sources/baostock.py 里经 importlib 懒加载,静态分析扫不到,需显式声明。
hiddenimports += collect_submodules("baostock")
# pandas / numpy / scipy 由 PyInstaller 自带 hook 处理(见
# PyInstaller/hooks/hook-pandas.* 等),无需手动 collect_submodules——
# 手动全量收集会把 numpy.typing.tests / pandas._numba.kernels 等可选/测试
+31 -3
View File
@@ -6,6 +6,7 @@
示例::
easy-tdx warehouse sync --symbols SH:600519,SZ:000001
easy-tdx warehouse sync --symbols SH:600519 --source baostock # 仅 baostock 源
easy-tdx warehouse query SH 600519 --count 30
easy-tdx warehouse stats
easy-tdx warehouse check --symbols SH:600519
@@ -48,6 +49,16 @@ def warehouse() -> None:
type=click.Choice(["NONE", "QFQ", "HFQ"]),
help="复权口径(默认 QFQ",
)
@click.option(
"--source",
"source",
default="auto",
type=click.Choice(["auto", "tdx", "baostock"]),
help=(
"数据源(默认 auto):auto=TDX 失败/为空自动转 baostock 兜底(需 "
"easy-tdx[baostock]/ tdx=仅通达信 / baostock=仅 baostock"
),
)
@click.option(
"--db", "db_path", default=None, help="仓库文件路径(默认 ~/.easy_tdx/warehouse.duckdb"
)
@@ -57,6 +68,7 @@ def warehouse_sync(
max_bars: int,
tail_bars: int,
adjust: str,
source: str,
db_path: str | None,
) -> None:
"""增量同步行情进仓库(首同步全量、此后只补尾部)。"""
@@ -81,13 +93,29 @@ def warehouse_sync(
def _progress(done: int, total: int, sym: str) -> None:
click.echo(f"[{done}/{total}] {sym}", err=True)
from ..cli.conn import get_mac_client
if source == "baostock":
from easy_tdx.sources.baostock import BaostockClient
with get_mac_client() as client:
kline_client: Any = BaostockClient()
syncer = WarehouseSyncer(
client, wh, max_bars=max_bars, tail_bars=tail_bars, adjust=adjust
kline_client, wh, max_bars=max_bars, tail_bars=tail_bars, adjust=adjust
)
summary = syncer.sync(symbol_list, period=period, progress=_progress)
else:
from ..cli.conn import get_mac_client
with get_mac_client() as client:
if source == "auto":
from easy_tdx.sources import AutoKlineClient
from easy_tdx.sources.baostock import BaostockClient
kline_client = AutoKlineClient(client, BaostockClient())
else:
kline_client = client
syncer = WarehouseSyncer(
kline_client, wh, max_bars=max_bars, tail_bars=tail_bars, adjust=adjust
)
summary = syncer.sync(symbol_list, period=period, progress=_progress)
click.echo(
json.dumps(
{k: v for k, v in summary.items() if k != "details"},
+42 -1
View File
@@ -1 +1,42 @@
"""备选数据源(自动兜底)。"""
"""备选数据源(自动兜底)。
- :mod:`easy_tdx.sources.baostock`baostock EOD 兜底源(TDX 全部路径失败时
的最后一级回退,仅日线及以上)。
- :class:`AutoKlineClient`TDX 优先、备选源兜底的组合客户端,供
``WarehouseSyncer`` 等只认 ``get_stock_kline`` 协议的组件使用。
"""
from __future__ import annotations
from typing import Any
import pandas as pd
__all__ = ["AutoKlineClient"]
class AutoKlineClient:
"""TDX 优先、备选源兜底的组合 K 线客户端。
满足 ``get_stock_kline(market:int, code, **kwargs)`` 协议:primary 出错
**或返回空**时自动转 fallback;fallback 的结果(或异常)直接透传——
异常信息通常带安装提示(如 baostock 未安装),便于上层定位。
Example::
client = AutoKlineClient(mac_client, BaostockClient())
syncer = WarehouseSyncer(client, warehouse)
"""
def __init__(self, primary: Any, fallback: Any) -> None:
self._primary = primary
self._fallback = fallback
def get_stock_kline(self, market: int, code: str, **kwargs: Any) -> pd.DataFrame:
try:
df = self._primary.get_stock_kline(market, code, **kwargs)
if df is not None and len(df) > 0:
return df
except Exception: # noqa: BLE001 — 主源失败是兜底触发的正常路径
pass
return self._fallback.get_stock_kline(market, code, **kwargs)
+45
View File
@@ -169,3 +169,48 @@ def fetch_bars(
return None
return df[["date", "open", "close", "high", "low", "vol", "amount"]].reset_index(drop=True)
# ---------------------------------------------------------------------------
# Warehouse 适配:满足 WarehouseSyncer 客户端协议(get_stock_kline
# ---------------------------------------------------------------------------
# TDX Market 枚举值 → baostock 市场前缀(0=SZ 1=SH2=BJ 不覆盖)
_TDX_TO_MARKET_STR = {0: "SZ", 1: "SH"}
# 仓库周期名(Period 名)→ 本模块 category 键
_PERIOD_TO_CATEGORY = {"DAILY": "DAY", "WEEKLY": "WEEK", "MONTHLY": "MONTH"}
class BaostockClient:
"""把 baostock 包装成 ``WarehouseSyncer`` 可直接使用的行情客户端。
只需实现 ``get_stock_kline(market:int, code, period=, start=, count=,
adjust=)`` 签名(``MacClient`` 同款)。语义差异:
- 数据缺失(未上市 / 超出范围)返回**空 DataFrame**——上层按"无数据"处理;
- baostock 未安装 / 已禁用 / 查询失败抛 ``RuntimeError``——错误信息带
安装提示,便于 ``--source baostock`` 显式使用时定位。
"""
def get_stock_kline(
self,
market: int,
code: str,
period: str = "DAILY",
start: int = 0,
count: int = 8000,
adjust: str = "QFQ",
) -> pd.DataFrame:
market_str = _TDX_TO_MARKET_STR.get(int(market))
category = _PERIOD_TO_CATEGORY.get(str(period).upper())
if market_str is None or category is None:
# 不覆盖的市场/周期:按"无数据"处理而非报错(调用方可跳过)
return pd.DataFrame()
if not is_enabled():
raise RuntimeError(
"baostock 未安装或已禁用(EASY_TDX_BAOSTOCK=0)。"
"安装: pip install easy-tdx[baostock]"
)
df = fetch_bars(market_str, code, category, start, count, adjust)
if df is None or df.empty:
return pd.DataFrame()
return df.rename(columns={"date": "datetime"})
+103
View File
@@ -282,3 +282,106 @@ def test_index_endpoint_falls_back_to_baostock(fake_bs):
body = resp.json()
assert body["source"] == "baostock"
assert fake_bs["code"] == "sh.000001"
# ---------------------------------------------------------------------------
# Warehouse 适配(BaostockClient / AutoKlineClient
# ---------------------------------------------------------------------------
def test_baostock_client_maps_and_returns_datetime(fake_bs):
"""适配器满足 WarehouseSyncer 协议:market/period 数字与名称映射正确,
输出 datetime 列(仓库 schema)。"""
from easy_tdx.sources.baostock import BaostockClient
df = BaostockClient().get_stock_kline(
1, "600519", period="DAILY", start=0, count=5, adjust="QFQ"
)
assert len(df) == 5
assert "datetime" in df.columns
assert fake_bs["code"] == "sh.600519"
assert fake_bs["frequency"] == "d"
def test_baostock_client_unsupported_market_returns_empty(fake_bs):
"""BJ(market=2)等不覆盖范围:返回空表(上层按无数据跳过),不报错。"""
from easy_tdx.sources.baostock import BaostockClient
df = BaostockClient().get_stock_kline(2, "430047", period="DAILY")
assert len(df) == 0
assert "calls" not in fake_bs
def test_baostock_client_no_data_returns_empty_not_raise(fake_bs):
"""无数据(如超出上市范围)返回空表而非异常。"""
from easy_tdx.sources.baostock import BaostockClient
_install_fake_bs([], fake_bs)
df = BaostockClient().get_stock_kline(0, "000001", period="DAILY")
assert len(df) == 0
def test_baostock_client_not_installed_raises_with_hint(monkeypatch: pytest.MonkeyPatch):
"""显式 --source baostock 但未安装:报错且信息带安装提示。"""
monkeypatch.delenv("EASY_TDX_BAOSTOCK", raising=False)
monkeypatch.setitem(sys.modules, "baostock", None)
from easy_tdx.sources.baostock import BaostockClient
with pytest.raises(RuntimeError, match="easy-tdx\[baostock\]"):
BaostockClient().get_stock_kline(1, "600519", period="DAILY")
class _OkClient:
def __init__(self) -> None:
self.calls = 0
def get_stock_kline(self, market, code, **kwargs): # noqa: ANN001, ANN003
self.calls += 1
return pd.DataFrame({"datetime": [1], "close": [10.0]})
class _EmptyThenOkClient(_OkClient):
def get_stock_kline(self, market, code, **kwargs): # noqa: ANN001, ANN003
self.calls += 1
return pd.DataFrame()
class _RaisingClient(_OkClient):
def get_stock_kline(self, market, code, **kwargs): # noqa: ANN001, ANN003
self.calls += 1
raise RuntimeError("主源失败")
def test_auto_kline_client_primary_ok_skips_fallback():
from easy_tdx.sources import AutoKlineClient
primary, fallback = _OkClient(), _OkClient()
df = AutoKlineClient(primary, fallback).get_stock_kline(1, "600519", period="DAILY")
assert len(df) == 1
assert primary.calls == 1
assert fallback.calls == 0
def test_auto_kline_client_primary_empty_falls_back():
from easy_tdx.sources import AutoKlineClient
primary, fallback = _EmptyThenOkClient(), _OkClient()
df = AutoKlineClient(primary, fallback).get_stock_kline(1, "600519", period="DAILY")
assert len(df) == 1
assert fallback.calls == 1
def test_auto_kline_client_primary_error_falls_back():
from easy_tdx.sources import AutoKlineClient
fallback = _OkClient()
df = AutoKlineClient(_RaisingClient(), fallback).get_stock_kline(1, "600519")
assert len(df) == 1
assert fallback.calls == 1
def test_auto_kline_client_fallback_error_propagates():
from easy_tdx.sources import AutoKlineClient
with pytest.raises(RuntimeError, match="主源失败"):
AutoKlineClient(_RaisingClient(), _RaisingClient()).get_stock_kline(1, "600519")