From ddb265b0f88088495bc177643b4bbc8da54f9e07 Mon Sep 17 00:00:00 2001 From: shy3130 <415333856@qq.com> Date: Sun, 6 Sep 2026 23:16:27 +0800 Subject: [PATCH] =?UTF-8?q?feat(ext-data):=20=E5=87=BA=E7=AB=99=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E9=BB=98=E8=AE=A4=E6=90=BA=E5=B8=A6=20tsp=20=E6=A0=87?= =?UTF-8?q?=E8=AF=86=E5=A4=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 扩展数据的三个出站点 (定时/手动拉取与历史回补、内置预设拉取、URL 探测 测试) 统一经 outbound_headers() 注入 User-Agent: tsp/<版本> 与 X-TSP-Client: tick-stock-panel, 服务端 (如 tickflow-hub) 可据此识别 本项目的请求来源。 用户在拉取配置里显式设置的同名请求头优先 (大小写不敏感匹配, 不重复 发送), 需要特定 UA 的数据源不受影响。 验证: pytest 22/22 (含 3 个新用例: 默认标识头/用户头优先/实际请求携带); 逐文件 ruff 与 HEAD 对比零新增。 --- backend/app/api/ext_data.py | 4 +-- backend/app/services/ext_presets.py | 5 +++- backend/app/services/ext_pull.py | 22 ++++++++++++++- backend/tests/test_ext_backfill.py | 43 +++++++++++++++++++++++++++++ docs/features.md | 2 +- 5 files changed, 71 insertions(+), 5 deletions(-) diff --git a/backend/app/api/ext_data.py b/backend/app/api/ext_data.py index e0a2d56..5facb6a 100644 --- a/backend/app/api/ext_data.py +++ b/backend/app/api/ext_data.py @@ -864,13 +864,13 @@ async def test_pull(request: Request, config_id: str): raise HTTPException(400, "拉取未配置或 URL 为空") # 临时构建一个带新配置的 config 用于测试 - from app.services.ext_pull import _extract_rows, _apply_field_map + from app.services.ext_pull import _extract_rows, _apply_field_map, outbound_headers import httpx pull = config.pull try: async with httpx.AsyncClient(timeout=30) as client: - headers = pull.headers or {} + headers = outbound_headers(pull.headers) kwargs: dict = {"headers": headers} if pull.method.upper() == "POST" and pull.body: kwargs["content"] = pull.body diff --git a/backend/app/services/ext_presets.py b/backend/app/services/ext_presets.py index 7ff77e1..527753b 100644 --- a/backend/app/services/ext_presets.py +++ b/backend/app/services/ext_presets.py @@ -182,8 +182,11 @@ async def _fetch_json(url: str) -> list[dict]: """ import httpx + # 延迟导入避免与 ext_pull 循环依赖; 出站请求带 tsp 标识头 + from app.services.ext_pull import outbound_headers + async with httpx.AsyncClient(timeout=30) as client: - resp = await client.get(url) + resp = await client.get(url, headers=outbound_headers()) resp.raise_for_status() data = resp.json() diff --git a/backend/app/services/ext_pull.py b/backend/app/services/ext_pull.py index 16a7a8a..eb791d0 100644 --- a/backend/app/services/ext_pull.py +++ b/backend/app/services/ext_pull.py @@ -21,6 +21,26 @@ from app.services.ext_data import ( logger = logging.getLogger(__name__) +def outbound_headers(user_headers: dict[str, str] | None = None) -> dict[str, str]: + """扩展数据出站请求的默认标识头。 + + 默认携带 User-Agent: tsp/<版本> 与 X-TSP-Client: tick-stock-panel, + 供服务端 (如 tickflow-hub) 识别本项目的请求。用户在拉取配置里显式 + 设置的同名头优先 (大小写不敏感), 不被标识头覆盖。 + """ + from app import __version__ + + defaults = { + "User-Agent": f"tsp/{__version__}", + "X-TSP-Client": "tick-stock-panel", + } + override = {k.lower() for k in (user_headers or {})} + return { + **{k: v for k, v in defaults.items() if k.lower() not in override}, + **(user_headers or {}), + } + + def _in_time_window(start: str | None, end: str | None) -> bool: """检查当前本地时间是否在每日时间窗口内。 @@ -144,7 +164,7 @@ async def fetch_rows_for_date(config: ExtConfig, target_date: date) -> list[dict url = _with_date_param(pull.url, pull.date_param, target_date) async with httpx.AsyncClient(timeout=30) as client: - headers = pull.headers or {} + headers = outbound_headers(pull.headers) kwargs: dict[str, Any] = {"headers": headers} if pull.method.upper() == "POST" and pull.body: diff --git a/backend/tests/test_ext_backfill.py b/backend/tests/test_ext_backfill.py index 8313e3c..17f34e7 100644 --- a/backend/tests/test_ext_backfill.py +++ b/backend/tests/test_ext_backfill.py @@ -60,6 +60,7 @@ class _FakeClient: responses: ClassVar[dict[str, list]] = {} calls: ClassVar[list[str]] = [] + header_calls: ClassVar[list[dict]] = [] # 每次请求实际发送的 headers errors: ClassVar[dict[str, Exception]] = {} fail_times: ClassVar[dict[str, int]] = {} # url -> 还需失败的次数 error_sequence: ClassVar[dict[str, list[Exception]]] = {} # url -> 按序抛出后耗尽 @@ -75,6 +76,7 @@ class _FakeClient: async def request(self, method: str, url: str, **kwargs): _FakeClient.calls.append(url) + _FakeClient.header_calls.append(dict(kwargs.get("headers") or {})) seq = _FakeClient.error_sequence.get(url) if seq: raise seq.pop(0) @@ -88,6 +90,7 @@ class _FakeClient: def fake_http(monkeypatch): _FakeClient.responses = {} _FakeClient.calls = [] + _FakeClient.header_calls = [] _FakeClient.errors = {} _FakeClient.fail_times = {} _FakeClient.error_sequence = {} @@ -97,6 +100,46 @@ def fake_http(monkeypatch): # ── 纯函数 ──────────────────────────────────────────────── +def test_outbound_headers_default_and_override(): + """出站标识头: 默认带 tsp UA + X-TSP-Client; 用户同名头优先 (大小写不敏感)。""" + from app.services.ext_pull import outbound_headers + + h = outbound_headers() + assert h["User-Agent"].startswith("tsp/") + assert h["X-TSP-Client"] == "tick-stock-panel" + + h2 = outbound_headers({"user-agent": "my-ua", "X-Custom": "1"}) + assert h2["user-agent"] == "my-ua" # 小写同名覆盖默认 UA + assert "User-Agent" not in h2 # 不重复发送 + assert h2["X-TSP-Client"] == "tick-stock-panel" # 未覆盖的标识头保留 + assert h2["X-Custom"] == "1" + + +async def test_fetch_rows_carries_tsp_identity(fake_http): + fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-01-05")] + await fetch_rows_for_date(_cfg(), date(2026, 1, 5)) + headers = fake_http.header_calls[-1] + assert headers["User-Agent"].startswith("tsp/") + assert headers["X-TSP-Client"] == "tick-stock-panel" + + +async def test_fetch_rows_user_headers_take_precedence(fake_http): + cfg = ExtConfig( + id="hot", label="人气", mode="timeseries", + fields=_cfg().fields, + pull=PullConfig( + url="https://example.test/rank", + headers={"User-Agent": "custom-ua"}, + date_param="date", + ), + ) + fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-01-05")] + await fetch_rows_for_date(cfg, date(2026, 1, 5)) + headers = fake_http.header_calls[-1] + assert headers["User-Agent"] == "custom-ua" + assert headers["X-TSP-Client"] == "tick-stock-panel" + + def test_with_date_param_url_building(): d = date(2026, 1, 5) assert _with_date_param("https://x/api", "date", d) == "https://x/api?date=2026-01-05" diff --git a/docs/features.md b/docs/features.md index 03a8afa..f071217 100644 --- a/docs/features.md +++ b/docs/features.md @@ -181,7 +181,7 @@ | CSV / Excel 上传 | 页面直接上传文件 | | JSON 写入 | 程序化写入 | -接入后自动 schema 发现 + 符号归一,页面可视化配置,最终并入 DuckDB 同台分析。 +接入后自动 schema 发现 + 符号归一,页面可视化配置,最终并入 DuckDB 同台分析。HTTP 拉取的出站请求默认携带标识头 (`User-Agent: tsp/<版本>` 与 `X-TSP-Client: tick-stock-panel`),服务端可据此识别本项目来源;拉取配置里的自定义同名请求头优先。 ##### 时序表历史回补