feat(ext-data): 出站请求默认携带 tsp 标识头

扩展数据的三个出站点 (定时/手动拉取与历史回补、内置预设拉取、URL 探测
测试) 统一经 outbound_headers() 注入 User-Agent: tsp/<版本> 与
X-TSP-Client: tick-stock-panel, 服务端 (如 tickflow-hub) 可据此识别
本项目的请求来源。

用户在拉取配置里显式设置的同名请求头优先 (大小写不敏感匹配, 不重复
发送), 需要特定 UA 的数据源不受影响。

验证: pytest 22/22 (含 3 个新用例: 默认标识头/用户头优先/实际请求携带);
逐文件 ruff 与 HEAD 对比零新增。
This commit is contained in:
shy3130
2026-09-06 23:16:27 +08:00
parent d949d9618c
commit ddb265b0f8
5 changed files with 71 additions and 5 deletions
+2 -2
View File
@@ -864,13 +864,13 @@ async def test_pull(request: Request, config_id: str):
raise HTTPException(400, "拉取未配置或 URL 为空")
# 临时构建一个带新配置的 config 用于测试
from app.services.ext_pull import _extract_rows, _apply_field_map
from app.services.ext_pull import _extract_rows, _apply_field_map, outbound_headers
import httpx
pull = config.pull
try:
async with httpx.AsyncClient(timeout=30) as client:
headers = pull.headers or {}
headers = outbound_headers(pull.headers)
kwargs: dict = {"headers": headers}
if pull.method.upper() == "POST" and pull.body:
kwargs["content"] = pull.body
+4 -1
View File
@@ -182,8 +182,11 @@ async def _fetch_json(url: str) -> list[dict]:
"""
import httpx
# 延迟导入避免与 ext_pull 循环依赖; 出站请求带 tsp 标识头
from app.services.ext_pull import outbound_headers
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.get(url)
resp = await client.get(url, headers=outbound_headers())
resp.raise_for_status()
data = resp.json()
+21 -1
View File
@@ -21,6 +21,26 @@ from app.services.ext_data import (
logger = logging.getLogger(__name__)
def outbound_headers(user_headers: dict[str, str] | None = None) -> dict[str, str]:
"""扩展数据出站请求的默认标识头。
默认携带 User-Agent: tsp/<版本> 与 X-TSP-Client: tick-stock-panel,
供服务端 (如 tickflow-hub) 识别本项目的请求。用户在拉取配置里显式
设置的同名头优先 (大小写不敏感), 不被标识头覆盖。
"""
from app import __version__
defaults = {
"User-Agent": f"tsp/{__version__}",
"X-TSP-Client": "tick-stock-panel",
}
override = {k.lower() for k in (user_headers or {})}
return {
**{k: v for k, v in defaults.items() if k.lower() not in override},
**(user_headers or {}),
}
def _in_time_window(start: str | None, end: str | None) -> bool:
"""检查当前本地时间是否在每日时间窗口内。
@@ -144,7 +164,7 @@ async def fetch_rows_for_date(config: ExtConfig, target_date: date) -> list[dict
url = _with_date_param(pull.url, pull.date_param, target_date)
async with httpx.AsyncClient(timeout=30) as client:
headers = pull.headers or {}
headers = outbound_headers(pull.headers)
kwargs: dict[str, Any] = {"headers": headers}
if pull.method.upper() == "POST" and pull.body:
+43
View File
@@ -60,6 +60,7 @@ class _FakeClient:
responses: ClassVar[dict[str, list]] = {}
calls: ClassVar[list[str]] = []
header_calls: ClassVar[list[dict]] = [] # 每次请求实际发送的 headers
errors: ClassVar[dict[str, Exception]] = {}
fail_times: ClassVar[dict[str, int]] = {} # url -> 还需失败的次数
error_sequence: ClassVar[dict[str, list[Exception]]] = {} # url -> 按序抛出后耗尽
@@ -75,6 +76,7 @@ class _FakeClient:
async def request(self, method: str, url: str, **kwargs):
_FakeClient.calls.append(url)
_FakeClient.header_calls.append(dict(kwargs.get("headers") or {}))
seq = _FakeClient.error_sequence.get(url)
if seq:
raise seq.pop(0)
@@ -88,6 +90,7 @@ class _FakeClient:
def fake_http(monkeypatch):
_FakeClient.responses = {}
_FakeClient.calls = []
_FakeClient.header_calls = []
_FakeClient.errors = {}
_FakeClient.fail_times = {}
_FakeClient.error_sequence = {}
@@ -97,6 +100,46 @@ def fake_http(monkeypatch):
# ── 纯函数 ────────────────────────────────────────────────
def test_outbound_headers_default_and_override():
"""出站标识头: 默认带 tsp UA + X-TSP-Client; 用户同名头优先 (大小写不敏感)。"""
from app.services.ext_pull import outbound_headers
h = outbound_headers()
assert h["User-Agent"].startswith("tsp/")
assert h["X-TSP-Client"] == "tick-stock-panel"
h2 = outbound_headers({"user-agent": "my-ua", "X-Custom": "1"})
assert h2["user-agent"] == "my-ua" # 小写同名覆盖默认 UA
assert "User-Agent" not in h2 # 不重复发送
assert h2["X-TSP-Client"] == "tick-stock-panel" # 未覆盖的标识头保留
assert h2["X-Custom"] == "1"
async def test_fetch_rows_carries_tsp_identity(fake_http):
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-01-05")]
await fetch_rows_for_date(_cfg(), date(2026, 1, 5))
headers = fake_http.header_calls[-1]
assert headers["User-Agent"].startswith("tsp/")
assert headers["X-TSP-Client"] == "tick-stock-panel"
async def test_fetch_rows_user_headers_take_precedence(fake_http):
cfg = ExtConfig(
id="hot", label="人气", mode="timeseries",
fields=_cfg().fields,
pull=PullConfig(
url="https://example.test/rank",
headers={"User-Agent": "custom-ua"},
date_param="date",
),
)
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-01-05")]
await fetch_rows_for_date(cfg, date(2026, 1, 5))
headers = fake_http.header_calls[-1]
assert headers["User-Agent"] == "custom-ua"
assert headers["X-TSP-Client"] == "tick-stock-panel"
def test_with_date_param_url_building():
d = date(2026, 1, 5)
assert _with_date_param("https://x/api", "date", d) == "https://x/api?date=2026-01-05"