mirror of
https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git
synced 2026-09-12 17:54:15 +08:00
feat(ext-data): 出站请求默认携带 tsp 标识头
扩展数据的三个出站点 (定时/手动拉取与历史回补、内置预设拉取、URL 探测 测试) 统一经 outbound_headers() 注入 User-Agent: tsp/<版本> 与 X-TSP-Client: tick-stock-panel, 服务端 (如 tickflow-hub) 可据此识别 本项目的请求来源。 用户在拉取配置里显式设置的同名请求头优先 (大小写不敏感匹配, 不重复 发送), 需要特定 UA 的数据源不受影响。 验证: pytest 22/22 (含 3 个新用例: 默认标识头/用户头优先/实际请求携带); 逐文件 ruff 与 HEAD 对比零新增。
This commit is contained in:
@@ -864,13 +864,13 @@ async def test_pull(request: Request, config_id: str):
|
||||
raise HTTPException(400, "拉取未配置或 URL 为空")
|
||||
|
||||
# 临时构建一个带新配置的 config 用于测试
|
||||
from app.services.ext_pull import _extract_rows, _apply_field_map
|
||||
from app.services.ext_pull import _extract_rows, _apply_field_map, outbound_headers
|
||||
import httpx
|
||||
|
||||
pull = config.pull
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
headers = pull.headers or {}
|
||||
headers = outbound_headers(pull.headers)
|
||||
kwargs: dict = {"headers": headers}
|
||||
if pull.method.upper() == "POST" and pull.body:
|
||||
kwargs["content"] = pull.body
|
||||
|
||||
@@ -182,8 +182,11 @@ async def _fetch_json(url: str) -> list[dict]:
|
||||
"""
|
||||
import httpx
|
||||
|
||||
# 延迟导入避免与 ext_pull 循环依赖; 出站请求带 tsp 标识头
|
||||
from app.services.ext_pull import outbound_headers
|
||||
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.get(url)
|
||||
resp = await client.get(url, headers=outbound_headers())
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
|
||||
@@ -21,6 +21,26 @@ from app.services.ext_data import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def outbound_headers(user_headers: dict[str, str] | None = None) -> dict[str, str]:
|
||||
"""扩展数据出站请求的默认标识头。
|
||||
|
||||
默认携带 User-Agent: tsp/<版本> 与 X-TSP-Client: tick-stock-panel,
|
||||
供服务端 (如 tickflow-hub) 识别本项目的请求。用户在拉取配置里显式
|
||||
设置的同名头优先 (大小写不敏感), 不被标识头覆盖。
|
||||
"""
|
||||
from app import __version__
|
||||
|
||||
defaults = {
|
||||
"User-Agent": f"tsp/{__version__}",
|
||||
"X-TSP-Client": "tick-stock-panel",
|
||||
}
|
||||
override = {k.lower() for k in (user_headers or {})}
|
||||
return {
|
||||
**{k: v for k, v in defaults.items() if k.lower() not in override},
|
||||
**(user_headers or {}),
|
||||
}
|
||||
|
||||
|
||||
def _in_time_window(start: str | None, end: str | None) -> bool:
|
||||
"""检查当前本地时间是否在每日时间窗口内。
|
||||
|
||||
@@ -144,7 +164,7 @@ async def fetch_rows_for_date(config: ExtConfig, target_date: date) -> list[dict
|
||||
|
||||
url = _with_date_param(pull.url, pull.date_param, target_date)
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
headers = pull.headers or {}
|
||||
headers = outbound_headers(pull.headers)
|
||||
kwargs: dict[str, Any] = {"headers": headers}
|
||||
|
||||
if pull.method.upper() == "POST" and pull.body:
|
||||
|
||||
@@ -60,6 +60,7 @@ class _FakeClient:
|
||||
|
||||
responses: ClassVar[dict[str, list]] = {}
|
||||
calls: ClassVar[list[str]] = []
|
||||
header_calls: ClassVar[list[dict]] = [] # 每次请求实际发送的 headers
|
||||
errors: ClassVar[dict[str, Exception]] = {}
|
||||
fail_times: ClassVar[dict[str, int]] = {} # url -> 还需失败的次数
|
||||
error_sequence: ClassVar[dict[str, list[Exception]]] = {} # url -> 按序抛出后耗尽
|
||||
@@ -75,6 +76,7 @@ class _FakeClient:
|
||||
|
||||
async def request(self, method: str, url: str, **kwargs):
|
||||
_FakeClient.calls.append(url)
|
||||
_FakeClient.header_calls.append(dict(kwargs.get("headers") or {}))
|
||||
seq = _FakeClient.error_sequence.get(url)
|
||||
if seq:
|
||||
raise seq.pop(0)
|
||||
@@ -88,6 +90,7 @@ class _FakeClient:
|
||||
def fake_http(monkeypatch):
|
||||
_FakeClient.responses = {}
|
||||
_FakeClient.calls = []
|
||||
_FakeClient.header_calls = []
|
||||
_FakeClient.errors = {}
|
||||
_FakeClient.fail_times = {}
|
||||
_FakeClient.error_sequence = {}
|
||||
@@ -97,6 +100,46 @@ def fake_http(monkeypatch):
|
||||
|
||||
# ── 纯函数 ────────────────────────────────────────────────
|
||||
|
||||
def test_outbound_headers_default_and_override():
|
||||
"""出站标识头: 默认带 tsp UA + X-TSP-Client; 用户同名头优先 (大小写不敏感)。"""
|
||||
from app.services.ext_pull import outbound_headers
|
||||
|
||||
h = outbound_headers()
|
||||
assert h["User-Agent"].startswith("tsp/")
|
||||
assert h["X-TSP-Client"] == "tick-stock-panel"
|
||||
|
||||
h2 = outbound_headers({"user-agent": "my-ua", "X-Custom": "1"})
|
||||
assert h2["user-agent"] == "my-ua" # 小写同名覆盖默认 UA
|
||||
assert "User-Agent" not in h2 # 不重复发送
|
||||
assert h2["X-TSP-Client"] == "tick-stock-panel" # 未覆盖的标识头保留
|
||||
assert h2["X-Custom"] == "1"
|
||||
|
||||
|
||||
async def test_fetch_rows_carries_tsp_identity(fake_http):
|
||||
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-01-05")]
|
||||
await fetch_rows_for_date(_cfg(), date(2026, 1, 5))
|
||||
headers = fake_http.header_calls[-1]
|
||||
assert headers["User-Agent"].startswith("tsp/")
|
||||
assert headers["X-TSP-Client"] == "tick-stock-panel"
|
||||
|
||||
|
||||
async def test_fetch_rows_user_headers_take_precedence(fake_http):
|
||||
cfg = ExtConfig(
|
||||
id="hot", label="人气", mode="timeseries",
|
||||
fields=_cfg().fields,
|
||||
pull=PullConfig(
|
||||
url="https://example.test/rank",
|
||||
headers={"User-Agent": "custom-ua"},
|
||||
date_param="date",
|
||||
),
|
||||
)
|
||||
fake_http.responses["https://example.test/rank?date=2026-01-05"] = [_row("A", "2026-01-05")]
|
||||
await fetch_rows_for_date(cfg, date(2026, 1, 5))
|
||||
headers = fake_http.header_calls[-1]
|
||||
assert headers["User-Agent"] == "custom-ua"
|
||||
assert headers["X-TSP-Client"] == "tick-stock-panel"
|
||||
|
||||
|
||||
def test_with_date_param_url_building():
|
||||
d = date(2026, 1, 5)
|
||||
assert _with_date_param("https://x/api", "date", d) == "https://x/api?date=2026-01-05"
|
||||
|
||||
Reference in New Issue
Block a user