From 96fcbe4fff066dbd91ae2c4906034463744a1f0e Mon Sep 17 00:00:00 2001 From: shy3130 Date: Sat, 25 Jul 2026 22:13:38 +0800 Subject: [PATCH] fix(data-source): validate custom request settings --- backend/app/api/settings.py | 4 +- backend/app/data_providers/custom/config.py | 20 ++++-- backend/app/data_providers/custom/loader.py | 48 ++++++++----- backend/tests/test_custom_minute_config.py | 67 ++++++++++++++++++- .../src/pages/settings/DataSourceEditor.tsx | 33 ++++++--- 5 files changed, 135 insertions(+), 37 deletions(-) diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 4bcebfd..8214316 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -8,7 +8,7 @@ import logging import time from fastapi import APIRouter, HTTPException, Request -from pydantic import BaseModel +from pydantic import BaseModel, Field from app import secrets_store from app.tickflow import client as tf_client @@ -373,7 +373,7 @@ class DatasetConfigIn(BaseModel): end_param: str = "end_time" asset_type_param: str | None = None freq_param: str | None = None - timeout: float | None = None + timeout: float | None = Field(default=None, gt=0, allow_inf_nan=False) class AuthConfigIn(BaseModel): diff --git a/backend/app/data_providers/custom/config.py b/backend/app/data_providers/custom/config.py index ae52033..4395630 100644 --- a/backend/app/data_providers/custom/config.py +++ b/backend/app/data_providers/custom/config.py @@ -1,6 +1,7 @@ """Custom HTTP data source configuration.""" from __future__ import annotations +import math from dataclasses import dataclass, field from pathlib import Path from typing import Any, Literal @@ -60,22 +61,29 @@ def _auth_from_dict(raw: dict[str, Any] | None) -> AuthConfig: def _dataset_from_dict(raw: dict[str, Any]) -> DatasetConfig: + try: + timeout = float(raw.get("timeout", 30.0) or 30.0) + except (TypeError, ValueError): + timeout = 30.0 + if not math.isfinite(timeout) or timeout <= 0: + timeout = 30.0 + return DatasetConfig( url=str(raw.get("url", "") or ""), method=str(raw.get("method", "GET") or "GET").upper(), batch=int(raw["batch"]) if raw.get("batch") is not None else None, rpm=int(raw["rpm"]) if raw.get("rpm") is not None else None, - timeout=float(raw.get("timeout", 30.0) or 30.0), + timeout=timeout, response_path=str(raw.get("response_path", "") or ""), field_map={str(k): str(v) for k, v in (raw.get("field_map") or {}).items()}, transforms={str(k): str(v) for k, v in (raw.get("transforms") or {}).items()}, params=dict(raw.get("params") or {}), body=dict(raw.get("body") or {}), - symbols_param=str(raw.get("symbols_param", "symbols") or "symbols"), - start_param=str(raw.get("start_param", "start_time") or "start_time"), - end_param=str(raw.get("end_param", "end_time") or "end_time"), - asset_type_param=str(raw.get("asset_type_param")) if raw.get("asset_type_param") else None, - freq_param=str(raw.get("freq_param")) if raw.get("freq_param") else None, + symbols_param=str(raw.get("symbols_param", "symbols") or "symbols").strip() or "symbols", + start_param=str(raw.get("start_param", "start_time") or "start_time").strip() or "start_time", + end_param=str(raw.get("end_param", "end_time") or "end_time").strip() or "end_time", + asset_type_param=(str(raw.get("asset_type_param") or "").strip() or None), + freq_param=(str(raw.get("freq_param") or "").strip() or None), ) diff --git a/backend/app/data_providers/custom/loader.py b/backend/app/data_providers/custom/loader.py index 40dac1d..0aa4a35 100644 --- a/backend/app/data_providers/custom/loader.py +++ b/backend/app/data_providers/custom/loader.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib import logging +import math import re import shutil import subprocess @@ -294,11 +295,13 @@ def _config_to_dict(config: CustomSourceConfig) -> dict: "response_path": ds.response_path, "field_map": dict(ds.field_map), **({"transforms": dict(ds.transforms)} if ds.transforms else {}), - "symbols_param": ds.symbols_param, - "start_param": ds.start_param, - "end_param": ds.end_param, - **({"asset_type_param": ds.asset_type_param} if ds.asset_type_param else {}), - **({"freq_param": ds.freq_param} if ds.freq_param else {}), + **({ + "symbols_param": ds.symbols_param, + "start_param": ds.start_param, + "end_param": ds.end_param, + } if ds_name != "realtime" else {}), + **({"asset_type_param": ds.asset_type_param} if ds_name == "minute" and ds.asset_type_param else {}), + **({"freq_param": ds.freq_param} if ds_name == "minute" and ds.freq_param else {}), } return out @@ -358,14 +361,14 @@ def _sanitize_for_yaml(config: dict) -> dict: continue if not isinstance(ds_cfg, dict): continue - ds = _sanitize_dataset(ds_cfg) + ds = _sanitize_dataset(ds_name, ds_cfg) if ds: datasets_out[ds_name] = ds out["datasets"] = datasets_out return out -def _sanitize_dataset(ds_cfg: dict) -> dict: +def _sanitize_dataset(ds_name: str, ds_cfg: dict) -> dict: out: dict = {} url = str(ds_cfg.get("url", "") or "").strip() if not url: @@ -385,7 +388,9 @@ def _sanitize_dataset(ds_cfg: dict) -> dict: pass if ds_cfg.get("timeout") is not None: try: - out["timeout"] = float(ds_cfg["timeout"]) + timeout = float(ds_cfg["timeout"]) + if math.isfinite(timeout) and timeout > 0: + out["timeout"] = timeout except (TypeError, ValueError): pass out["response_path"] = str(ds_cfg.get("response_path", "") or "") @@ -403,16 +408,23 @@ def _sanitize_dataset(ds_cfg: dict) -> dict: } if transforms: out["transforms"] = transforms - if ds_cfg.get("symbols_param"): - out["symbols_param"] = str(ds_cfg["symbols_param"]) - if ds_cfg.get("start_param"): - out["start_param"] = str(ds_cfg["start_param"]) - if ds_cfg.get("end_param"): - out["end_param"] = str(ds_cfg["end_param"]) - if ds_cfg.get("asset_type_param"): - out["asset_type_param"] = str(ds_cfg["asset_type_param"]) - if ds_cfg.get("freq_param"): - out["freq_param"] = str(ds_cfg["freq_param"]) + if ds_name != "realtime": + symbols_param = str(ds_cfg.get("symbols_param") or "").strip() + start_param = str(ds_cfg.get("start_param") or "").strip() + end_param = str(ds_cfg.get("end_param") or "").strip() + if symbols_param: + out["symbols_param"] = symbols_param + if start_param: + out["start_param"] = start_param + if end_param: + out["end_param"] = end_param + if ds_name == "minute": + asset_type_param = str(ds_cfg.get("asset_type_param") or "").strip() + freq_param = str(ds_cfg.get("freq_param") or "").strip() + if asset_type_param: + out["asset_type_param"] = asset_type_param + if freq_param: + out["freq_param"] = freq_param return out diff --git a/backend/tests/test_custom_minute_config.py b/backend/tests/test_custom_minute_config.py index d34927e..39ba6e9 100644 --- a/backend/tests/test_custom_minute_config.py +++ b/backend/tests/test_custom_minute_config.py @@ -1,3 +1,8 @@ +import math + +import pytest +from pydantic import ValidationError + from app.api.settings import DatasetConfigIn from app.data_providers.custom.config import CustomSourceConfig, _dataset_from_dict from app.data_providers.custom.loader import _config_to_dict, _sanitize_for_yaml @@ -66,4 +71,64 @@ def test_timeout_survives_config_round_trip(): datasets={"realtime": parsed2}, )) assert parsed2.timeout == 30.0 - assert "timeout" not in exposed2["datasets"]["realtime"] + realtime = exposed2["datasets"]["realtime"] + assert "timeout" not in realtime + assert "symbols_param" not in realtime + assert "start_param" not in realtime + assert "end_param" not in realtime + + +@pytest.mark.parametrize("timeout", [0, -1, math.nan, math.inf, -math.inf]) +def test_timeout_api_rejects_non_positive_or_non_finite_values(timeout): + with pytest.raises(ValidationError): + DatasetConfigIn(url="https://example.test/daily", timeout=timeout) + + +def test_invalid_yaml_timeout_falls_back_to_default(): + for timeout in (0, -1, math.nan, math.inf, -math.inf, "invalid"): + parsed = _dataset_from_dict({ + "url": "https://example.test/daily", + "timeout": timeout, + }) + assert parsed.timeout == 30.0 + + +@pytest.mark.parametrize("timeout", [0, -1, math.nan, math.inf, -math.inf, "invalid"]) +def test_sanitizer_drops_invalid_timeout_and_realtime_request_params(timeout): + cleaned = _sanitize_for_yaml({ + "name": "test_source", + "datasets": { + "realtime": { + "url": "https://example.test/realtime", + "timeout": timeout, + "symbols_param": "codes", + "start_param": "from", + "end_param": "to", + }, + }, + }) + + dataset = cleaned["datasets"]["realtime"] + assert "timeout" not in dataset + assert "symbols_param" not in dataset + assert "start_param" not in dataset + assert "end_param" not in dataset + + +def test_empty_request_parameter_names_restore_defaults(): + cleaned = _sanitize_for_yaml({ + "name": "test_source", + "datasets": { + "minute": { + "url": "https://example.test/minute", + "symbols_param": " ", + "start_param": "\t", + "end_param": "", + }, + }, + }) + parsed = _dataset_from_dict(cleaned["datasets"]["minute"]) + + assert parsed.symbols_param == "symbols" + assert parsed.start_param == "start_time" + assert parsed.end_param == "end_time" diff --git a/frontend/src/pages/settings/DataSourceEditor.tsx b/frontend/src/pages/settings/DataSourceEditor.tsx index 4087efc..70b295d 100644 --- a/frontend/src/pages/settings/DataSourceEditor.tsx +++ b/frontend/src/pages/settings/DataSourceEditor.tsx @@ -1,7 +1,7 @@ import { useState, useEffect, useRef } from 'react' import { useMutation, useQuery } from '@tanstack/react-query' import { motion, AnimatePresence } from 'framer-motion' - import { KeyRound, Play, Plus, Save, Trash2, X, Zap, Check, ChevronDown } from 'lucide-react' +import { KeyRound, Play, Plus, Save, Trash2, X, Zap, Check, ChevronDown } from 'lucide-react' import { api, type CustomSourceConfig, type DatasetConfig } from '@/lib/api' import { toast } from '@/components/Toast' @@ -96,6 +96,9 @@ export function DataSourceEditor({ if (!ds.url.trim()) { throw new Error(`数据集「${DATASET_LABEL[key as DatasetKey] || key}」未填写接口 URL`) } + if (ds.timeout != null && (!Number.isFinite(ds.timeout) || ds.timeout <= 0)) { + throw new Error(`数据集「${DATASET_LABEL[key as DatasetKey] || key}」超时必须大于 0 秒`) + } } // 提交时去掉 field_map 里的 __pending_ 临时 key (未填外部字段名的草稿行) const cleaned: CustomSourceConfig = { @@ -103,15 +106,23 @@ export function DataSourceEditor({ name: config.name.toLowerCase().trim(), display_name: config.display_name.trim() || config.name.toLowerCase().trim(), datasets: Object.fromEntries( - Object.entries(config.datasets).map(([k, ds]) => [ - k, - { - ...ds, - field_map: Object.fromEntries( - Object.entries(ds.field_map).filter(([src]) => !src.startsWith('__pending_')) - ), - }, - ]) + Object.entries(config.datasets).map(([k, ds]) => { + const normalized = { ...ds } + if (k === 'realtime') { + delete normalized.symbols_param + delete normalized.start_param + delete normalized.end_param + } + return [ + k, + { + ...normalized, + field_map: Object.fromEntries( + Object.entries(ds.field_map).filter(([src]) => !src.startsWith('__pending_')) + ), + }, + ] + }) ), } return api.saveDataSource(cleaned) @@ -390,6 +401,8 @@ function DatasetDetail({ onUpdate({ timeout: e.target.value ? Number(e.target.value) : null })} onWheel={e => e.currentTarget.blur()}