fix(data-source): validate custom request settings

This commit is contained in:
shy3130
2026-07-25 22:13:38 +08:00
parent fffd89fce0
commit 96fcbe4fff
5 changed files with 135 additions and 37 deletions
+2 -2
View File
@@ -8,7 +8,7 @@ import logging
import time
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel
from pydantic import BaseModel, Field
from app import secrets_store
from app.tickflow import client as tf_client
@@ -373,7 +373,7 @@ class DatasetConfigIn(BaseModel):
end_param: str = "end_time"
asset_type_param: str | None = None
freq_param: str | None = None
timeout: float | None = None
timeout: float | None = Field(default=None, gt=0, allow_inf_nan=False)
class AuthConfigIn(BaseModel):
+14 -6
View File
@@ -1,6 +1,7 @@
"""Custom HTTP data source configuration."""
from __future__ import annotations
import math
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Literal
@@ -60,22 +61,29 @@ def _auth_from_dict(raw: dict[str, Any] | None) -> AuthConfig:
def _dataset_from_dict(raw: dict[str, Any]) -> DatasetConfig:
try:
timeout = float(raw.get("timeout", 30.0) or 30.0)
except (TypeError, ValueError):
timeout = 30.0
if not math.isfinite(timeout) or timeout <= 0:
timeout = 30.0
return DatasetConfig(
url=str(raw.get("url", "") or ""),
method=str(raw.get("method", "GET") or "GET").upper(),
batch=int(raw["batch"]) if raw.get("batch") is not None else None,
rpm=int(raw["rpm"]) if raw.get("rpm") is not None else None,
timeout=float(raw.get("timeout", 30.0) or 30.0),
timeout=timeout,
response_path=str(raw.get("response_path", "") or ""),
field_map={str(k): str(v) for k, v in (raw.get("field_map") or {}).items()},
transforms={str(k): str(v) for k, v in (raw.get("transforms") or {}).items()},
params=dict(raw.get("params") or {}),
body=dict(raw.get("body") or {}),
symbols_param=str(raw.get("symbols_param", "symbols") or "symbols"),
start_param=str(raw.get("start_param", "start_time") or "start_time"),
end_param=str(raw.get("end_param", "end_time") or "end_time"),
asset_type_param=str(raw.get("asset_type_param")) if raw.get("asset_type_param") else None,
freq_param=str(raw.get("freq_param")) if raw.get("freq_param") else None,
symbols_param=str(raw.get("symbols_param", "symbols") or "symbols").strip() or "symbols",
start_param=str(raw.get("start_param", "start_time") or "start_time").strip() or "start_time",
end_param=str(raw.get("end_param", "end_time") or "end_time").strip() or "end_time",
asset_type_param=(str(raw.get("asset_type_param") or "").strip() or None),
freq_param=(str(raw.get("freq_param") or "").strip() or None),
)
+30 -18
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import importlib
import logging
import math
import re
import shutil
import subprocess
@@ -294,11 +295,13 @@ def _config_to_dict(config: CustomSourceConfig) -> dict:
"response_path": ds.response_path,
"field_map": dict(ds.field_map),
**({"transforms": dict(ds.transforms)} if ds.transforms else {}),
"symbols_param": ds.symbols_param,
"start_param": ds.start_param,
"end_param": ds.end_param,
**({"asset_type_param": ds.asset_type_param} if ds.asset_type_param else {}),
**({"freq_param": ds.freq_param} if ds.freq_param else {}),
**({
"symbols_param": ds.symbols_param,
"start_param": ds.start_param,
"end_param": ds.end_param,
} if ds_name != "realtime" else {}),
**({"asset_type_param": ds.asset_type_param} if ds_name == "minute" and ds.asset_type_param else {}),
**({"freq_param": ds.freq_param} if ds_name == "minute" and ds.freq_param else {}),
}
return out
@@ -358,14 +361,14 @@ def _sanitize_for_yaml(config: dict) -> dict:
continue
if not isinstance(ds_cfg, dict):
continue
ds = _sanitize_dataset(ds_cfg)
ds = _sanitize_dataset(ds_name, ds_cfg)
if ds:
datasets_out[ds_name] = ds
out["datasets"] = datasets_out
return out
def _sanitize_dataset(ds_cfg: dict) -> dict:
def _sanitize_dataset(ds_name: str, ds_cfg: dict) -> dict:
out: dict = {}
url = str(ds_cfg.get("url", "") or "").strip()
if not url:
@@ -385,7 +388,9 @@ def _sanitize_dataset(ds_cfg: dict) -> dict:
pass
if ds_cfg.get("timeout") is not None:
try:
out["timeout"] = float(ds_cfg["timeout"])
timeout = float(ds_cfg["timeout"])
if math.isfinite(timeout) and timeout > 0:
out["timeout"] = timeout
except (TypeError, ValueError):
pass
out["response_path"] = str(ds_cfg.get("response_path", "") or "")
@@ -403,16 +408,23 @@ def _sanitize_dataset(ds_cfg: dict) -> dict:
}
if transforms:
out["transforms"] = transforms
if ds_cfg.get("symbols_param"):
out["symbols_param"] = str(ds_cfg["symbols_param"])
if ds_cfg.get("start_param"):
out["start_param"] = str(ds_cfg["start_param"])
if ds_cfg.get("end_param"):
out["end_param"] = str(ds_cfg["end_param"])
if ds_cfg.get("asset_type_param"):
out["asset_type_param"] = str(ds_cfg["asset_type_param"])
if ds_cfg.get("freq_param"):
out["freq_param"] = str(ds_cfg["freq_param"])
if ds_name != "realtime":
symbols_param = str(ds_cfg.get("symbols_param") or "").strip()
start_param = str(ds_cfg.get("start_param") or "").strip()
end_param = str(ds_cfg.get("end_param") or "").strip()
if symbols_param:
out["symbols_param"] = symbols_param
if start_param:
out["start_param"] = start_param
if end_param:
out["end_param"] = end_param
if ds_name == "minute":
asset_type_param = str(ds_cfg.get("asset_type_param") or "").strip()
freq_param = str(ds_cfg.get("freq_param") or "").strip()
if asset_type_param:
out["asset_type_param"] = asset_type_param
if freq_param:
out["freq_param"] = freq_param
return out
+66 -1
View File
@@ -1,3 +1,8 @@
import math
import pytest
from pydantic import ValidationError
from app.api.settings import DatasetConfigIn
from app.data_providers.custom.config import CustomSourceConfig, _dataset_from_dict
from app.data_providers.custom.loader import _config_to_dict, _sanitize_for_yaml
@@ -66,4 +71,64 @@ def test_timeout_survives_config_round_trip():
datasets={"realtime": parsed2},
))
assert parsed2.timeout == 30.0
assert "timeout" not in exposed2["datasets"]["realtime"]
realtime = exposed2["datasets"]["realtime"]
assert "timeout" not in realtime
assert "symbols_param" not in realtime
assert "start_param" not in realtime
assert "end_param" not in realtime
@pytest.mark.parametrize("timeout", [0, -1, math.nan, math.inf, -math.inf])
def test_timeout_api_rejects_non_positive_or_non_finite_values(timeout):
with pytest.raises(ValidationError):
DatasetConfigIn(url="https://example.test/daily", timeout=timeout)
def test_invalid_yaml_timeout_falls_back_to_default():
for timeout in (0, -1, math.nan, math.inf, -math.inf, "invalid"):
parsed = _dataset_from_dict({
"url": "https://example.test/daily",
"timeout": timeout,
})
assert parsed.timeout == 30.0
@pytest.mark.parametrize("timeout", [0, -1, math.nan, math.inf, -math.inf, "invalid"])
def test_sanitizer_drops_invalid_timeout_and_realtime_request_params(timeout):
cleaned = _sanitize_for_yaml({
"name": "test_source",
"datasets": {
"realtime": {
"url": "https://example.test/realtime",
"timeout": timeout,
"symbols_param": "codes",
"start_param": "from",
"end_param": "to",
},
},
})
dataset = cleaned["datasets"]["realtime"]
assert "timeout" not in dataset
assert "symbols_param" not in dataset
assert "start_param" not in dataset
assert "end_param" not in dataset
def test_empty_request_parameter_names_restore_defaults():
cleaned = _sanitize_for_yaml({
"name": "test_source",
"datasets": {
"minute": {
"url": "https://example.test/minute",
"symbols_param": " ",
"start_param": "\t",
"end_param": "",
},
},
})
parsed = _dataset_from_dict(cleaned["datasets"]["minute"])
assert parsed.symbols_param == "symbols"
assert parsed.start_param == "start_time"
assert parsed.end_param == "end_time"
@@ -1,7 +1,7 @@
import { useState, useEffect, useRef } from 'react'
import { useMutation, useQuery } from '@tanstack/react-query'
import { motion, AnimatePresence } from 'framer-motion'
import { KeyRound, Play, Plus, Save, Trash2, X, Zap, Check, ChevronDown } from 'lucide-react'
import { KeyRound, Play, Plus, Save, Trash2, X, Zap, Check, ChevronDown } from 'lucide-react'
import { api, type CustomSourceConfig, type DatasetConfig } from '@/lib/api'
import { toast } from '@/components/Toast'
@@ -96,6 +96,9 @@ export function DataSourceEditor({
if (!ds.url.trim()) {
throw new Error(`数据集「${DATASET_LABEL[key as DatasetKey] || key}」未填写接口 URL`)
}
if (ds.timeout != null && (!Number.isFinite(ds.timeout) || ds.timeout <= 0)) {
throw new Error(`数据集「${DATASET_LABEL[key as DatasetKey] || key}」超时必须大于 0 秒`)
}
}
// 提交时去掉 field_map 里的 __pending_ 临时 key (未填外部字段名的草稿行)
const cleaned: CustomSourceConfig = {
@@ -103,15 +106,23 @@ export function DataSourceEditor({
name: config.name.toLowerCase().trim(),
display_name: config.display_name.trim() || config.name.toLowerCase().trim(),
datasets: Object.fromEntries(
Object.entries(config.datasets).map(([k, ds]) => [
k,
{
...ds,
field_map: Object.fromEntries(
Object.entries(ds.field_map).filter(([src]) => !src.startsWith('__pending_'))
),
},
])
Object.entries(config.datasets).map(([k, ds]) => {
const normalized = { ...ds }
if (k === 'realtime') {
delete normalized.symbols_param
delete normalized.start_param
delete normalized.end_param
}
return [
k,
{
...normalized,
field_map: Object.fromEntries(
Object.entries(ds.field_map).filter(([src]) => !src.startsWith('__pending_'))
),
},
]
})
),
}
return api.saveDataSource(cleaned)
@@ -390,6 +401,8 @@ function DatasetDetail({
<Field label="超时">
<input
type="number"
min="0.1"
step="any"
value={cfg.timeout ?? ''}
onChange={e => onUpdate({ timeout: e.target.value ? Number(e.target.value) : null })}
onWheel={e => e.currentTarget.blur()}