mirror of
https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git
synced 2026-09-12 15:34:16 +08:00
fix(data-source): validate custom request settings
This commit is contained in:
@@ -8,7 +8,7 @@ import logging
|
||||
import time
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app import secrets_store
|
||||
from app.tickflow import client as tf_client
|
||||
@@ -373,7 +373,7 @@ class DatasetConfigIn(BaseModel):
|
||||
end_param: str = "end_time"
|
||||
asset_type_param: str | None = None
|
||||
freq_param: str | None = None
|
||||
timeout: float | None = None
|
||||
timeout: float | None = Field(default=None, gt=0, allow_inf_nan=False)
|
||||
|
||||
|
||||
class AuthConfigIn(BaseModel):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Custom HTTP data source configuration."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
@@ -60,22 +61,29 @@ def _auth_from_dict(raw: dict[str, Any] | None) -> AuthConfig:
|
||||
|
||||
|
||||
def _dataset_from_dict(raw: dict[str, Any]) -> DatasetConfig:
|
||||
try:
|
||||
timeout = float(raw.get("timeout", 30.0) or 30.0)
|
||||
except (TypeError, ValueError):
|
||||
timeout = 30.0
|
||||
if not math.isfinite(timeout) or timeout <= 0:
|
||||
timeout = 30.0
|
||||
|
||||
return DatasetConfig(
|
||||
url=str(raw.get("url", "") or ""),
|
||||
method=str(raw.get("method", "GET") or "GET").upper(),
|
||||
batch=int(raw["batch"]) if raw.get("batch") is not None else None,
|
||||
rpm=int(raw["rpm"]) if raw.get("rpm") is not None else None,
|
||||
timeout=float(raw.get("timeout", 30.0) or 30.0),
|
||||
timeout=timeout,
|
||||
response_path=str(raw.get("response_path", "") or ""),
|
||||
field_map={str(k): str(v) for k, v in (raw.get("field_map") or {}).items()},
|
||||
transforms={str(k): str(v) for k, v in (raw.get("transforms") or {}).items()},
|
||||
params=dict(raw.get("params") or {}),
|
||||
body=dict(raw.get("body") or {}),
|
||||
symbols_param=str(raw.get("symbols_param", "symbols") or "symbols"),
|
||||
start_param=str(raw.get("start_param", "start_time") or "start_time"),
|
||||
end_param=str(raw.get("end_param", "end_time") or "end_time"),
|
||||
asset_type_param=str(raw.get("asset_type_param")) if raw.get("asset_type_param") else None,
|
||||
freq_param=str(raw.get("freq_param")) if raw.get("freq_param") else None,
|
||||
symbols_param=str(raw.get("symbols_param", "symbols") or "symbols").strip() or "symbols",
|
||||
start_param=str(raw.get("start_param", "start_time") or "start_time").strip() or "start_time",
|
||||
end_param=str(raw.get("end_param", "end_time") or "end_time").strip() or "end_time",
|
||||
asset_type_param=(str(raw.get("asset_type_param") or "").strip() or None),
|
||||
freq_param=(str(raw.get("freq_param") or "").strip() or None),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -294,11 +295,13 @@ def _config_to_dict(config: CustomSourceConfig) -> dict:
|
||||
"response_path": ds.response_path,
|
||||
"field_map": dict(ds.field_map),
|
||||
**({"transforms": dict(ds.transforms)} if ds.transforms else {}),
|
||||
"symbols_param": ds.symbols_param,
|
||||
"start_param": ds.start_param,
|
||||
"end_param": ds.end_param,
|
||||
**({"asset_type_param": ds.asset_type_param} if ds.asset_type_param else {}),
|
||||
**({"freq_param": ds.freq_param} if ds.freq_param else {}),
|
||||
**({
|
||||
"symbols_param": ds.symbols_param,
|
||||
"start_param": ds.start_param,
|
||||
"end_param": ds.end_param,
|
||||
} if ds_name != "realtime" else {}),
|
||||
**({"asset_type_param": ds.asset_type_param} if ds_name == "minute" and ds.asset_type_param else {}),
|
||||
**({"freq_param": ds.freq_param} if ds_name == "minute" and ds.freq_param else {}),
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -358,14 +361,14 @@ def _sanitize_for_yaml(config: dict) -> dict:
|
||||
continue
|
||||
if not isinstance(ds_cfg, dict):
|
||||
continue
|
||||
ds = _sanitize_dataset(ds_cfg)
|
||||
ds = _sanitize_dataset(ds_name, ds_cfg)
|
||||
if ds:
|
||||
datasets_out[ds_name] = ds
|
||||
out["datasets"] = datasets_out
|
||||
return out
|
||||
|
||||
|
||||
def _sanitize_dataset(ds_cfg: dict) -> dict:
|
||||
def _sanitize_dataset(ds_name: str, ds_cfg: dict) -> dict:
|
||||
out: dict = {}
|
||||
url = str(ds_cfg.get("url", "") or "").strip()
|
||||
if not url:
|
||||
@@ -385,7 +388,9 @@ def _sanitize_dataset(ds_cfg: dict) -> dict:
|
||||
pass
|
||||
if ds_cfg.get("timeout") is not None:
|
||||
try:
|
||||
out["timeout"] = float(ds_cfg["timeout"])
|
||||
timeout = float(ds_cfg["timeout"])
|
||||
if math.isfinite(timeout) and timeout > 0:
|
||||
out["timeout"] = timeout
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
out["response_path"] = str(ds_cfg.get("response_path", "") or "")
|
||||
@@ -403,16 +408,23 @@ def _sanitize_dataset(ds_cfg: dict) -> dict:
|
||||
}
|
||||
if transforms:
|
||||
out["transforms"] = transforms
|
||||
if ds_cfg.get("symbols_param"):
|
||||
out["symbols_param"] = str(ds_cfg["symbols_param"])
|
||||
if ds_cfg.get("start_param"):
|
||||
out["start_param"] = str(ds_cfg["start_param"])
|
||||
if ds_cfg.get("end_param"):
|
||||
out["end_param"] = str(ds_cfg["end_param"])
|
||||
if ds_cfg.get("asset_type_param"):
|
||||
out["asset_type_param"] = str(ds_cfg["asset_type_param"])
|
||||
if ds_cfg.get("freq_param"):
|
||||
out["freq_param"] = str(ds_cfg["freq_param"])
|
||||
if ds_name != "realtime":
|
||||
symbols_param = str(ds_cfg.get("symbols_param") or "").strip()
|
||||
start_param = str(ds_cfg.get("start_param") or "").strip()
|
||||
end_param = str(ds_cfg.get("end_param") or "").strip()
|
||||
if symbols_param:
|
||||
out["symbols_param"] = symbols_param
|
||||
if start_param:
|
||||
out["start_param"] = start_param
|
||||
if end_param:
|
||||
out["end_param"] = end_param
|
||||
if ds_name == "minute":
|
||||
asset_type_param = str(ds_cfg.get("asset_type_param") or "").strip()
|
||||
freq_param = str(ds_cfg.get("freq_param") or "").strip()
|
||||
if asset_type_param:
|
||||
out["asset_type_param"] = asset_type_param
|
||||
if freq_param:
|
||||
out["freq_param"] = freq_param
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
import math
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.api.settings import DatasetConfigIn
|
||||
from app.data_providers.custom.config import CustomSourceConfig, _dataset_from_dict
|
||||
from app.data_providers.custom.loader import _config_to_dict, _sanitize_for_yaml
|
||||
@@ -66,4 +71,64 @@ def test_timeout_survives_config_round_trip():
|
||||
datasets={"realtime": parsed2},
|
||||
))
|
||||
assert parsed2.timeout == 30.0
|
||||
assert "timeout" not in exposed2["datasets"]["realtime"]
|
||||
realtime = exposed2["datasets"]["realtime"]
|
||||
assert "timeout" not in realtime
|
||||
assert "symbols_param" not in realtime
|
||||
assert "start_param" not in realtime
|
||||
assert "end_param" not in realtime
|
||||
|
||||
|
||||
@pytest.mark.parametrize("timeout", [0, -1, math.nan, math.inf, -math.inf])
|
||||
def test_timeout_api_rejects_non_positive_or_non_finite_values(timeout):
|
||||
with pytest.raises(ValidationError):
|
||||
DatasetConfigIn(url="https://example.test/daily", timeout=timeout)
|
||||
|
||||
|
||||
def test_invalid_yaml_timeout_falls_back_to_default():
|
||||
for timeout in (0, -1, math.nan, math.inf, -math.inf, "invalid"):
|
||||
parsed = _dataset_from_dict({
|
||||
"url": "https://example.test/daily",
|
||||
"timeout": timeout,
|
||||
})
|
||||
assert parsed.timeout == 30.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("timeout", [0, -1, math.nan, math.inf, -math.inf, "invalid"])
|
||||
def test_sanitizer_drops_invalid_timeout_and_realtime_request_params(timeout):
|
||||
cleaned = _sanitize_for_yaml({
|
||||
"name": "test_source",
|
||||
"datasets": {
|
||||
"realtime": {
|
||||
"url": "https://example.test/realtime",
|
||||
"timeout": timeout,
|
||||
"symbols_param": "codes",
|
||||
"start_param": "from",
|
||||
"end_param": "to",
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
dataset = cleaned["datasets"]["realtime"]
|
||||
assert "timeout" not in dataset
|
||||
assert "symbols_param" not in dataset
|
||||
assert "start_param" not in dataset
|
||||
assert "end_param" not in dataset
|
||||
|
||||
|
||||
def test_empty_request_parameter_names_restore_defaults():
|
||||
cleaned = _sanitize_for_yaml({
|
||||
"name": "test_source",
|
||||
"datasets": {
|
||||
"minute": {
|
||||
"url": "https://example.test/minute",
|
||||
"symbols_param": " ",
|
||||
"start_param": "\t",
|
||||
"end_param": "",
|
||||
},
|
||||
},
|
||||
})
|
||||
parsed = _dataset_from_dict(cleaned["datasets"]["minute"])
|
||||
|
||||
assert parsed.symbols_param == "symbols"
|
||||
assert parsed.start_param == "start_time"
|
||||
assert parsed.end_param == "end_time"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useState, useEffect, useRef } from 'react'
|
||||
import { useMutation, useQuery } from '@tanstack/react-query'
|
||||
import { motion, AnimatePresence } from 'framer-motion'
|
||||
import { KeyRound, Play, Plus, Save, Trash2, X, Zap, Check, ChevronDown } from 'lucide-react'
|
||||
import { KeyRound, Play, Plus, Save, Trash2, X, Zap, Check, ChevronDown } from 'lucide-react'
|
||||
import { api, type CustomSourceConfig, type DatasetConfig } from '@/lib/api'
|
||||
import { toast } from '@/components/Toast'
|
||||
|
||||
@@ -96,6 +96,9 @@ export function DataSourceEditor({
|
||||
if (!ds.url.trim()) {
|
||||
throw new Error(`数据集「${DATASET_LABEL[key as DatasetKey] || key}」未填写接口 URL`)
|
||||
}
|
||||
if (ds.timeout != null && (!Number.isFinite(ds.timeout) || ds.timeout <= 0)) {
|
||||
throw new Error(`数据集「${DATASET_LABEL[key as DatasetKey] || key}」超时必须大于 0 秒`)
|
||||
}
|
||||
}
|
||||
// 提交时去掉 field_map 里的 __pending_ 临时 key (未填外部字段名的草稿行)
|
||||
const cleaned: CustomSourceConfig = {
|
||||
@@ -103,15 +106,23 @@ export function DataSourceEditor({
|
||||
name: config.name.toLowerCase().trim(),
|
||||
display_name: config.display_name.trim() || config.name.toLowerCase().trim(),
|
||||
datasets: Object.fromEntries(
|
||||
Object.entries(config.datasets).map(([k, ds]) => [
|
||||
k,
|
||||
{
|
||||
...ds,
|
||||
field_map: Object.fromEntries(
|
||||
Object.entries(ds.field_map).filter(([src]) => !src.startsWith('__pending_'))
|
||||
),
|
||||
},
|
||||
])
|
||||
Object.entries(config.datasets).map(([k, ds]) => {
|
||||
const normalized = { ...ds }
|
||||
if (k === 'realtime') {
|
||||
delete normalized.symbols_param
|
||||
delete normalized.start_param
|
||||
delete normalized.end_param
|
||||
}
|
||||
return [
|
||||
k,
|
||||
{
|
||||
...normalized,
|
||||
field_map: Object.fromEntries(
|
||||
Object.entries(ds.field_map).filter(([src]) => !src.startsWith('__pending_'))
|
||||
),
|
||||
},
|
||||
]
|
||||
})
|
||||
),
|
||||
}
|
||||
return api.saveDataSource(cleaned)
|
||||
@@ -390,6 +401,8 @@ function DatasetDetail({
|
||||
<Field label="超时">
|
||||
<input
|
||||
type="number"
|
||||
min="0.1"
|
||||
step="any"
|
||||
value={cfg.timeout ?? ''}
|
||||
onChange={e => onUpdate({ timeout: e.target.value ? Number(e.target.value) : null })}
|
||||
onWheel={e => e.currentTarget.blur()}
|
||||
|
||||
Reference in New Issue
Block a user