feat(ext-data): 新增扩展数据支持 URL 创建并调整设置弹窗 tab (#64)

- 新建扩展数据弹窗加入 URL/文件/手动 三种接入方式, URL 模式支持探测字段、预览、保存拉取配置并可选立即导入或定时拉取
- 后端新增 /api/ext-data/detect-url 探测接口, 自动识别 JSON 行数组路径、字段类型与 symbol/code 候选列
- 抽出 infer_fields_from_df / detect_symbol_candidates / apply_config_mapping 公共逻辑, 文件上传与 URL 探测复用
- 拉取写入支持仅含 code 的数据, 按 code_map 自动生成标准 symbol
- 设置弹窗 tab 顺序调整为 拉取/推送/上传, 默认显示拉取
This commit is contained in:
wshy
2026-07-07 18:04:49 +08:00
committed by GitHub
parent f8914c9140
commit 5b2282f8cb
5 changed files with 723 additions and 210 deletions
+105 -87
View File
@@ -19,9 +19,11 @@ from app.services.ext_data import (
ExtConfigStore,
ExtField,
PullConfig,
apply_config_mapping,
detect_symbol_candidates,
ensure_utf8_csv,
fix_symbol_format,
normalize_symbol,
infer_fields_from_df,
parse_upload_file,
write_ext_parquet,
rows_to_parquet,
@@ -78,6 +80,16 @@ class PullConfigReq(BaseModel):
enabled: bool = False
class DetectUrlReq(BaseModel):
"""URL 探测请求,不依赖已存在的扩展配置。"""
url: str = Field(..., min_length=1)
method: str = "GET"
headers: dict[str, str] | None = None
body: str | None = None
response_path: str = ""
field_map: dict[str, str] | None = None
# ---------------------------------------------------------------------------
# 辅助
# ---------------------------------------------------------------------------
@@ -95,52 +107,7 @@ def _data_dir(request: Request) -> Path:
# ---------------------------------------------------------------------------
def _apply_mapping(df: pl.DataFrame, config: ExtConfig, data_dir: Path) -> pl.DataFrame:
"""根据 config 的 symbol_map / code_map 自动生成 symbol 和 code 列。
执行顺序:先 mapped(从文件列复制),再 computed(从已生成的列计算)。
"""
sm = config.symbol_map or {}
cm = config.code_map or {}
# --- 第一步:mapped 类型,直接从文件列映射 ---
if sm.get("type") == "mapped" and sm["col"] in df.columns:
df = df.with_columns(df[sm["col"]].cast(pl.Utf8).alias("symbol"))
if cm.get("type") == "mapped" and cm["col"] in df.columns:
df = df.with_columns(df[cm["col"]].cast(pl.Utf8).alias("code"))
# --- 第二步:computed 类型,从已生成的列计算 ---
if "symbol" not in df.columns and sm.get("type") == "computed":
if sm.get("from") == "code" and "code" in df.columns:
# code → symbol: 000001 → 000001.SZ
from app.services.ext_data import build_code_lookup
lookup = build_code_lookup(data_dir)
df = df.with_columns(normalize_symbol(df["code"].cast(pl.Utf8), lookup).alias("symbol"))
if "code" not in df.columns and cm.get("type") == "computed":
if cm.get("from") == "symbol" and "symbol" in df.columns:
# symbol → code: 000001.SZ → 000001
df = df.with_columns(
df["symbol"].cast(pl.Utf8).str.split(".").list.first().alias("code")
)
# --- 兜底:如果只有一个,自动生成另一个 ---
if "symbol" in df.columns and "code" not in df.columns:
df = df.with_columns(
df["symbol"].cast(pl.Utf8).str.split(".").list.first().alias("code")
)
elif "code" in df.columns and "symbol" not in df.columns:
from app.services.ext_data import build_code_lookup
lookup = build_code_lookup(data_dir)
df = df.with_columns(normalize_symbol(df["code"].cast(pl.Utf8), lookup).alias("symbol"))
# 标准化 symbol 列
if "symbol" in df.columns:
from app.services.ext_data import build_code_lookup
lookup = build_code_lookup(data_dir)
df = df.with_columns(normalize_symbol(df["symbol"].cast(pl.Utf8), lookup))
return df
return apply_config_mapping(df, config, data_dir)
def _clean_col_names(df: pl.DataFrame) -> pl.DataFrame:
@@ -681,16 +648,6 @@ def fix_symbol(request: Request, config_id: str):
# Schema 发现
# ---------------------------------------------------------------------------
_POLARS_TYPE_MAP = {
"Int64": "int", "Int32": "int", "Int16": "int", "Int8": "int",
"UInt64": "int", "UInt32": "int", "UInt16": "int", "UInt8": "int",
"Float64": "float", "Float32": "float",
"Boolean": "bool",
"Utf8": "string", "String": "string",
"Date": "string", "Datetime": "string", "Duration": "string",
"Categorical": "string",
}
@router.post("/detect-fields")
async def detect_fields(
@@ -729,44 +686,105 @@ async def detect_fields(
# 清洗列名:去掉括号内的时间戳等信息
df = _clean_col_names(df)
# 构建字段列表
fields = []
for col_name in df.columns:
pl_type = df[col_name].dtype
dtype = _POLARS_TYPE_MAP.get(str(pl_type.base_type()), "string")
fields.append({"name": col_name, "dtype": dtype, "label": col_name})
# --- 分别检测 symbol 候选 (000001.SZ) 和 code 候选 (6位纯数字) ---
import re
_CODE_PAT = re.compile(r"^\d{6}$")
_SYMBOL_PAT = re.compile(r"^\d{6}\.[A-Z]{2}$")
symbol_candidates: list[str] = [] # 数据匹配 000001.SZ 格式
code_candidates: list[str] = [] # 数据匹配 000001 格式
for col in df.columns:
col_data = df[col].cast(pl.Utf8).drop_nulls()
if len(col_data) == 0:
continue
sample = col_data.head(200).to_list()
sym_hits = sum(1 for v in sample if _SYMBOL_PAT.match(str(v).strip()))
code_hits = sum(1 for v in sample if _CODE_PAT.match(str(v).strip()))
total = len(sample)
if total > 0:
if sym_hits / total > 0.5:
symbol_candidates.append(col)
elif code_hits / total > 0.5:
code_candidates.append(col)
symbol_candidates, code_candidates = detect_symbol_candidates(df)
return {
"fields": fields,
"fields": infer_fields_from_df(df),
"rows": len(df),
"symbol_candidates": symbol_candidates,
"code_candidates": code_candidates,
}
def _find_row_arrays(data, prefix: str = "", limit: int = 8) -> list[str]:
"""自动寻找 JSON 中可能的数据数组路径。"""
found: list[str] = []
def walk(value, path: str) -> None:
if len(found) >= limit:
return
if isinstance(value, list):
if value and isinstance(value[0], dict):
found.append(path)
elif value and isinstance(value[0], list):
for i, item in enumerate(value[:3]):
walk(item, f"{path}.{i}" if path else str(i))
elif isinstance(value, dict):
for key, child in value.items():
next_path = f"{path}.{key}" if path else key
walk(child, next_path)
walk(data, prefix)
return found
@router.post("/detect-url")
async def detect_url(body: DetectUrlReq):
"""请求外部 URL,自动检测 JSON 行数据的字段和标的代码列。"""
from app.services.ext_pull import _extract_rows, _apply_field_map
import httpx
method = body.method.upper()
if method not in ("GET", "POST"):
raise HTTPException(400, "仅支持 GET / POST")
try:
async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
headers = body.headers or {}
kwargs: dict = {"headers": headers}
if method == "POST" and body.body:
kwargs["content"] = body.body
if "content-type" not in {k.lower() for k in headers}:
kwargs["headers"]["Content-Type"] = "application/json"
resp = await client.request(method, body.url, **kwargs)
resp.raise_for_status()
data = resp.json()
except Exception as e:
raise HTTPException(400, f"URL 请求失败: {e}") from e
path_candidates = _find_row_arrays(data)
response_path = body.response_path
if not response_path:
if not path_candidates:
raise HTTPException(400, "未在响应中找到对象数组,请填写响应数据路径")
response_path = path_candidates[0]
try:
rows = _extract_rows(data, response_path)
rows = _apply_field_map(rows, body.field_map or {})
except Exception as e:
raise HTTPException(400, f"响应解析失败: {e}") from e
if not rows:
raise HTTPException(400, "提取到的行数为 0")
if not all(isinstance(row, dict) for row in rows[:200]):
raise HTTPException(400, "响应数据数组中的元素必须是对象")
sample_rows = rows[: min(len(rows), 500)]
try:
df = pl.DataFrame(sample_rows)
except Exception as e:
raise HTTPException(400, f"样例数据解析失败: {e}") from e
df = _clean_col_names(df)
symbol_candidates, code_candidates = detect_symbol_candidates(df)
preview = [
{k: _safe_json_value(v) for k, v in row.items()}
for row in df.head(10).to_dicts()
]
return {
"status": "ok",
"total_rows": len(rows),
"response_path": response_path,
"response_path_candidates": path_candidates,
"fields": infer_fields_from_df(df),
"symbol_candidates": symbol_candidates,
"code_candidates": code_candidates,
"preview": preview,
}
@router.get("/schema/{config_id}")
def discover_schema(request: Request, config_id: str):
"""发现扩展数据的实际 Parquet schema(基于已有数据)。"""
+87
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import json
import logging
import re
from datetime import date, datetime
from pathlib import Path
from typing import Literal
@@ -266,6 +267,54 @@ _POLARS_DTYPE_MAP = {
"bool": pl.Boolean,
}
_POLARS_TYPE_MAP = {
"Int64": "int", "Int32": "int", "Int16": "int", "Int8": "int",
"UInt64": "int", "UInt32": "int", "UInt16": "int", "UInt8": "int",
"Float64": "float", "Float32": "float",
"Boolean": "bool",
"Utf8": "string", "String": "string",
"Date": "string", "Datetime": "string", "Duration": "string",
"Categorical": "string",
}
_CODE_PAT = re.compile(r"^\d{6}$")
_SYMBOL_PAT = re.compile(r"^\d{6}\.[A-Z]{2}$")
def infer_fields_from_df(df: pl.DataFrame) -> list[dict]:
"""从 DataFrame 推断扩展字段定义。"""
fields = []
for col_name in df.columns:
pl_type = df[col_name].dtype
dtype = _POLARS_TYPE_MAP.get(str(pl_type.base_type()), "string")
fields.append({"name": col_name, "dtype": dtype, "label": col_name})
return fields
def detect_symbol_candidates(df: pl.DataFrame) -> tuple[list[str], list[str]]:
"""识别 symbol/code 候选列。"""
symbol_candidates: list[str] = []
code_candidates: list[str] = []
for col in df.columns:
try:
col_data = df[col].cast(pl.Utf8).drop_nulls()
except Exception:
continue
if len(col_data) == 0:
continue
sample = col_data.head(200).to_list()
sym_hits = sum(1 for v in sample if _SYMBOL_PAT.match(str(v).strip()))
code_hits = sum(1 for v in sample if _CODE_PAT.match(str(v).strip()))
total = len(sample)
if total > 0:
if sym_hits / total > 0.5:
symbol_candidates.append(col)
elif code_hits / total > 0.5:
code_candidates.append(col)
return symbol_candidates, code_candidates
def build_code_lookup(data_dir: Path) -> dict[str, str]:
"""从 instruments 维表构建 code → symbol 映射。"""
@@ -309,6 +358,43 @@ def normalize_symbol(series: pl.Series, lookup: dict[str, str] | None = None) ->
return series.map_elements(_fix_one, return_dtype=pl.Utf8)
def apply_config_mapping(df: pl.DataFrame, config: ExtConfig, data_dir: Path) -> pl.DataFrame:
"""根据 config 的 symbol_map / code_map 自动生成 symbol 和 code 列。"""
sm = config.symbol_map or {}
cm = config.code_map or {}
if sm.get("type") == "mapped" and sm["col"] in df.columns:
df = df.with_columns(df[sm["col"]].cast(pl.Utf8).alias("symbol"))
if cm.get("type") == "mapped" and cm["col"] in df.columns:
df = df.with_columns(df[cm["col"]].cast(pl.Utf8).alias("code"))
if "symbol" not in df.columns and sm.get("type") == "computed":
if sm.get("from") == "code" and "code" in df.columns:
lookup = build_code_lookup(data_dir)
df = df.with_columns(normalize_symbol(df["code"].cast(pl.Utf8), lookup).alias("symbol"))
if "code" not in df.columns and cm.get("type") == "computed":
if cm.get("from") == "symbol" and "symbol" in df.columns:
df = df.with_columns(
df["symbol"].cast(pl.Utf8).str.split(".").list.first().alias("code")
)
if "symbol" in df.columns and "code" not in df.columns:
df = df.with_columns(
df["symbol"].cast(pl.Utf8).str.split(".").list.first().alias("code")
)
elif "code" in df.columns and "symbol" not in df.columns:
lookup = build_code_lookup(data_dir)
df = df.with_columns(normalize_symbol(df["code"].cast(pl.Utf8), lookup).alias("symbol"))
if "symbol" in df.columns:
lookup = build_code_lookup(data_dir)
df = df.with_columns(normalize_symbol(df["symbol"].cast(pl.Utf8), lookup))
return df
def ensure_utf8_csv(file_path: Path) -> Path:
"""确保 CSV 文件以 UTF-8 编码可读,非 UTF-8(如 GBK/GB18030)则转换。
@@ -516,6 +602,7 @@ def rows_to_parquet(
写入行数。
"""
df = pl.DataFrame(rows)
df = apply_config_mapping(df, config, data_dir)
if "symbol" in df.columns:
df = df.with_columns(pl.col("symbol").cast(pl.Utf8))
return write_ext_parquet(df, config, data_dir, snapshot_date=snapshot_date)
+9 -3
View File
@@ -133,9 +133,15 @@ async def fetch_and_ingest(
# 字段映射
rows = _apply_field_map(rows, pull.field_map)
# 校验 symbol 列
if rows and "symbol" not in rows[0]:
raise ValueError("数据行中缺少 symbol 字段,请配置 field_map 映射")
# 校验可关联标的的字段:直接 symbol/code,或配置里声明的映射源列。
row_keys = set(rows[0]) if rows else set()
mapped_cols = {
m.get("col")
for m in (config.symbol_map or {}, config.code_map or {})
if m.get("type") == "mapped" and m.get("col")
}
if rows and not ({"symbol", "code"} & row_keys or mapped_cols & row_keys):
raise ValueError("数据行中缺少 symbol/code 字段,请配置字段映射或标的映射")
# 写入
snap = date.today()
@@ -1,17 +1,38 @@
import { useRef, useState } from 'react'
import { useMutation, useQueryClient } from '@tanstack/react-query'
import { motion } from 'framer-motion'
import { X, Loader2, Upload, Plus, AlertCircle, Tag, Clock } from 'lucide-react'
import { api, type ExtDataField } from '@/lib/api'
import {
X,
Loader2,
Upload,
Plus,
AlertCircle,
Tag,
Clock,
Link2,
FileText,
Keyboard,
RefreshCw,
} from 'lucide-react'
import { api, type ExtDataDetectUrlResult, type ExtDataField } from '@/lib/api'
import { QK } from '@/lib/queryKeys'
type SourceMode = 'url' | 'file' | 'manual'
type MappingChoice = {
fields: { name: string; dtype: string; label: string }[]
need: 'symbol' | 'code' | 'both'
}
export function CreateExtDialog({ onClose }: { onClose: () => void }) {
const qc = useQueryClient()
const [sourceMode, setSourceMode] = useState<SourceMode>('url')
const [id, setId] = useState('')
const [label, setLabel] = useState('')
const [description, setDescription] = useState('')
const [mode, setMode] = useState<'snapshot' | 'timeseries'>('snapshot')
const [fields, setFields] = useState<ExtDataField[]>([])
const [detectedSourceNames, setDetectedSourceNames] = useState<string[]>([])
const [error, setError] = useState('')
const detectFileRef = useRef<HTMLInputElement>(null)
const [detecting, setDetecting] = useState(false)
@@ -19,12 +40,74 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
const [symbolMap, setSymbolMap] = useState<Record<string, string>>({})
const [codeMap, setCodeMap] = useState<Record<string, string>>({})
const [matchStatus, setMatchStatus] = useState<'none' | 'partial' | 'full'>('none')
const [selectMapping, setSelectMapping] = useState<{ fields: { name: string; dtype: string; label: string }[]; need: 'symbol' | 'code' | 'both' } | null>(null)
const [selectMapping, setSelectMapping] = useState<MappingChoice | null>(null)
const [url, setUrl] = useState('')
const [method, setMethod] = useState<'GET' | 'POST'>('GET')
const [headerStr, setHeaderStr] = useState('')
const [body, setBody] = useState('')
const [responsePath, setResponsePath] = useState('')
const [fieldMapStr, setFieldMapStr] = useState('')
const [schedule, setSchedule] = useState(1440)
const [savePull, setSavePull] = useState(true)
const [importNow, setImportNow] = useState(true)
const [enablePull, setEnablePull] = useState(false)
const [urlPreview, setUrlPreview] = useState<ExtDataDetectUrlResult | null>(null)
const userFields = fields.filter(f => f.name !== 'symbol' && f.name !== 'code')
const parseJsonObject = (str: string, labelText: string): Record<string, string> | undefined => {
if (!str.trim()) return undefined
try {
const parsed = JSON.parse(str)
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new Error('not object')
}
return Object.fromEntries(
Object.entries(parsed).map(([k, v]) => [k, String(v)]),
)
} catch {
throw new Error(`${labelText} 不是有效 JSON 对象`)
}
}
const buildPullFieldMap = () => {
const base = parseJsonObject(fieldMapStr, '字段映射') ?? {}
const finalMap: Record<string, string> = { ...base }
fields.forEach((field, index) => {
const source = detectedSourceNames[index]
const target = field.name.trim()
if (!source || !target || source === target) return
const upstreamKeys = Object.entries(base)
.filter(([, mapped]) => mapped === source)
.map(([raw]) => raw)
if (upstreamKeys.length) {
upstreamKeys.forEach((raw) => { finalMap[raw] = target })
} else {
finalMap[source] = target
}
})
return finalMap
}
const mapSubmitCol = (map: Record<string, string>, fieldMap: Record<string, string>) => {
if (sourceMode !== 'url' || map.type !== 'mapped' || !map.col) return map
const mappedCol = fieldMap[map.col] ?? map.col
return { ...map, col: mappedCol }
}
const create = useMutation({
mutationFn: () => {
mutationFn: async () => {
const userF = fields.filter((f) => f.name.trim() && f.name !== 'symbol' && f.name !== 'code')
return api.extDataCreate({
const pullFieldMap = sourceMode === 'url' ? buildPullFieldMap() : {}
const submittedSymbolMap = mapSubmitCol(symbolMap, pullFieldMap)
const submittedCodeMap = mapSubmitCol(codeMap, pullFieldMap)
const config = await api.extDataCreate({
id,
label,
mode,
@@ -34,27 +117,60 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
...userF,
],
description: description.trim() || undefined,
symbol_map: symbolMap,
code_map: codeMap,
symbol_map: submittedSymbolMap,
code_map: submittedCodeMap,
})
if (sourceMode === 'url' && (savePull || importNow || enablePull)) {
const headers = parseJsonObject(headerStr, 'Headers')
const finalFieldMap = Object.keys(pullFieldMap).length ? pullFieldMap : undefined
await api.extDataPullConfig(config.id, {
url: url.trim(),
method,
headers,
body: method === 'POST' && body.trim() ? body : undefined,
response_path: responsePath.trim() || urlPreview?.response_path || '',
field_map: finalFieldMap,
schedule_minutes: schedule,
enabled: enablePull,
})
if (importNow) {
await api.extDataPullRun(config.id)
}
}
return config
},
onSuccess: () => {
qc.invalidateQueries({ queryKey: QK.extData })
qc.invalidateQueries({ queryKey: QK.dataStatus })
onClose()
},
onError: (err) => setError(String(err)),
onError: (err) => setError(err instanceof Error ? err.message : String(err)),
})
const addField = () =>
const addField = () => {
setFields([...fields, { name: '', dtype: 'string', label: '' }])
setDetectedSourceNames([...detectedSourceNames, ''])
}
const removeField = (i: number) =>
const removeField = (i: number) => {
setFields(fields.filter((_, idx) => idx !== i))
setDetectedSourceNames(detectedSourceNames.filter((_, idx) => idx !== i))
}
const updateField = (i: number, key: keyof ExtDataField, val: string) =>
const updateField = (i: number, key: keyof ExtDataField, val: string) => {
setFields(fields.map((f, idx) => (idx === i ? { ...f, [key]: val } : f)))
}
const valid = id.trim() && label.trim() && fields.some((f) => f.name.trim()) && matchStatus !== 'none'
const valid = Boolean(
id.trim()
&& label.trim()
&& fields.some((f) => f.name.trim())
&& matchStatus !== 'none'
&& (sourceMode !== 'url' || url.trim()),
)
const processDetection = (
detected: { name: string; dtype: string; label: string }[],
@@ -65,6 +181,8 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
let cm: Record<string, string> = {}
let status: 'none' | 'partial' | 'full' = 'none'
setDetectedSourceNames(detected.map(f => f.name))
if (symCands.length === 1 && codeCands.length === 1) {
sm = { type: 'mapped', col: symCands[0] }
cm = { type: 'mapped', col: codeCands[0] }
@@ -97,11 +215,12 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
setSelectMapping(null)
}
const handleDetectFile = (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0]
if (!file) return
e.target.value = ''
setDetecting(true); setError(''); setSelectMapping(null)
const detectFile = (file: File) => {
setSourceMode('file')
setDetecting(true)
setError('')
setSelectMapping(null)
setUrlPreview(null)
api.extDataDetectFields(file)
.then((res) => {
processDetection(res.fields, res.symbol_candidates, res.code_candidates)
@@ -110,7 +229,82 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
.finally(() => setDetecting(false))
}
const userFields = fields.filter(f => f.name !== 'symbol' && f.name !== 'code')
const handleDetectFile = (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0]
if (!file) return
e.target.value = ''
detectFile(file)
}
const handleDetectUrl = () => {
if (!url.trim()) {
setError('请先填写 URL')
return
}
setDetecting(true)
setError('')
setSelectMapping(null)
setUrlPreview(null)
let headers: Record<string, string> | undefined
let fieldMap: Record<string, string> | undefined
try {
headers = parseJsonObject(headerStr, 'Headers')
fieldMap = parseJsonObject(fieldMapStr, '字段映射')
} catch (err) {
setError(err instanceof Error ? err.message : String(err))
setDetecting(false)
return
}
api.extDataDetectUrl({
url: url.trim(),
method,
headers,
body: method === 'POST' && body.trim() ? body : undefined,
response_path: responsePath.trim() || undefined,
field_map: fieldMap,
})
.then((res) => {
setUrlPreview(res)
setResponsePath(res.response_path || responsePath)
processDetection(res.fields, res.symbol_candidates, res.code_candidates)
})
.catch((err) => setError(String(err)))
.finally(() => setDetecting(false))
}
const selectSource = (next: SourceMode) => {
if (next !== sourceMode) {
setFields([])
setDetectedSourceNames([])
setSymbolMap({})
setCodeMap({})
setMatchStatus('none')
setSelectMapping(null)
setUrlPreview(null)
}
setSourceMode(next)
setError('')
if (next === 'manual') {
setSymbolMap({ type: 'mapped', col: 'symbol' })
setCodeMap({ type: 'computed', from: 'symbol', method: 'strip_exchange' })
setMatchStatus('full')
}
}
const applyManualMapping = (fieldName: string) => {
const sm = { type: 'mapped', col: fieldName }
const cm = { type: 'computed', from: 'symbol', method: 'strip_exchange' }
setSymbolMap(sm)
setCodeMap(cm)
setMatchStatus('full')
setSelectMapping(null)
}
const previewColumns = urlPreview?.preview?.[0]
? Object.keys(urlPreview.preview[0]).slice(0, 5)
: []
return (
<div className="fixed inset-0 z-50 flex items-center justify-center">
@@ -120,14 +314,14 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
animate={{ opacity: 1, scale: 1, y: 0 }}
exit={{ opacity: 0, scale: 0.97, y: 8 }}
transition={{ duration: 0.2, ease: [0.16, 1, 0.3, 1] }}
className="relative rounded-2xl border border-border bg-surface shadow-2xl mx-4 w-full max-w-xl max-h-[85vh] flex flex-col overflow-hidden"
className="relative rounded-2xl border border-border bg-surface shadow-2xl mx-4 w-full max-w-2xl max-h-[88vh] flex flex-col overflow-hidden"
>
<div className="px-6 pt-5 pb-4">
<div className="flex items-center justify-between">
<div>
<h3 className="text-base font-semibold text-foreground"></h3>
<p className="text-[11px] mt-1 inline-flex items-center gap-1 bg-amber-500/10 text-amber-400 px-2 py-0.5 rounded-md font-medium">
CSV/Excel
CSV/Excel
</p>
</div>
<button onClick={onClose} className="p-1 rounded-lg hover:bg-elevated text-secondary transition-colors">
@@ -137,6 +331,31 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
</div>
<div className="flex-1 overflow-y-auto px-6 pb-6 space-y-5">
<div>
<div className="text-[11px] font-medium text-secondary mb-2"></div>
<div className="grid grid-cols-3 gap-2 rounded-xl bg-elevated/40 p-1">
{([
['url', 'URL', Link2],
['file', '文件', FileText],
['manual', '手动', Keyboard],
] as const).map(([key, text, Icon]) => {
const active = sourceMode === key
return (
<button
key={key}
onClick={() => selectSource(key)}
className={`inline-flex items-center justify-center gap-1.5 rounded-lg px-3 py-2 text-xs font-medium transition-colors ${
active ? 'bg-surface text-foreground shadow-sm' : 'text-muted hover:text-secondary'
}`}
>
<Icon className="h-3.5 w-3.5" />
{text}
</button>
)
})}
</div>
</div>
<div>
<div className="text-[11px] font-medium text-secondary mb-2"></div>
<div className="grid grid-cols-2 gap-2">
@@ -204,8 +423,151 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
/>
</div>
{sourceMode === 'url' && (
<div className="rounded-xl border border-border/60 bg-elevated/20 p-3 space-y-3">
<div className="flex items-center gap-1.5 text-[11px] font-medium text-secondary">
<Link2 className="h-3.5 w-3.5 text-muted" />
<span>URL </span>
</div>
<div className="flex gap-2">
<select
value={method}
onChange={(e) => setMethod(e.target.value as 'GET' | 'POST')}
className="h-8 shrink-0 rounded-lg border border-border bg-base px-2 text-[11px] text-foreground"
>
<option value="GET">GET</option>
<option value="POST">POST</option>
</select>
<input
value={url}
onChange={(e) => setUrl(e.target.value)}
placeholder="https://api.example.com/data"
className="h-8 flex-1 min-w-0 rounded-lg border border-border bg-base px-3 text-[11px] font-mono text-foreground placeholder:text-muted/40 focus:outline-none focus:border-accent/50"
/>
<button
onClick={handleDetectUrl}
disabled={detecting || !url.trim()}
className="h-8 inline-flex items-center gap-1.5 rounded-lg bg-accent px-3 text-xs font-medium text-base hover:bg-accent/90 disabled:opacity-40 transition-colors"
>
{detecting ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <RefreshCw className="h-3.5 w-3.5" />}
</button>
</div>
<div className="grid grid-cols-2 gap-2">
<div>
<div className="text-[10px] text-muted mb-1"></div>
<input
value={responsePath}
onChange={(e) => setResponsePath(e.target.value)}
placeholder="data.list(可留空自动识别)"
className="w-full h-8 rounded-lg border border-border bg-base px-2 text-[10px] font-mono text-foreground placeholder:text-muted/40 focus:outline-none focus:border-accent/50"
/>
</div>
<div>
<div className="text-[10px] text-muted mb-1"></div>
<input
type="number"
min={1}
value={schedule}
onChange={(e) => setSchedule(Number(e.target.value) || 1)}
className="w-full h-8 rounded-lg border border-border bg-base px-2 text-[10px] font-mono text-foreground focus:outline-none focus:border-accent/50"
/>
</div>
</div>
<div className="grid grid-cols-2 gap-2">
<div>
<div className="text-[10px] text-muted mb-1">HeadersJSON</div>
<textarea
value={headerStr}
onChange={(e) => setHeaderStr(e.target.value)}
rows={2}
placeholder='{"Authorization":"Bearer xxx"}'
className="w-full rounded-lg border border-border bg-base px-2 py-1.5 text-[10px] font-mono text-foreground placeholder:text-muted/40 resize-none focus:outline-none focus:border-accent/50"
/>
</div>
<div>
<div className="text-[10px] text-muted mb-1"> </div>
<textarea
value={fieldMapStr}
onChange={(e) => setFieldMapStr(e.target.value)}
rows={2}
placeholder='{"code":"symbol","val":"score"}'
className="w-full rounded-lg border border-border bg-base px-2 py-1.5 text-[10px] font-mono text-foreground placeholder:text-muted/40 resize-none focus:outline-none focus:border-accent/50"
/>
</div>
</div>
{method === 'POST' && (
<div>
<div className="text-[10px] text-muted mb-1">JSON</div>
<textarea
value={body}
onChange={(e) => setBody(e.target.value)}
rows={3}
placeholder='{"page":1}'
className="w-full rounded-lg border border-border bg-base px-2 py-1.5 text-[10px] font-mono text-foreground placeholder:text-muted/40 resize-none focus:outline-none focus:border-accent/50"
/>
</div>
)}
{urlPreview && (
<div className="rounded-lg border border-border/50 bg-base/60 p-2.5 space-y-2">
<div className="flex items-center justify-between text-[10px]">
<span className="text-secondary"> {urlPreview.total_rows} · {urlPreview.response_path || '根数组'}</span>
{urlPreview.response_path_candidates.length > 1 && (
<span className="text-muted"> {urlPreview.response_path_candidates.length} </span>
)}
</div>
{previewColumns.length > 0 && (
<div className="overflow-x-auto rounded-md border border-border/40">
<table className="min-w-full text-left text-[10px]">
<thead className="bg-elevated/50 text-muted">
<tr>
{previewColumns.map(col => <th key={col} className="px-2 py-1 font-medium">{col}</th>)}
</tr>
</thead>
<tbody>
{urlPreview.preview.slice(0, 3).map((row, idx) => (
<tr key={idx} className="border-t border-border/30 text-secondary">
{previewColumns.map(col => (
<td key={col} className="max-w-[140px] truncate px-2 py-1 font-mono">
{row[col] == null ? '—' : String(row[col])}
</td>
))}
</tr>
))}
</tbody>
</table>
</div>
)}
</div>
)}
</div>
)}
<div>
<div className="text-[11px] font-medium text-secondary mb-2"></div>
<div className="flex items-center justify-between mb-2">
<div className="text-[11px] font-medium text-secondary"></div>
{(sourceMode === 'manual' || fields.length > 0) && (
<div className="flex items-center gap-1.5">
{sourceMode !== 'url' && (
<button
onClick={() => detectFileRef.current?.click()}
disabled={detecting}
className="inline-flex items-center gap-1 px-2 py-0.5 rounded-md text-[10px] text-muted hover:text-accent hover:bg-accent/[0.06] disabled:opacity-40 transition-colors"
>
{detecting ? <Loader2 className="h-3 w-3 animate-spin" /> : <Upload className="h-3 w-3" />}
</button>
)}
<button
onClick={addField}
className="inline-flex items-center gap-0.5 px-2 py-0.5 rounded-md text-[10px] text-muted hover:text-accent hover:bg-accent/[0.06] transition-colors"
>
<Plus className="h-3 w-3" />
</button>
</div>
)}
</div>
<input
ref={detectFileRef}
@@ -223,20 +585,13 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
className="mb-2 rounded-lg border border-amber-500/30 bg-amber-500/[0.06] px-3 py-2.5 space-y-2"
>
<div className="text-[10px] text-amber-400 font-medium">
symbol
</div>
<div className="flex flex-wrap gap-1.5">
{selectMapping.fields.map(f => (
<button
key={f.name}
onClick={() => {
const sm = { type: 'mapped', col: f.name }
const cm = { type: 'computed', from: 'symbol', method: 'strip_exchange' }
setSymbolMap(sm)
setCodeMap(cm)
setMatchStatus('full')
setSelectMapping(null)
}}
onClick={() => applyManualMapping(f.name)}
className="px-2.5 py-1 rounded-md bg-accent/10 text-accent text-[10px] font-medium hover:bg-accent/20 transition-colors"
>
{f.name}
@@ -252,7 +607,7 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
</motion.div>
)}
{userFields.length === 0 ? (
{sourceMode === 'file' && fields.length === 0 ? (
<div
onClick={() => detectFileRef.current?.click()}
onDragOver={e => { e.preventDefault(); setDragOver(true) }}
@@ -261,7 +616,7 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
e.preventDefault()
setDragOver(false)
const file = e.dataTransfer.files[0]
if (file) { setDetecting(true); setError(''); setSelectMapping(null); api.extDataDetectFields(file).then(res => { processDetection(res.fields, res.symbol_candidates, res.code_candidates); }).catch(err => setError(String(err))).finally(() => setDetecting(false)) }
if (file) detectFile(file)
}}
className={`rounded-xl border-2 border-dashed py-6 flex flex-col items-center justify-center gap-2 cursor-pointer transition-colors ${
dragOver ? 'border-accent bg-accent/[0.06]' : detecting ? 'border-border/40 pointer-events-none' : 'border-border/30 hover:border-accent/40 hover:bg-accent/[0.02]'
@@ -273,97 +628,144 @@ export function CreateExtDialog({ onClose }: { onClose: () => void }) {
<>
<Upload className="h-5 w-5 text-muted/60" />
<span className="text-[11px] text-secondary">CSV / Excel</span>
<span className="text-[10px] text-amber-400/70">symbol </span>
<span className="text-[10px] text-amber-400/70">symbol/code </span>
</>
)}
</div>
) : sourceMode === 'url' && fields.length === 0 ? (
<div className="rounded-xl border border-dashed border-border/40 bg-elevated/20 py-5 text-center text-[11px] text-muted">
URL
</div>
) : sourceMode === 'manual' && fields.length === 0 ? (
<div className="rounded-xl border border-dashed border-border/40 bg-elevated/20 py-5 flex flex-col items-center gap-2">
<div className="text-[11px] text-muted"></div>
<button
onClick={addField}
className="inline-flex items-center gap-1 rounded-lg bg-accent px-3 py-1.5 text-xs font-medium text-base hover:bg-accent/90 transition-colors"
>
<Plus className="h-3.5 w-3.5" />
</button>
</div>
) : (
<>
<div className="flex items-center justify-end gap-1.5 mb-1.5">
<button
onClick={() => detectFileRef.current?.click()}
disabled={detecting}
className="inline-flex items-center gap-1 px-2 py-0.5 rounded-md text-[10px] text-muted hover:text-accent hover:bg-accent/[0.06] disabled:opacity-40 transition-colors"
>
{detecting ? <Loader2 className="h-3 w-3 animate-spin" /> : <Upload className="h-3 w-3" />}
</button>
<button
onClick={addField}
className="inline-flex items-center gap-0.5 px-2 py-0.5 rounded-md text-[10px] text-muted hover:text-accent hover:bg-accent/[0.06] transition-colors"
>
<Plus className="h-3 w-3" />
</button>
<div className="space-y-1">
<div className={`flex items-center gap-2 px-2.5 py-1.5 rounded-md border ${matchStatus !== 'none' ? 'border-border/40 bg-elevated/20' : 'border-danger/30 bg-danger/[0.04]'}`}>
<span className="w-[72px] shrink-0 text-[11px] text-muted"></span>
<span className="flex-1 text-[11px] font-mono text-muted">symbol</span>
<span className="w-[52px] text-center text-[10px] text-muted/40"></span>
{matchStatus !== 'none'
? <span className="text-[9px] text-green-500/70 shrink-0">
{symbolMap.type === 'mapped' ? `${symbolMap.col}` : '← 计算'}
</span>
: <AlertCircle className="h-3.5 w-3.5 text-danger/60 shrink-0" />}
</div>
<div className="space-y-1">
<div className={`flex items-center gap-2 px-2.5 py-1.5 rounded-md border ${matchStatus !== 'none' ? 'border-border/40 bg-elevated/20' : 'border-danger/30 bg-danger/[0.04]'}`}>
<span className="w-[72px] shrink-0 text-[11px] text-muted"></span>
<span className="flex-1 text-[11px] font-mono text-muted">symbol</span>
<span className="w-[52px] text-center text-[10px] text-muted/40"></span>
{matchStatus !== 'none'
? <span className="text-[9px] text-green-500/70 shrink-0">
{symbolMap.type === 'mapped' ? `${symbolMap.col}` : '← 计算'}
</span>
: <AlertCircle className="h-3.5 w-3.5 text-danger/60 shrink-0" />}
</div>
<div className={`flex items-center gap-2 px-2.5 py-1.5 rounded-md border ${matchStatus !== 'none' ? 'border-border/40 bg-elevated/20' : 'border-danger/30 bg-danger/[0.04]'}`}>
<span className="w-[72px] shrink-0 text-[11px] text-muted"></span>
<span className="flex-1 text-[11px] font-mono text-muted">code</span>
<span className="w-[52px] text-center text-[10px] text-muted/40"></span>
{matchStatus !== 'none'
? <span className="text-[9px] text-green-500/70 shrink-0">
{codeMap.type === 'mapped' ? `${codeMap.col}` : codeMap.method === 'strip_exchange' ? '← symbol截取' : '← 推算'}
</span>
: <AlertCircle className="h-3.5 w-3.5 text-danger/60 shrink-0" />}
</div>
{userFields.map((f) => {
const idx = fields.indexOf(f)
return (
<div key={idx} className="flex items-center gap-1.5 group">
<input
value={f.label}
onChange={(e) => updateField(idx, 'label', e.target.value)}
placeholder="显示名"
className="w-[72px] h-7 px-2 rounded-md border border-border bg-base text-[11px] text-foreground placeholder:text-muted/40 focus:outline-none focus:border-accent/40"
/>
<input
value={f.name}
onChange={(e) => updateField(idx, 'name', e.target.value)}
placeholder="字段名"
className="flex-1 h-7 px-2 rounded-md border border-border bg-base text-[11px] font-mono text-foreground placeholder:text-muted/40 focus:outline-none focus:border-accent/40"
/>
<select
value={f.dtype}
onChange={(e) => updateField(idx, 'dtype', e.target.value)}
className="h-7 px-2 rounded-md border border-border bg-base text-[11px] text-foreground"
>
<option value="string"></option>
<option value="int"></option>
<option value="float"></option>
<option value="bool"></option>
</select>
<button
onClick={() => removeField(idx)}
className="p-1 rounded text-muted/40 hover:text-danger hover:bg-danger/10 opacity-0 group-hover:opacity-100 transition-all"
>
<X className="h-3 w-3" />
</button>
</div>
)
})}
<div className={`flex items-center gap-2 px-2.5 py-1.5 rounded-md border ${matchStatus !== 'none' ? 'border-border/40 bg-elevated/20' : 'border-danger/30 bg-danger/[0.04]'}`}>
<span className="w-[72px] shrink-0 text-[11px] text-muted"></span>
<span className="flex-1 text-[11px] font-mono text-muted">code</span>
<span className="w-[52px] text-center text-[10px] text-muted/40"></span>
{matchStatus !== 'none'
? <span className="text-[9px] text-green-500/70 shrink-0">
{codeMap.type === 'mapped' ? `${codeMap.col}` : codeMap.method === 'strip_exchange' ? '← symbol截取' : '← 推算'}
</span>
: <AlertCircle className="h-3.5 w-3.5 text-danger/60 shrink-0" />}
</div>
</>
{userFields.map((f) => {
const idx = fields.indexOf(f)
return (
<div key={idx} className="flex items-center gap-1.5 group">
<input
value={f.label}
onChange={(e) => updateField(idx, 'label', e.target.value)}
placeholder="显示名"
className="w-[72px] h-7 px-2 rounded-md border border-border bg-base text-[11px] text-foreground placeholder:text-muted/40 focus:outline-none focus:border-accent/40"
/>
<input
value={f.name}
onChange={(e) => updateField(idx, 'name', e.target.value)}
placeholder="字段名"
className="flex-1 h-7 px-2 rounded-md border border-border bg-base text-[11px] font-mono text-foreground placeholder:text-muted/40 focus:outline-none focus:border-accent/40"
/>
<select
value={f.dtype}
onChange={(e) => updateField(idx, 'dtype', e.target.value)}
className="h-7 px-2 rounded-md border border-border bg-base text-[11px] text-foreground"
>
<option value="string"></option>
<option value="int"></option>
<option value="float"></option>
<option value="bool"></option>
</select>
<button
onClick={() => removeField(idx)}
className="p-1 rounded text-muted/40 hover:text-danger hover:bg-danger/10 opacity-0 group-hover:opacity-100 transition-all"
>
<X className="h-3 w-3" />
</button>
</div>
)
})}
</div>
)}
</div>
{sourceMode === 'url' && (
<div className="rounded-xl border border-border/60 bg-elevated/20 p-3 space-y-2">
<div className="text-[11px] font-medium text-secondary"></div>
<label className="flex items-center gap-2 text-[11px] text-secondary">
<input
type="checkbox"
checked={savePull}
onChange={(e) => {
setSavePull(e.target.checked)
if (!e.target.checked) {
setImportNow(false)
setEnablePull(false)
}
}}
className="h-3.5 w-3.5 accent-accent"
/>
</label>
<label className="flex items-center gap-2 text-[11px] text-secondary">
<input
type="checkbox"
checked={importNow}
onChange={(e) => {
setImportNow(e.target.checked)
if (e.target.checked) setSavePull(true)
}}
className="h-3.5 w-3.5 accent-accent"
/>
</label>
<label className="flex items-center gap-2 text-[11px] text-secondary">
<input
type="checkbox"
checked={enablePull}
onChange={(e) => {
setEnablePull(e.target.checked)
if (e.target.checked) setSavePull(true)
}}
className="h-3.5 w-3.5 accent-accent"
/>
</label>
</div>
)}
{error && (
<div className="text-[11px] text-danger bg-danger/5 rounded-lg px-3 py-2">{error}</div>
)}
</div>
<div className="flex items-center justify-between px-6 py-4 border-t border-border/50 bg-elevated/20">
<div className="text-[10px] text-muted/60"></div>
<div className="text-[10px] text-muted/60">
{sourceMode === 'url'
? '创建后可在扩展数据卡片中继续调整拉取配置'
: sourceMode === 'file'
? '文件识别用于生成表结构,创建后仍可上传文件写入数据'
: '手动创建后可通过文件、推送或拉取写入数据'}
</div>
<div className="flex items-center gap-2">
<button onClick={onClose} className="px-4 py-2 rounded-lg bg-elevated text-secondary text-xs hover:bg-elevated/80 transition-colors">
@@ -21,7 +21,7 @@ export function ExtDataStatCard({ config, onDelete, deleting, onEdit }: {
const [showDelete, setShowDelete] = useState(false)
const [settingsOpen, setSettingsOpen] = useState(false)
const [dragOver, setDragOver] = useState(false)
const [ingestTab, setIngestTab] = useState<'file' | 'api' | 'pull'>('file')
const [ingestTab, setIngestTab] = useState<'file' | 'api' | 'pull'>('pull')
const [copied, setCopied] = useState(false)
const [fieldsExpanded, setFieldsExpanded] = useState(false)
const fieldsRef = useRef<HTMLDivElement>(null)
@@ -172,12 +172,12 @@ export function ExtDataStatCard({ config, onDelete, deleting, onEdit }: {
<div className="flex gap-1 rounded-lg bg-elevated/60 p-0.5">
<button
onClick={() => setIngestTab('file')}
onClick={() => setIngestTab('pull')}
className={`flex-1 inline-flex items-center justify-center gap-1 py-1.5 rounded-md text-[10px] font-medium transition-colors ${
ingestTab === 'file' ? 'bg-surface text-foreground shadow-sm' : 'text-muted hover:text-secondary'
ingestTab === 'pull' ? 'bg-surface text-foreground shadow-sm' : 'text-muted hover:text-secondary'
}`}
>
<Upload className="h-3 w-3" />
<RefreshCw className="h-3 w-3" />
</button>
<button
onClick={() => setIngestTab('api')}
@@ -188,16 +188,20 @@ export function ExtDataStatCard({ config, onDelete, deleting, onEdit }: {
<Code className="h-3 w-3" />
</button>
<button
onClick={() => setIngestTab('pull')}
onClick={() => setIngestTab('file')}
className={`flex-1 inline-flex items-center justify-center gap-1 py-1.5 rounded-md text-[10px] font-medium transition-colors ${
ingestTab === 'pull' ? 'bg-surface text-foreground shadow-sm' : 'text-muted hover:text-secondary'
ingestTab === 'file' ? 'bg-surface text-foreground shadow-sm' : 'text-muted hover:text-secondary'
}`}
>
<RefreshCw className="h-3 w-3" />
<Upload className="h-3 w-3" />
</button>
</div>
{ingestTab === 'file' ? (
{ingestTab === 'pull' ? (
<ExtDataPullPanel config={config} onSaved={() => qc.invalidateQueries({ queryKey: QK.extData })} />
) : ingestTab === 'api' ? (
<ExtDataApiPanel config={config} copied={copied} setCopied={setCopied} />
) : (
<>
<input
ref={fileRef}
@@ -235,10 +239,6 @@ export function ExtDataStatCard({ config, onDelete, deleting, onEdit }: {
)}
</div>
</>
) : ingestTab === 'api' ? (
<ExtDataApiPanel config={config} copied={copied} setCopied={setCopied} />
) : (
<ExtDataPullPanel config={config} onSaved={() => qc.invalidateQueries({ queryKey: QK.extData })} />
)}
{uploadResult && (