mirror of
https://ghfast.top/https://github.com/aeroxw/tick-stock-panel.git
synced 2026-09-12 14:24:15 +08:00
Merge pull request #299 from kevin9327/fix/ext-data-rows-date-guard
fix(ext-data): rows 的 date 入参先校验再拼分区路径
This commit is contained in:
@@ -204,6 +204,20 @@ def _safe_json_value(value):
|
||||
return value
|
||||
|
||||
|
||||
def _partition_date(raw: str) -> str:
|
||||
"""把 `date` 入参规范成 `YYYY-MM-DD` 分区名。
|
||||
|
||||
这个值直接拼进分区目录名 (`timeseries/date=<value>`), 所以非法值不只是格式问题:
|
||||
`date=x/../../../../kline_daily` 会让读取路径离开 `ext_data/<id>/timeseries/`。
|
||||
同一文件的 `/sync`、`/ingest`、`/backfill` 都先 `date.fromisoformat` 再用, 只有
|
||||
`/rows` 和 `/dimension-members` 走的这条路把原始字符串直接拼进了路径。
|
||||
"""
|
||||
try:
|
||||
return date.fromisoformat(raw).isoformat()
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, f"日期格式错误: {raw}") from e
|
||||
|
||||
|
||||
def _read_ext_dataframe(
|
||||
config: ExtConfig,
|
||||
data_dir: Path,
|
||||
@@ -222,10 +236,11 @@ def _read_ext_dataframe(
|
||||
return pl.DataFrame(), None
|
||||
|
||||
if snapshot_date:
|
||||
path = base / f"date={snapshot_date}" / "part.parquet"
|
||||
day = _partition_date(snapshot_date)
|
||||
path = base / f"date={day}" / "part.parquet"
|
||||
if not path.exists():
|
||||
return pl.DataFrame(), snapshot_date
|
||||
return pl.read_parquet(path), snapshot_date
|
||||
return pl.DataFrame(), day
|
||||
return pl.read_parquet(path), day
|
||||
|
||||
partitions = sorted(
|
||||
d for d in base.iterdir()
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""`/rows`、`/dimension-members` 的 date 入参校验 — 非法日期返回 400 而不是拼进路径。
|
||||
|
||||
`_read_ext_dataframe` 用 `date=<入参>` 拼出时序分区目录名。同一文件的 `/sync`、
|
||||
`/ingest`、`/backfill` 都先 `date.fromisoformat` 再用, 只有这条读取路径把原始
|
||||
字符串直接拼进 `Path`, 于是 `date=x/../../../../kline_daily` 读到的是配置目录
|
||||
之外的 `part.parquet`。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.api.ext_data import _read_ext_dataframe
|
||||
from app.services.ext_data import ExtConfig, ExtField
|
||||
|
||||
|
||||
def _cfg() -> ExtConfig:
|
||||
return ExtConfig(
|
||||
id="hot",
|
||||
label="人气",
|
||||
mode="timeseries",
|
||||
fields=[ExtField("symbol", "string"), ExtField("heat", "float")],
|
||||
)
|
||||
|
||||
|
||||
def _layout(tmp_path: Path) -> Path:
|
||||
"""建出真实目录形状: data/ext_data/hot/timeseries/ 和它外面的一份 parquet。"""
|
||||
data_dir = tmp_path / "data"
|
||||
partition = data_dir / "ext_data" / "hot" / "timeseries" / "date=2026-09-11"
|
||||
partition.mkdir(parents=True)
|
||||
pl.DataFrame({"symbol": ["000001.SZ"], "heat": [1.0]}).write_parquet(
|
||||
partition / "part.parquet"
|
||||
)
|
||||
|
||||
outside = data_dir / "kline_daily"
|
||||
outside.mkdir(parents=True)
|
||||
pl.DataFrame({"symbol": ["不该被读到"], "heat": [9.9]}).write_parquet(
|
||||
outside / "part.parquet"
|
||||
)
|
||||
return data_dir
|
||||
|
||||
|
||||
def test_rows_date_does_not_leave_the_config_directory(tmp_path: Path) -> None:
|
||||
data_dir = _layout(tmp_path)
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
_read_ext_dataframe(_cfg(), data_dir, "x/../../../../kline_daily")
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad",
|
||||
["not-a-date", "2026-13-01", "2026/09/11", "../../../../kline_daily"],
|
||||
)
|
||||
def test_rows_rejects_a_date_that_is_not_a_date(tmp_path: Path, bad: str) -> None:
|
||||
data_dir = _layout(tmp_path)
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
_read_ext_dataframe(_cfg(), data_dir, bad)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
|
||||
|
||||
def test_rows_still_reads_the_named_partition(tmp_path: Path) -> None:
|
||||
data_dir = _layout(tmp_path)
|
||||
|
||||
df, active = _read_ext_dataframe(_cfg(), data_dir, "2026-09-11")
|
||||
|
||||
assert active == "2026-09-11"
|
||||
assert df.get_column("symbol").to_list() == ["000001.SZ"]
|
||||
|
||||
|
||||
def test_rows_returns_empty_for_a_valid_date_with_no_partition(tmp_path: Path) -> None:
|
||||
data_dir = _layout(tmp_path)
|
||||
|
||||
df, active = _read_ext_dataframe(_cfg(), data_dir, "2026-01-02")
|
||||
|
||||
assert active == "2026-01-02"
|
||||
assert df.is_empty()
|
||||
|
||||
|
||||
def test_rows_without_a_date_still_picks_the_latest_partition(tmp_path: Path) -> None:
|
||||
data_dir = _layout(tmp_path)
|
||||
|
||||
df, active = _read_ext_dataframe(_cfg(), data_dir, None)
|
||||
|
||||
assert active == "2026-09-11"
|
||||
assert df.get_column("symbol").to_list() == ["000001.SZ"]
|
||||
Reference in New Issue
Block a user