Merge pull request #299 from kevin9327/fix/ext-data-rows-date-guard

fix(ext-data): rows 的 date 入参先校验再拼分区路径
This commit is contained in:
wshy
2026-09-11 09:02:26 +08:00
committed by GitHub
2 changed files with 110 additions and 3 deletions
+18 -3
View File
@@ -204,6 +204,20 @@ def _safe_json_value(value):
return value
def _partition_date(raw: str) -> str:
"""把 `date` 入参规范成 `YYYY-MM-DD` 分区名。
这个值直接拼进分区目录名 (`timeseries/date=<value>`), 所以非法值不只是格式问题:
`date=x/../../../../kline_daily` 会让读取路径离开 `ext_data/<id>/timeseries/`。
同一文件的 `/sync`、`/ingest`、`/backfill` 都先 `date.fromisoformat` 再用, 只有
`/rows` 和 `/dimension-members` 走的这条路把原始字符串直接拼进了路径。
"""
try:
return date.fromisoformat(raw).isoformat()
except ValueError as e:
raise HTTPException(400, f"日期格式错误: {raw}") from e
def _read_ext_dataframe(
config: ExtConfig,
data_dir: Path,
@@ -222,10 +236,11 @@ def _read_ext_dataframe(
return pl.DataFrame(), None
if snapshot_date:
path = base / f"date={snapshot_date}" / "part.parquet"
day = _partition_date(snapshot_date)
path = base / f"date={day}" / "part.parquet"
if not path.exists():
return pl.DataFrame(), snapshot_date
return pl.read_parquet(path), snapshot_date
return pl.DataFrame(), day
return pl.read_parquet(path), day
partitions = sorted(
d for d in base.iterdir()
@@ -0,0 +1,92 @@
"""`/rows`、`/dimension-members` 的 date 入参校验 — 非法日期返回 400 而不是拼进路径。
`_read_ext_dataframe` 用 `date=<入参>` 拼出时序分区目录名。同一文件的 `/sync`、
`/ingest`、`/backfill` 都先 `date.fromisoformat` 再用, 只有这条读取路径把原始
字符串直接拼进 `Path`, 于是 `date=x/../../../../kline_daily` 读到的是配置目录
之外的 `part.parquet`。
"""
from __future__ import annotations
from pathlib import Path
import polars as pl
import pytest
from fastapi import HTTPException
from app.api.ext_data import _read_ext_dataframe
from app.services.ext_data import ExtConfig, ExtField
def _cfg() -> ExtConfig:
return ExtConfig(
id="hot",
label="人气",
mode="timeseries",
fields=[ExtField("symbol", "string"), ExtField("heat", "float")],
)
def _layout(tmp_path: Path) -> Path:
"""建出真实目录形状: data/ext_data/hot/timeseries/ 和它外面的一份 parquet。"""
data_dir = tmp_path / "data"
partition = data_dir / "ext_data" / "hot" / "timeseries" / "date=2026-09-11"
partition.mkdir(parents=True)
pl.DataFrame({"symbol": ["000001.SZ"], "heat": [1.0]}).write_parquet(
partition / "part.parquet"
)
outside = data_dir / "kline_daily"
outside.mkdir(parents=True)
pl.DataFrame({"symbol": ["不该被读到"], "heat": [9.9]}).write_parquet(
outside / "part.parquet"
)
return data_dir
def test_rows_date_does_not_leave_the_config_directory(tmp_path: Path) -> None:
data_dir = _layout(tmp_path)
with pytest.raises(HTTPException) as excinfo:
_read_ext_dataframe(_cfg(), data_dir, "x/../../../../kline_daily")
assert excinfo.value.status_code == 400
@pytest.mark.parametrize(
"bad",
["not-a-date", "2026-13-01", "2026/09/11", "../../../../kline_daily"],
)
def test_rows_rejects_a_date_that_is_not_a_date(tmp_path: Path, bad: str) -> None:
data_dir = _layout(tmp_path)
with pytest.raises(HTTPException) as excinfo:
_read_ext_dataframe(_cfg(), data_dir, bad)
assert excinfo.value.status_code == 400
def test_rows_still_reads_the_named_partition(tmp_path: Path) -> None:
data_dir = _layout(tmp_path)
df, active = _read_ext_dataframe(_cfg(), data_dir, "2026-09-11")
assert active == "2026-09-11"
assert df.get_column("symbol").to_list() == ["000001.SZ"]
def test_rows_returns_empty_for_a_valid_date_with_no_partition(tmp_path: Path) -> None:
data_dir = _layout(tmp_path)
df, active = _read_ext_dataframe(_cfg(), data_dir, "2026-01-02")
assert active == "2026-01-02"
assert df.is_empty()
def test_rows_without_a_date_still_picks_the_latest_partition(tmp_path: Path) -> None:
data_dir = _layout(tmp_path)
df, active = _read_ext_dataframe(_cfg(), data_dir, None)
assert active == "2026-09-11"
assert df.get_column("symbol").to_list() == ["000001.SZ"]