From 79eb150de2a0ec9de740ab810ad34de9b73d1a68 Mon Sep 17 00:00:00 2001 From: kevin9327 Date: Fri, 11 Sep 2026 00:50:33 +0900 Subject: [PATCH] =?UTF-8?q?fix(ext-data):=20rows=20=E7=9A=84=20date=20?= =?UTF-8?q?=E5=85=A5=E5=8F=82=E5=85=88=E6=A0=A1=E9=AA=8C=E5=86=8D=E6=8B=BC?= =?UTF-8?q?=E5=88=86=E5=8C=BA=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_read_ext_dataframe` 把 `date` 查询参数原样拼进 `timeseries/date=<值>`, 未经校验, 于是 `date=x/../../../../kline_daily` 读到的是 `ext_data//timeseries/` 之外的 `part.parquet` 并从 /rows 返回。同一文件的 /sync、/ingest、/backfill 都先 `date.fromisoformat` 再用, 只有这条读取路径没有。 拼路径前规范成 YYYY-MM-DD, 非法值按本文件既有约定返回 400。合法日期、 分区不存在、不传 date 三种情况的行为不变。 --- backend/app/api/ext_data.py | 21 ++++- .../tests/test_ext_data_rows_date_guard.py | 92 +++++++++++++++++++ 2 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 backend/tests/test_ext_data_rows_date_guard.py diff --git a/backend/app/api/ext_data.py b/backend/app/api/ext_data.py index 92c498d..08ef85a 100644 --- a/backend/app/api/ext_data.py +++ b/backend/app/api/ext_data.py @@ -204,6 +204,20 @@ def _safe_json_value(value): return value +def _partition_date(raw: str) -> str: + """把 `date` 入参规范成 `YYYY-MM-DD` 分区名。 + + 这个值直接拼进分区目录名 (`timeseries/date=`), 所以非法值不只是格式问题: + `date=x/../../../../kline_daily` 会让读取路径离开 `ext_data//timeseries/`。 + 同一文件的 `/sync`、`/ingest`、`/backfill` 都先 `date.fromisoformat` 再用, 只有 + `/rows` 和 `/dimension-members` 走的这条路把原始字符串直接拼进了路径。 + """ + try: + return date.fromisoformat(raw).isoformat() + except ValueError as e: + raise HTTPException(400, f"日期格式错误: {raw}") from e + + def _read_ext_dataframe( config: ExtConfig, data_dir: Path, @@ -222,10 +236,11 @@ def _read_ext_dataframe( return pl.DataFrame(), None if snapshot_date: - path = base / f"date={snapshot_date}" / "part.parquet" + day = _partition_date(snapshot_date) + path = base / f"date={day}" / "part.parquet" if not path.exists(): - return pl.DataFrame(), snapshot_date - return pl.read_parquet(path), snapshot_date + return pl.DataFrame(), day + return pl.read_parquet(path), day partitions = sorted( d for d in base.iterdir() diff --git a/backend/tests/test_ext_data_rows_date_guard.py b/backend/tests/test_ext_data_rows_date_guard.py new file mode 100644 index 0000000..0773862 --- /dev/null +++ b/backend/tests/test_ext_data_rows_date_guard.py @@ -0,0 +1,92 @@ +"""`/rows`、`/dimension-members` 的 date 入参校验 — 非法日期返回 400 而不是拼进路径。 + +`_read_ext_dataframe` 用 `date=<入参>` 拼出时序分区目录名。同一文件的 `/sync`、 +`/ingest`、`/backfill` 都先 `date.fromisoformat` 再用, 只有这条读取路径把原始 +字符串直接拼进 `Path`, 于是 `date=x/../../../../kline_daily` 读到的是配置目录 +之外的 `part.parquet`。 +""" +from __future__ import annotations + +from pathlib import Path + +import polars as pl +import pytest +from fastapi import HTTPException + +from app.api.ext_data import _read_ext_dataframe +from app.services.ext_data import ExtConfig, ExtField + + +def _cfg() -> ExtConfig: + return ExtConfig( + id="hot", + label="人气", + mode="timeseries", + fields=[ExtField("symbol", "string"), ExtField("heat", "float")], + ) + + +def _layout(tmp_path: Path) -> Path: + """建出真实目录形状: data/ext_data/hot/timeseries/ 和它外面的一份 parquet。""" + data_dir = tmp_path / "data" + partition = data_dir / "ext_data" / "hot" / "timeseries" / "date=2026-09-11" + partition.mkdir(parents=True) + pl.DataFrame({"symbol": ["000001.SZ"], "heat": [1.0]}).write_parquet( + partition / "part.parquet" + ) + + outside = data_dir / "kline_daily" + outside.mkdir(parents=True) + pl.DataFrame({"symbol": ["不该被读到"], "heat": [9.9]}).write_parquet( + outside / "part.parquet" + ) + return data_dir + + +def test_rows_date_does_not_leave_the_config_directory(tmp_path: Path) -> None: + data_dir = _layout(tmp_path) + + with pytest.raises(HTTPException) as excinfo: + _read_ext_dataframe(_cfg(), data_dir, "x/../../../../kline_daily") + + assert excinfo.value.status_code == 400 + + +@pytest.mark.parametrize( + "bad", + ["not-a-date", "2026-13-01", "2026/09/11", "../../../../kline_daily"], +) +def test_rows_rejects_a_date_that_is_not_a_date(tmp_path: Path, bad: str) -> None: + data_dir = _layout(tmp_path) + + with pytest.raises(HTTPException) as excinfo: + _read_ext_dataframe(_cfg(), data_dir, bad) + + assert excinfo.value.status_code == 400 + + +def test_rows_still_reads_the_named_partition(tmp_path: Path) -> None: + data_dir = _layout(tmp_path) + + df, active = _read_ext_dataframe(_cfg(), data_dir, "2026-09-11") + + assert active == "2026-09-11" + assert df.get_column("symbol").to_list() == ["000001.SZ"] + + +def test_rows_returns_empty_for_a_valid_date_with_no_partition(tmp_path: Path) -> None: + data_dir = _layout(tmp_path) + + df, active = _read_ext_dataframe(_cfg(), data_dir, "2026-01-02") + + assert active == "2026-01-02" + assert df.is_empty() + + +def test_rows_without_a_date_still_picks_the_latest_partition(tmp_path: Path) -> None: + data_dir = _layout(tmp_path) + + df, active = _read_ext_dataframe(_cfg(), data_dir, None) + + assert active == "2026-09-11" + assert df.get_column("symbol").to_list() == ["000001.SZ"]