fix: restore strategy and data compatibility

This commit is contained in:
shy3130
2026-07-16 14:34:07 +08:00
parent c5049d26a7
commit c763a6970f
12 changed files with 433 additions and 62 deletions
+23 -21
View File
@@ -18,7 +18,7 @@ from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from app.strategy import config as strategy_config
from app.strategy.ai_generator import AIStrategyGenerator
from app.strategy.ai_generator import AIStrategyGenerator, find_meta_assignment
from app.strategy.engine import StrategyDef, StrategyEngine
from app.strategy.monitor import StrategyMonitorService
from app.strategy.prompt_builder import build_step1, build_step2
@@ -372,25 +372,10 @@ def _py_string(value: str) -> str:
def _find_meta_dict(code: str) -> ast.Dict:
# 兼容两种 LLM 常见写法:
# META = {...} → ast.Assign
# META: dict = {...} → ast.AnnAssign (类型注解, 合法但旧逻辑漏匹配)
tree = ast.parse(code)
for node in ast.walk(tree):
value = None
if isinstance(node, ast.Assign):
for target in node.targets:
if isinstance(target, ast.Name) and target.id == "META":
value = node.value
break
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) \
and node.target.id == "META":
value = node.value
if value is not None:
if not isinstance(value, ast.Dict):
raise ValueError("META 必须是字面量字典")
return value
raise ValueError("找不到 META 字典")
found = find_meta_assignment(code)
if found is None:
raise ValueError("找不到 META 字典")
return found[1]
def _set_meta_string_field(block: str, field: str, value: str) -> str:
@@ -437,7 +422,22 @@ def _normalize_strategy_meta(code: str, strategy_id: str,
name: str | None = None,
description: str | None = None) -> str:
"""Force persisted strategy identity to match the caller-owned identity."""
meta_node = _find_meta_dict(code)
found = find_meta_assignment(code)
if found is None:
raise ValueError("找不到 META 字典")
target, meta_node = found
if target.id != "META":
lines = code.splitlines(keepends=True)
index = target.lineno - 1
raw_line = lines[index].encode("utf-8")
lines[index] = (
raw_line[:target.col_offset]
+ b"META"
+ raw_line[target.end_col_offset:]
).decode("utf-8")
code = "".join(lines)
meta_node = _find_meta_dict(code)
lines = code.splitlines(keepends=True)
start = meta_node.lineno - 1
end = meta_node.end_lineno or meta_node.lineno
@@ -695,6 +695,8 @@ async def build_strategy_stream(req: BuildRequest, request: Request):
chunks.append(chunk)
yield json.dumps({"type": "delta", "content": chunk}, ensure_ascii=False) + "\n"
result = gen.validate_code("".join(chunks))
if gen.needs_structural_repair(result):
result = await gen.repair_code(result["code"], result["error"])
if req.step == 1:
result = _normalize_build_result(result, req.strategy_id, req.name, req.description)
elif req.strategy_id:
+14 -1
View File
@@ -25,7 +25,20 @@ import polars as pl
import pyarrow as pa
import pyarrow.compute as pc
import pyarrow.dataset as pads
from numba import njit, prange
try:
from numba import njit, prange
except ImportError:
def njit(*args, **kwargs):
if len(args) == 1 and callable(args[0]) and not kwargs:
return args[0]
def decorator(func):
return func
return decorator
prange = range
_MATRIX_CACHE_VERSION = 1
_DIRECT_MATRIX_LOADER_VERSION = 3
+12 -2
View File
@@ -16,6 +16,7 @@
from __future__ import annotations
import logging
import math
from pathlib import Path
from app.services.ext_data import (
@@ -108,6 +109,13 @@ def _symbol_to_code(symbol: str) -> str:
return symbol.split(".", 1)[0] if "." in symbol else symbol
def _dimension_label(value: object) -> str:
if value is None or (isinstance(value, float) and not math.isfinite(value)):
return ""
text = str(value).strip()
return "" if text.casefold() in {"nan", "none", "null"} else text
def _flatten_concept_rows(raw_rows: list[dict]) -> list[dict]:
"""概念: concepts 数组 → 分号拼接成「所属概念」字符串。
@@ -120,10 +128,11 @@ def _flatten_concept_rows(raw_rows: list[dict]) -> list[dict]:
if not sym:
continue
concepts = r.get("concepts") or []
labels = [label for c in concepts if (label := _dimension_label(c))]
out.append({
"股票代码": sym,
"股票简称": r.get("name") or "",
"所属概念": ";".join(str(c) for c in concepts if c),
"所属概念": ";".join(labels),
"symbol": sym,
"code": _symbol_to_code(sym),
})
@@ -141,10 +150,11 @@ def _flatten_industry_rows(raw_rows: list[dict]) -> list[dict]:
if not sym:
continue
inds = r.get("industries") or []
labels = [label for i in inds if (label := _dimension_label(i))]
out.append({
"股票代码": sym,
"股票简称": r.get("name") or "",
"所属同花顺行业": "-".join(str(i) for i in inds if i),
"所属同花顺行业": "-".join(labels),
"symbol": sym,
"code": _symbol_to_code(sym),
})
@@ -216,7 +216,11 @@ def _read_ext_rows(data_dir, config: ExtConfig, dimension_field: str) -> list[di
def _dimension_values(raw: Any) -> list[str]:
if raw is None:
return []
values = [v.strip() for v in _DIMENSION_SEP.split(str(raw).strip()) if v.strip()]
values = [
v.strip()
for v in _DIMENSION_SEP.split(str(raw).strip())
if v.strip() and v.strip().casefold() not in {"nan", "none", "null"}
]
return values
+177 -26
View File
@@ -7,6 +7,7 @@ from __future__ import annotations
import ast
import logging
import re
from pathlib import Path
logger = logging.getLogger(__name__)
@@ -29,11 +30,85 @@ _SYSTEM_PREFIX = """你是A股量化策略设计专家。根据用户描述的
4. scoring 权重根据策略核心逻辑定制,总和 = 1.0
5. 优先使用 Polars 表达式、窗口函数、聚合和 with_columns/filter 实现,避免逐行/逐股 Python 循环;只有表达式难以描述的复杂状态机才使用 partition_by/to_dicts
6. 直接输出Python代码,不要输出其他内容
7. 元数据必须使用模块顶层的 META = {...} 或 META: dict = {...},不得省略或改名;并且必须定义所选执行后端要求的策略入口
--- 策略开发指南 ---
"""
_META_NAMES = ("META", "STRATEGY_META", "meta")
_FENCED_CODE_RE = re.compile(
r"```(?P<language>[^\n`]*)\r?\n(?P<code>.*?)```",
re.DOTALL,
)
_POLARS_ENTRYPOINT_ERROR = "找不到策略入口函数 filter() 或 filter_history()"
_MATRIX_ENTRYPOINT_ERROR = "找不到 Matrix 策略入口 MATRIX_STRATEGY"
def _top_level_assignment(
tree: ast.Module,
name: str,
) -> tuple[ast.Name, ast.expr | None] | None:
for node in tree.body:
if isinstance(node, ast.Assign):
target = next(
(item for item in node.targets
if isinstance(item, ast.Name) and item.id == name),
None,
)
if target is not None:
return target, node.value
elif isinstance(node, ast.AnnAssign) \
and isinstance(node.target, ast.Name) \
and node.target.id == name:
return node.target, node.value
return None
def find_meta_assignment(code: str) -> tuple[ast.Name, ast.Dict] | None:
"""Find a supported module-level META assignment without executing code."""
tree = ast.parse(code)
for name in _META_NAMES:
found = _top_level_assignment(tree, name)
if found is not None:
target, value = found
if not isinstance(value, ast.Dict):
raise ValueError(f"{name} 必须是字面量字典")
return target, value
return None
def _strategy_execution_backend(tree: ast.Module, meta: dict | None = None) -> str:
found = _top_level_assignment(tree, "EXECUTION_BACKEND")
if found is not None:
try:
value = ast.literal_eval(found[1])
except (ValueError, SyntaxError):
value = None
if isinstance(value, str):
return value
if isinstance(meta, dict) and isinstance(meta.get("execution_backend"), str):
return meta["execution_backend"]
if any(
isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
and node.name == "filter_history"
for node in tree.body
):
return "python_history_legacy"
return "polars_expr"
def _strategy_entrypoint_error(code: str, meta: dict | None = None) -> str | None:
tree = ast.parse(code)
if _strategy_execution_backend(tree, meta) == "matrix_native":
return None if _top_level_assignment(tree, "MATRIX_STRATEGY") else _MATRIX_ENTRYPOINT_ERROR
has_polars_entrypoint = any(
isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
and node.name in {"filter", "filter_history"}
for node in tree.body
)
return None if has_polars_entrypoint else _POLARS_ENTRYPOINT_ERROR
class AIStrategyGenerator:
"""AI 策略生成器"""
@@ -59,7 +134,10 @@ class AIStrategyGenerator:
# 调用 LLM
code = await self._call_llm(user_prompt, guide)
return self.validate_code(code)
result = self.validate_code(code)
if self.needs_structural_repair(result):
return await self.repair_code(result["code"], result["error"])
return result
async def stream(self, user_prompt: str):
"""Yield generated strategy code deltas from the configured AI provider."""
@@ -82,16 +160,84 @@ class AIStrategyGenerator:
# 验证
try:
self._validate_safety(code)
except SyntaxError as e:
return {
"code": code,
"meta": {},
"valid": False,
"error": f"Python 语法错误: {e.msg}",
}
except ValueError as e:
return {"code": code, "meta": {}, "valid": False, "error": str(e)}
return {
"code": code,
"meta": {},
"valid": False,
"error": str(e),
}
# 试加载获取 META
try:
meta = self._extract_meta(code)
except Exception as e:
return {"code": code, "meta": {}, "valid": False, "error": f"解析META失败: {e}"}
return {
"code": code,
"meta": {},
"valid": False,
"error": f"解析META失败: {e}",
}
return {"code": code, "meta": meta, "valid": True, "error": None}
entrypoint_error = _strategy_entrypoint_error(code, meta)
if entrypoint_error:
return {
"code": code,
"meta": meta,
"valid": False,
"error": entrypoint_error,
}
return {
"code": code,
"meta": meta,
"valid": True,
"error": None,
}
@staticmethod
def needs_structural_repair(result: dict) -> bool:
error = result.get("error") or ""
return error.startswith("解析META失败:") or error in {
_POLARS_ENTRYPOINT_ERROR,
_MATRIX_ENTRYPOINT_ERROR,
}
async def repair_code(self, code: str, error: str) -> dict:
"""Ask the model once for a complete replacement after a structural error."""
try:
backend = _strategy_execution_backend(ast.parse(code))
except SyntaxError:
backend = "polars_expr"
if backend == "matrix_native":
entrypoint_requirement = (
'保留 EXECUTION_BACKEND = "matrix_native",定义 MATRIX_STRATEGY'
"不得添加 filter() 或 filter_history()"
)
else:
entrypoint_requirement = (
"保留原执行后端,并定义对应的 filter() 或 filter_history()"
)
prompt = f"""上一次生成的策略代码未通过结构校验。
校验错误:{error}
请输出修复后的完整策略 Python 文件。必须保留原策略意图和参数,使用模块顶层
META = {{...}}{entrypoint_requirement}。只输出完整 Python 代码。
上一次代码:
```python
{code}
```"""
repaired = await self._call_llm(prompt, self._get_guide())
return self.validate_code(repaired)
async def _call_llm(self, user_prompt: str, guide: str) -> str:
"""Call the configured AI provider and return generated strategy code."""
@@ -109,11 +255,22 @@ class AIStrategyGenerator:
@staticmethod
def _extract_code_block(content: str) -> str:
# Extract fenced code if the model wrapped the answer in Markdown.
if "```python" in content:
return content.split("```python", 1)[1].split("```", 1)[0].strip()
if "```" in content:
return content.split("```", 1)[1].split("```", 1)[0].strip()
blocks = list(_FENCED_CODE_RE.finditer(content))
for match in blocks:
candidate = match.group("code").strip()
try:
found = find_meta_assignment(candidate)
if found is not None:
meta = ast.literal_eval(found[1])
if isinstance(meta, dict) and _strategy_entrypoint_error(candidate, meta) is None:
return candidate
except (SyntaxError, ValueError):
continue
for match in blocks:
if match.group("language").strip().lower() in {"python", "py"}:
return match.group("code").strip()
if blocks:
return blocks[0].group("code").strip()
return content.strip()
# import 白名单: Polars 与矩阵策略只开放执行协议所需模块。
@@ -196,20 +353,14 @@ class AIStrategyGenerator:
兼容两种声明: META = {...} (Assign) 和 META: dict = {...} (AnnAssign)。
与 api.strategy._find_meta_dict 保持同一套匹配逻辑。
"""
tree = ast.parse(code)
for node in ast.walk(tree):
value = None
if isinstance(node, ast.Assign):
for target in node.targets:
if isinstance(target, ast.Name) and target.id == "META":
value = node.value
break
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) \
and node.target.id == "META":
value = node.value
if value is not None:
try:
return ast.literal_eval(value)
except (ValueError, SyntaxError) as e:
raise ValueError(f"META 必须是纯字面量字典: {e}") from e
return {}
found = find_meta_assignment(code)
if found is None:
raise ValueError("找不到 META 字典")
_, value = found
try:
meta = ast.literal_eval(value)
except (ValueError, SyntaxError) as e:
raise ValueError(f"META 必须是纯字面量字典: {e}") from e
if not isinstance(meta, dict):
raise ValueError("META 必须是纯字面量字典")
return meta
+2 -1
View File
@@ -19,7 +19,8 @@ dependencies = [
"pyarrow>=16.0",
"pandas>=2.2", # 仅在 BacktestService 边界使用,见 §7.4 / ADR-19
"psutil>=5.9", # 独立回测 worker 的峰值 RSS 与退出后内存指标
"numba>=0.65.1", # Matrix 有效 K 线通用编译内核
# llvmlite 不再提供 macOS Intel wheel;该平台使用 Matrix 纯 Python fallback。
"numba>=0.65.1; sys_platform != 'darwin' or platform_machine != 'x86_64'",
"fastexcel>=0.10", # Polars 读取 xlsx/xls
# TickFlow 官方 SDK
"tickflow[all]>=0.1.23",
+20
View File
@@ -1,5 +1,7 @@
from __future__ import annotations
import pytest
from app.strategy.ai_generator import GUIDE_PATH, AIStrategyGenerator
from app.strategy.prompt_builder import build_step1
@@ -44,3 +46,21 @@ def test_matrix_backend_prompt_and_imports_are_supported():
"import numpy as np\n"
"from app.backtest.matrix import MarketDataMatrix, SignalMatrix, make_signal_matrix\n"
)
@pytest.mark.asyncio
async def test_generate_only_repairs_structural_output_once(monkeypatch):
calls = 0
async def fake_call_llm(self, user_prompt, guide):
nonlocal calls
calls += 1
return "import polars as pl\n\ndef filter(df, params):\n return pl.lit(True)"
monkeypatch.setattr(AIStrategyGenerator, "_call_llm", fake_call_llm)
result = await AIStrategyGenerator().generate("生成测试策略")
assert calls == 2
assert result["valid"] is False
assert "找不到 META 字典" in result["error"]
@@ -1,6 +1,8 @@
"""AI 策略 META 规范化回归测试。"""
from __future__ import annotations
import pytest
from app.api.strategy import _normalize_build_result, _normalize_strategy_meta
RAW_CODE = '''"""模型返回的策略"""
@@ -138,3 +140,99 @@ def test_normalize_build_result_succeeds_on_annotated_form():
assert normalized["valid"] is True
assert normalized["error"] is None
assert normalized["meta"]["id"] == "ai_build_ok"
@pytest.mark.parametrize("alias", ["STRATEGY_META", "meta"])
def test_normalize_strategy_meta_accepts_common_aliases(alias):
from app.strategy.ai_generator import AIStrategyGenerator
raw = RAW_CODE.replace("META =", f"{alias} =", 1)
code = _normalize_strategy_meta(raw, "ai_alias_ok", name="别名策略")
compile(code, "<strategy>", "exec")
assert "META =" in code
assert f"{alias} =" not in code
assert AIStrategyGenerator._extract_meta(code)["id"] == "ai_alias_ok"
def test_validate_code_rejects_missing_meta():
from app.strategy.ai_generator import AIStrategyGenerator
code = """import polars as pl
def filter(df, params):
return pl.lit(True)
"""
result = AIStrategyGenerator().validate_code(code)
assert result["valid"] is False
assert "找不到 META 字典" in result["error"]
def test_validate_code_ignores_meta_inside_function():
from app.strategy.ai_generator import AIStrategyGenerator
code = """import polars as pl
def filter(df, params):
META = {"id": "nested"}
return pl.lit(True)
"""
result = AIStrategyGenerator().validate_code(code)
assert result["valid"] is False
assert "找不到 META 字典" in result["error"]
def test_validate_code_rejects_missing_strategy_entrypoint():
from app.strategy.ai_generator import AIStrategyGenerator
result = AIStrategyGenerator().validate_code('META = {"id": "no_filter"}')
assert result["valid"] is False
assert result["error"] == "找不到策略入口函数 filter() 或 filter_history()"
def test_validate_code_accepts_matrix_strategy_entrypoint():
from app.strategy.ai_generator import AIStrategyGenerator
code = '''META = {"id": "matrix", "execution_backend": "matrix_native"}
EXECUTION_BACKEND = "matrix_native"
MATRIX_STRATEGY = object()
'''
result = AIStrategyGenerator().validate_code(code)
assert result["valid"] is True
assert result["error"] is None
def test_validate_code_rejects_missing_matrix_strategy_entrypoint():
from app.strategy.ai_generator import AIStrategyGenerator
code = '''META = {"id": "matrix", "execution_backend": "matrix_native"}
EXECUTION_BACKEND = "matrix_native"
'''
result = AIStrategyGenerator().validate_code(code)
assert result["valid"] is False
assert result["error"] == "找不到 Matrix 策略入口 MATRIX_STRATEGY"
def test_extract_code_block_prefers_complete_strategy():
from app.strategy.ai_generator import AIStrategyGenerator
content = f"""```python
print("draft")
```
```python
{RAW_CODE}
```
"""
assert AIStrategyGenerator._extract_code_block(content) == RAW_CODE.strip()
@@ -0,0 +1,28 @@
from __future__ import annotations
from app.services.ext_presets import _flatten_concept_rows, _flatten_industry_rows
from app.services.market_overview_builder import _dimension_values
def test_concept_flatten_drops_missing_value_placeholders():
rows = _flatten_concept_rows([{
"symbol": "600000.SH",
"name": "浦发银行",
"concepts": ["银行", "nan", None, float("nan"), " null ", "金融科技"],
}])
assert rows[0]["所属概念"] == "银行;金融科技"
def test_industry_flatten_drops_missing_value_placeholders():
rows = _flatten_industry_rows([{
"symbol": "600000.SH",
"name": "浦发银行",
"industries": ["金融", "None", "银行"],
}])
assert rows[0]["所属同花顺行业"] == "金融-银行"
def test_overview_dimension_values_ignore_legacy_nan_group():
assert _dimension_values("人工智能;nan;芯片;NULL") == ["人工智能", "芯片"]
+36 -1
View File
@@ -7,7 +7,6 @@ import pytest
from app.api.strategy import BuildRequest, build_strategy_stream
from app.strategy.ai_generator import AIStrategyGenerator
STREAM_CODE = '''"""测试策略"""
import polars as pl
@@ -66,3 +65,39 @@ async def test_build_strategy_stream_yields_delta_and_normalized_result(monkeypa
assert result["meta"]["id"] == "ai_streamed"
assert result["meta"]["name"] == "新策略"
assert '"id": "ai_streamed"' in result["code"]
@pytest.mark.asyncio
async def test_build_strategy_stream_repairs_missing_meta_once(monkeypatch):
calls = 0
async def fake_stream(self, prompt):
yield "import polars as pl\n\ndef filter(df, params):\n return pl.lit(True)\n"
async def fake_repair(self, code, error):
nonlocal calls
calls += 1
return self.validate_code(STREAM_CODE)
monkeypatch.setattr(AIStrategyGenerator, "stream", fake_stream)
monkeypatch.setattr(AIStrategyGenerator, "repair_code", fake_repair)
req = BuildRequest(
step=1,
name="修复后策略",
description="修复后描述",
direction="long",
rules="1. 测试规则",
strategy_id="ai_repaired",
)
response = await build_strategy_stream(req, None)
body = b""
async for chunk in response.body_iterator:
body += chunk.encode("utf-8") if isinstance(chunk, str) else chunk
result = json.loads(body.decode("utf-8").splitlines()[-1])
assert calls == 1
assert result["type"] == "result"
assert result["valid"] is True
assert result["meta"]["id"] == "ai_repaired"
assert result["meta"]["name"] == "修复后策略"
+2 -2
View File
@@ -2512,7 +2512,7 @@ dependencies = [
{ name = "fastapi" },
{ name = "fastexcel" },
{ name = "httpx" },
{ name = "numba" },
{ name = "numba", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" },
{ name = "openai" },
{ name = "pandas" },
{ name = "pillow" },
@@ -2558,7 +2558,7 @@ requires-dist = [
{ name = "fastexcel", specifier = ">=0.10" },
{ name = "httpx", specifier = ">=0.27" },
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" },
{ name = "numba", specifier = ">=0.65.1" },
{ name = "numba", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'", specifier = ">=0.65.1" },
{ name = "openai", specifier = ">=1.40" },
{ name = "pandas", specifier = ">=2.2" },
{ name = "pillow", specifier = ">=10.0" },
+16 -7
View File
@@ -56,6 +56,19 @@ const DIMENSION_NAME_KEYS = [
'name', '概念名称', '概念', '行业名称', '行业', '板块名称', '板块',
'concept', 'industry', 'sector', 'theme', 'title', 'label',
]
const INVALID_DIMENSION_VALUES = new Set(['nan', 'none', 'null'])
function dimensionValue(raw: unknown): string {
const text = String(raw ?? '').trim()
return INVALID_DIMENSION_VALUES.has(text.toLowerCase()) ? '' : text
}
function dimensionValues(raw: unknown): string[] {
if (raw == null) return []
return String(raw).split(SEPARATORS)
.map(dimensionValue)
.filter(Boolean)
}
/** 检测行是否是"板块维度"结构(含成分股列表字段) */
function detectConstituentField(fields: ExtDataField[]): string | null {
@@ -105,13 +118,9 @@ function parsePerStock(
const map = new Map<string, StockRow[]>()
for (const row of rows) {
const raw = row[dimensionField]
if (raw == null) continue
const text = String(raw).trim()
if (!text) continue
// 支持多值分隔(如 "人工智能,芯片,5G"
const values = text.split(SEPARATORS).map(s => s.trim()).filter(Boolean)
const values = dimensionValues(row[dimensionField])
if (!values.length) continue
const stock: StockRow = { ...row, symbol: row.symbol ?? row.code ?? '' }
for (const v of values) {
@@ -142,7 +151,7 @@ function parsePerDimension(
const allStocks: StockRow[] = []
const groups = rows.map(row => {
const key = String(row[nameField] ?? row[constituentField] ?? '').trim()
const key = dimensionValue(row[nameField] ?? row[constituentField])
if (!key) return null
// 成分股可能是字符串数组、对象数组、逗号分隔字符串