688f83fdc5
把「设计团队超时降级 / writer 撞轮数墙 / step_cap 熔断 / ¥ 软预算软停 / 墙钟超时」这类
编排层事件从 studio 运行态穿到批跑台账,让退路树能一眼归因。此前两败局其实 100% 集中在
「设计团队超时降级路」,但台账 fail_system 空 → decide_n5 按 fail_system 看"分散"出
R3_conditional,主持人只能人工翻日志归因。
做法(不塞 fail_system,游戏系统桶语义不污染;decide_n5 的 R1/R2 分流逻辑一字不动):
- RunRecord 新增 infra_flags: list[str](field(default_factory=list);受控值集 INFRA_FLAGS
五项常量 design_team_degraded/writer_iter_wall/step_cap_tripped/soft_budget_tripped/
wall_timeout,注释写明可扩)+ 纯映射 infra_flags_from_runtime(信号→值集,去重稳定序);
- studio 运行态采集:设计团队降级点(捕获异常,.kind=='timeout' 归 wall_timeout)、外层
resume 轮数墙、熔断返回 kind、¥ 软预算软停标记,经 _infra_flags_from_runtime 归一进 result;
design_team.DesignTeamError 加 kind 区分墙钟超时;
- batch_run.result_to_record 从 result 抽 infra_flags 落台账(防御式复制);
- decide_n5 只把各轮 infra_flags 分布打进 reasons(如「r1 infra 分布={design_team_degraded:2,
wall_timeout:2}」),不分支、不改任何出口。
向后兼容:旧 jsonl 行无 infra_flags → from_jsonl_line 读空列表不炸;r{N} 轮分组正则不动;
middleware.py 不动;退路树出口枚举与判定不动;decide_n5 签名不破坏。
验收:新增单测 5(纯映射/studio fake 运行态采集/batch 落账/旧行兼容/decide_n5 分布行+出口不变
含 --no-strong-baseline 语义原样);全套 102 passed(97 基线 + 5);run_record 与 fallback_tree
的 __main__ 自检、decide_n5_from_jsonl 新旧混合行端到端均通过。
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
246 lines
15 KiB
Python
246 lines
15 KiB
Python
"""test_infra_flags.py —— 收敛环台账 infra 归因穿线单测(工单 k,observe-only 第一迭代)。
|
|
|
|
跑:cd tier2/gen-worker && cheap-worker/.venv/bin/python -m pytest tests/test_infra_flags.py -q
|
|
|
|
验收门(≥4):
|
|
① studio 态 flag 采集(构造 fake 运行态)—— test_studio_collects_* / test_infra_flags_from_runtime_*;
|
|
② batch_run 落账含 infra_flags —— test_result_to_record_carries_infra_flags;
|
|
③ 旧行(无字段)解析兼容 —— test_from_jsonl_line_backward_compat;
|
|
④ decide_n5 reasons 含分布行且出口不变(含 --no-strong-baseline 语义原样)——
|
|
test_decide_n5_reasons_infra_dist_and_exit_unchanged。
|
|
|
|
穿线红线复核:不塞 fail_system(游戏系统桶语义不污染)、退路树出口枚举与判定不动、台账向后兼容。
|
|
"""
|
|
import json
|
|
|
|
from worker.run_record import (
|
|
RunRecord,
|
|
infra_flags_from_runtime,
|
|
INFRA_FLAGS,
|
|
INFRA_FLAG_DESIGN_TEAM_DEGRADED,
|
|
INFRA_FLAG_WRITER_ITER_WALL,
|
|
INFRA_FLAG_STEP_CAP_TRIPPED,
|
|
INFRA_FLAG_SOFT_BUDGET_TRIPPED,
|
|
INFRA_FLAG_WALL_TIMEOUT,
|
|
)
|
|
from worker import fallback_tree as ft
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# 构造小工具
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
def _mk(ok: bool, *, fail_system=None, infra=None) -> RunRecord:
|
|
"""造一条 RunRecord:ok=True → 过门款;ok=False → 失败款(可带 fail_system / infra_flags)。"""
|
|
if ok:
|
|
return RunRecord(run_id="cv-r1-M3-面包-0", model="MiniMax-M3", stage="play",
|
|
brief_variant="面包", pass_gate=True, repairs=0,
|
|
decision="accept", finished=True, infra_flags=list(infra or []))
|
|
return RunRecord(run_id="cv-r1-M3-面包-0", model="MiniMax-M3", stage="build",
|
|
brief_variant="面包", pass_gate=False, repairs=1,
|
|
decision="fix", finished=False, fail_system=fail_system,
|
|
infra_flags=list(infra or []))
|
|
|
|
|
|
def _round(n_ok: int, n_fail: int, *, fail_system=None, infra=None) -> list:
|
|
"""造一轮:n_ok 过门 + n_fail 失败(失败款统一带 fail_system / infra_flags)。"""
|
|
return ([_mk(True) for _ in range(n_ok)]
|
|
+ [_mk(False, fail_system=fail_system, infra=infra) for _ in range(n_fail)])
|
|
|
|
|
|
class _FakeTimeoutErr(Exception):
|
|
"""fake 设计团队超时异常(带 .kind='timeout',镜像 design_team.DesignTeamError 的 kind 约定)。"""
|
|
kind = "timeout"
|
|
|
|
|
|
class _FakeDegradeErr(Exception):
|
|
"""fake 设计团队非超时降级异常(无 kind,如预算耗尽 / leader 空产出 / 团队异常)。"""
|
|
|
|
|
|
class _FakeBreaker:
|
|
"""fake CircuitBreakerMiddleware:只暴露 infra 采集读的 budget_soft_tripped 标记。"""
|
|
def __init__(self, budget_soft_tripped: bool = False) -> None:
|
|
self.budget_soft_tripped = budget_soft_tripped
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# ① 纯映射:infra_flags_from_runtime(编排层信号 → 受控值集)
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
def test_infra_flags_from_runtime_pure_mapping():
|
|
# 无信号 → 空列表。
|
|
assert infra_flags_from_runtime() == []
|
|
|
|
# 设计团队超时降级 → design_team_degraded + wall_timeout(稳定序=值集登记序)。
|
|
assert infra_flags_from_runtime(design_degraded=True, design_timeout=True) == [
|
|
INFRA_FLAG_DESIGN_TEAM_DEGRADED, INFRA_FLAG_WALL_TIMEOUT]
|
|
|
|
# 设计团队非超时降级 → 只 design_team_degraded。
|
|
assert infra_flags_from_runtime(design_degraded=True) == [INFRA_FLAG_DESIGN_TEAM_DEGRADED]
|
|
|
|
# 单写撞轮数墙 → writer_iter_wall。
|
|
assert infra_flags_from_runtime(writer_iter_wall=True) == [INFRA_FLAG_WRITER_ITER_WALL]
|
|
|
|
# 熔断 kind 映射:step_cap → step_cap_tripped;timeout → wall_timeout。
|
|
assert infra_flags_from_runtime(breaker_kind="step_cap") == [INFRA_FLAG_STEP_CAP_TRIPPED]
|
|
assert infra_flags_from_runtime(breaker_kind="timeout") == [INFRA_FLAG_WALL_TIMEOUT]
|
|
# budget/stuck 不入首批值集(由 circuit_break 字段另记)→ 空。
|
|
assert infra_flags_from_runtime(breaker_kind="budget") == []
|
|
assert infra_flags_from_runtime(breaker_kind="stuck") == []
|
|
|
|
# ¥ 软预算软停 → soft_budget_tripped。
|
|
assert infra_flags_from_runtime(budget_soft_tripped=True) == [INFRA_FLAG_SOFT_BUDGET_TRIPPED]
|
|
|
|
# 去重 + 稳定序:设计团队超时(wall_timeout)叠加熔断 timeout(wall_timeout)→ wall_timeout 只一份;
|
|
# 多信号同现按 INFRA_FLAGS 登记序输出(不随触发先后抖动)。
|
|
got = infra_flags_from_runtime(
|
|
design_degraded=True, design_timeout=True, writer_iter_wall=True,
|
|
breaker_kind="timeout", budget_soft_tripped=True)
|
|
assert got == [INFRA_FLAG_DESIGN_TEAM_DEGRADED, INFRA_FLAG_WRITER_ITER_WALL,
|
|
INFRA_FLAG_SOFT_BUDGET_TRIPPED, INFRA_FLAG_WALL_TIMEOUT]
|
|
assert got.count(INFRA_FLAG_WALL_TIMEOUT) == 1 # 去重坐实
|
|
# 输出只含受控值集里的项。
|
|
assert all(f in INFRA_FLAGS for f in got)
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# ① studio 态采集(构造 fake 运行态,委托纯映射)
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
def test_studio_collects_infra_flags_from_fake_runtime():
|
|
# 延迟 import(studio 拉 agentscope;与既有 middleware 测试同环境,baseline 已证可 import)。
|
|
from worker.agent_loop import studio
|
|
|
|
# 设计团队墙钟超时降级 → design_team_degraded + wall_timeout(studio 从异常 .kind 抽 design_timeout)。
|
|
flags = studio._infra_flags_from_runtime(
|
|
design_error=_FakeTimeoutErr("超时"), breaker=_FakeBreaker(),
|
|
breaker_tripped=None, hit_writer_iter_wall=False)
|
|
assert flags == [INFRA_FLAG_DESIGN_TEAM_DEGRADED, INFRA_FLAG_WALL_TIMEOUT]
|
|
|
|
# 非超时降级 → 只 design_team_degraded。
|
|
flags = studio._infra_flags_from_runtime(
|
|
design_error=_FakeDegradeErr("预算耗尽"), breaker=_FakeBreaker(),
|
|
breaker_tripped=None, hit_writer_iter_wall=False)
|
|
assert flags == [INFRA_FLAG_DESIGN_TEAM_DEGRADED]
|
|
|
|
# 撞轮数墙 + step_cap 熔断 + 软停:三信号齐 → 稳定序输出。
|
|
flags = studio._infra_flags_from_runtime(
|
|
design_error=None, breaker=_FakeBreaker(budget_soft_tripped=True),
|
|
breaker_tripped={"kind": "step_cap", "reason": "步数超顶"}, hit_writer_iter_wall=True)
|
|
assert flags == [INFRA_FLAG_WRITER_ITER_WALL, INFRA_FLAG_STEP_CAP_TRIPPED,
|
|
INFRA_FLAG_SOFT_BUDGET_TRIPPED]
|
|
|
|
# 全无编排层事件(正常收敛的 run)→ 空列表。
|
|
flags = studio._infra_flags_from_runtime(
|
|
design_error=None, breaker=_FakeBreaker(),
|
|
breaker_tripped=None, hit_writer_iter_wall=False)
|
|
assert flags == []
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# ② batch_run 落账含 infra_flags(result → RunRecord 接线)
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
def test_result_to_record_carries_infra_flags():
|
|
import batch_run
|
|
|
|
# 造一个形状对齐 studio result 的假结果:带 infra_flags(设计团队超时降级路)。
|
|
result = {
|
|
"model": "MiniMax-M3", "finished": False,
|
|
"last_verdict": {"decision": "fix", "layerResults": {"L1": {"passed": False}}},
|
|
"circuit_break": None, "breaker_counters": {"tool_calls": 3, "model_calls": 5},
|
|
"tokens": {"prompt": 100000, "completion": 30000, "cached": 5000},
|
|
"cost": {"cost_rmb": 1.2, "tokens_by_model": {}},
|
|
"wall_s": 90.0, "file_tree": [], "source_project": {},
|
|
"infra_flags": [INFRA_FLAG_DESIGN_TEAM_DEGRADED, INFRA_FLAG_WALL_TIMEOUT],
|
|
}
|
|
rec = batch_run.result_to_record(result, run_id="feie-r1-M3-面包-0",
|
|
brief_variant="面包", timestamp=1750000000.0)
|
|
assert rec.infra_flags == [INFRA_FLAG_DESIGN_TEAM_DEGRADED, INFRA_FLAG_WALL_TIMEOUT]
|
|
# 防御式复制:改 result 里的列表不应回污染 rec(不共享引用)。
|
|
result["infra_flags"].append("xxx")
|
|
assert rec.infra_flags == [INFRA_FLAG_DESIGN_TEAM_DEGRADED, INFRA_FLAG_WALL_TIMEOUT]
|
|
# 落盘 JSONL 往返一致(infra_flags 原样保留)。
|
|
back = RunRecord.from_jsonl_line(rec.to_jsonl_line())
|
|
assert back.infra_flags == [INFRA_FLAG_DESIGN_TEAM_DEGRADED, INFRA_FLAG_WALL_TIMEOUT]
|
|
|
|
# result 缺 infra_flags 键(旧桩)→ 落空列表,不炸。
|
|
result.pop("infra_flags")
|
|
rec2 = batch_run.result_to_record(result, run_id="feie-r1-M3-面包-1",
|
|
brief_variant="面包", timestamp=1750000000.0)
|
|
assert rec2.infra_flags == []
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# ③ 旧行(无 infra_flags 字段)解析兼容 —— 台账向后兼容红线
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
def test_from_jsonl_line_backward_compat():
|
|
# 旧台账行:没有 infra_flags 列(模拟本工单前落的行)。
|
|
old_line = json.dumps({
|
|
"run_id": "feie-001#1", "model": "MiniMax-M3", "stage": "play",
|
|
"brief_variant": "面包店", "pass": True, "repairs": 4,
|
|
"cost_yuan": 2.13, "wall_seconds": 187.4, "decision": "accept", "finished": True,
|
|
}, ensure_ascii=False)
|
|
rec = RunRecord.from_jsonl_line(old_line) # 不炸
|
|
assert rec.infra_flags == [] # 缺字段读为空列表
|
|
assert rec.pass_gate is True and rec.model == "MiniMax-M3" # 其余字段照常
|
|
|
|
# 新行:带 infra_flags,往返一致。
|
|
new_rec = _mk(False, infra=[INFRA_FLAG_STEP_CAP_TRIPPED])
|
|
line = new_rec.to_jsonl_line()
|
|
assert '"infra_flags"' in line and "step_cap_tripped" in line
|
|
assert RunRecord.from_jsonl_line(line).infra_flags == [INFRA_FLAG_STEP_CAP_TRIPPED]
|
|
|
|
# 未知多余列(台账演进新增)仍被过滤,不炸。
|
|
weird = json.loads(old_line)
|
|
weird["some_future_col"] = 123
|
|
weird["infra_flags"] = ["design_team_degraded"]
|
|
rec3 = RunRecord.from_jsonl_line(json.dumps(weird, ensure_ascii=False))
|
|
assert rec3.infra_flags == ["design_team_degraded"]
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# ④ decide_n5 reasons 含分布行 且 出口不变(含 --no-strong-baseline 语义原样)
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
def test_decide_n5_reasons_infra_dist_and_exit_unchanged():
|
|
# —— F-2 复现场景:两轮不收敛、失败款 fail_system 全空(退路树按 fail_system 看「分散」)、
|
|
# 但 infra_flags 全集中「设计团队超时降级路」(design_team_degraded + wall_timeout)。
|
|
# 这正是被误判分散出 R3_conditional 的生产盲区:穿线后 reasons 应打印出集中的 infra 分布。——
|
|
infra = [INFRA_FLAG_DESIGN_TEAM_DEGRADED, INFRA_FLAG_WALL_TIMEOUT]
|
|
rounds_flagged = [_round(3, 2, fail_system=None, infra=infra),
|
|
_round(3, 2, fail_system=None, infra=infra)]
|
|
# 同构但不带 infra_flags 的对照(证「加 infra_flags 不改出口」)。
|
|
rounds_plain = [_round(3, 2, fail_system=None),
|
|
_round(3, 2, fail_system=None)]
|
|
|
|
# 无强档对照(--no-strong-baseline 语义):失败分散无集中 → R3_conditional(出口不变)。
|
|
out_flagged = ft.decide_n5(rounds_flagged, has_strong_baseline=False)
|
|
out_plain = ft.decide_n5(rounds_plain, has_strong_baseline=False)
|
|
assert out_flagged["exit"] == ft.EXIT_R3_CONDITIONAL # 误判出口原样(本工单不改判定)
|
|
assert out_flagged["exit"] == out_plain["exit"] # 加 infra_flags 不改出口
|
|
# reasons 含各轮 infra 分布行,且点名集中的编排层原因(治盲区的核心穿线证据)。
|
|
joined = "\n".join(out_flagged["reasons"])
|
|
assert "infra 分布" in joined
|
|
assert "design_team_degraded:2" in joined and "wall_timeout:2" in joined
|
|
assert "r1 infra 分布" in joined and "r2 infra 分布" in joined
|
|
# 不带 infra_flags 的对照:无分布行(无 flag 不加噪)。
|
|
assert "infra 分布" not in "\n".join(out_plain["reasons"])
|
|
|
|
# 有强档对照:同场景 → R3_天花板(前瞻支);加 infra_flags 亦不改出口。
|
|
assert ft.decide_n5(rounds_flagged, has_strong_baseline=True)["exit"] == ft.EXIT_R3_CEILING
|
|
assert (ft.decide_n5(rounds_flagged, has_strong_baseline=True)["exit"]
|
|
== ft.decide_n5(rounds_plain, has_strong_baseline=True)["exit"])
|
|
|
|
# —— 收敛(go)出口也不受 infra_flags 影响:软预算软停发生在最终干净轮上,仍判 go,且打印分布。——
|
|
go_flagged = [_round(5, 0), [_mk(True, infra=[INFRA_FLAG_SOFT_BUDGET_TRIPPED]) for _ in range(5)]]
|
|
go_plain = [_round(5, 0), _round(5, 0)]
|
|
o_go = ft.decide_n5(go_flagged, has_strong_baseline=False)
|
|
assert o_go["exit"] == ft.EXIT_GO
|
|
assert o_go["exit"] == ft.decide_n5(go_plain, has_strong_baseline=False)["exit"]
|
|
assert "soft_budget_tripped:5" in "\n".join(o_go["reasons"])
|
|
|
|
# —— 集中表现层仍走 R1(fail_system 分流不被 infra_flags 干扰):证 fail_system 与 infra_flags 各管一摊。——
|
|
r1_flagged = [_round(1, 4, fail_system="presentation", infra=infra),
|
|
_round(1, 4, fail_system="presentation", infra=infra)]
|
|
r1_plain = [_round(1, 4, fail_system="presentation"),
|
|
_round(1, 4, fail_system="presentation")]
|
|
assert ft.decide_n5(r1_flagged, has_strong_baseline=False)["exit"] == ft.EXIT_R1_TEMPLATE
|
|
assert (ft.decide_n5(r1_flagged, has_strong_baseline=False)["exit"]
|
|
== ft.decide_n5(r1_plain, has_strong_baseline=False)["exit"])
|