eb95441dd6
工单 f:_CheapRunCollector._flush 在 on_reply 的 REPLY_END / except / finally 三条降级路
会被调 ≥2 次,落盘每次覆盖是幂等的(采集语义不变、driver 恒读到最新),但「收口采集落盘」
成功行重复打印纯观感噪声。加实例标记 _flush_logged 只在首次成功落盘时打,后续静默覆盖
(失败行不受此门、每次都报便于诊断)。
工单 i:run_bakeoff 对无 brief 品类 skip+continue,全缺时 valid 空 → aggregate(genre_runs,[])
的 overall=(not [])and(not [])=True → 在 0 款上打「达标 ✅」exit 0(S5b 第一跑生产实录的
空过陷阱 / 无声截断)。抽 _resolve_briefs 纯解析 valid/missing;采样列表为空 → 立即
SystemExit(2)、stderr 打印缺失清单(品类 + 期望 run-summary 路径)、绝不进入判定;有样本
但部分缺 → 逐条告警后按既有逻辑继续(不升级为全停,judge_genre/aggregate 判定口径不改)。
测试 +6:工单 f 直接三次 _flush + 崩溃路端到端各验只打一行(capsys);工单 i 全缺 fail-fast
+缺失清单 / 正常路径行为不变 / 部分缺继续 / _resolve_briefs 拆分。全套 335 passed(329+6)。
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
253 lines
12 KiB
Python
253 lines
12 KiB
Python
"""
|
|
test_bake_off.py — 便宜档 ≥80% 达标门聚合/判定逻辑单测(M1 U3,mock verdict、不真跑)。
|
|
|
|
守的不变量:
|
|
· 质量口径:达标率分母 = 收敛款(finished=True);编排未收敛(finished=False)剔出、单列 unconverged。
|
|
· 按品类聚合 passed/converged、阈值 0.8(含边界 4/5=0.8 算达标)。
|
|
· 假绿防护:某品类全挂、其余全过 → 整体未达标(不被平均成总体达标)。
|
|
· 覆盖不足:核心品类无收敛样本 → missing、整体未达标。
|
|
· 未收敛剔除:未收敛款不拉低质量达标率,但 rawPassRate 并报(贴近生产真实交付率)。
|
|
· 并发上限夹取 ≤15。
|
|
· 判定零 LLM:judge/aggregate 纯函数,无任何模型调用、同输入恒同输出。
|
|
|
|
跑:cheap-worker/.venv/bin/python cheap-worker/tests/test_bake_off.py
|
|
"""
|
|
|
|
import asyncio
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # → cheap-worker/
|
|
import bake_off as B # noqa: E402
|
|
import compare_node as C # noqa: E402 工单 i:monkeypatch C.load_brief 造 brief 缺失/齐全
|
|
|
|
|
|
def _p(n):
|
|
"""n 款过九门(收敛 + verdict.pass)。"""
|
|
return [{"passed": True, "finished": True} for _ in range(n)]
|
|
|
|
|
|
def _f(n):
|
|
"""n 款九门质量挂(收敛但 verdict.pass=False)。"""
|
|
return [{"passed": False, "finished": True} for _ in range(n)]
|
|
|
|
|
|
def _u(n):
|
|
"""n 款编排未收敛(finished=False,无 verdict)。"""
|
|
return [{"passed": False, "finished": False} for _ in range(n)]
|
|
|
|
|
|
# ───────────────────────── 单品类判定 ─────────────────────────
|
|
|
|
def test_judge_genre_meets_full():
|
|
j = B.judge_genre(_p(5))
|
|
assert j["meets"] is True and j["passRate"] == 1.0 and j["status"] == "meets"
|
|
|
|
|
|
def test_judge_genre_boundary_080():
|
|
"""4/5 = 0.8 恰达线 → 达标。"""
|
|
j = B.judge_genre(_p(4) + _f(1))
|
|
assert j["passRate"] == 0.8 and j["meets"] is True
|
|
|
|
|
|
def test_judge_genre_below():
|
|
"""3/5 = 0.6 < 0.8 → below。"""
|
|
j = B.judge_genre(_p(3) + _f(2))
|
|
assert j["passRate"] == 0.6 and j["meets"] is False and j["status"] == "below"
|
|
|
|
|
|
def test_judge_genre_insufficient():
|
|
j = B.judge_genre([])
|
|
assert j["status"] == "insufficient" and j["meets"] is False and j["total"] == 0
|
|
|
|
|
|
def test_judge_genre_unconverged_excluded_from_denominator():
|
|
"""质量口径核心:11 过 + 1 质量挂 + 2 未收敛 = 质量 11/12=0.917 达标;原始 11/14=0.786。
|
|
创始人 2026-06-27 定:未收敛剔出分母、归 M2,达标门只衡量九门质量地板。"""
|
|
j = B.judge_genre(_p(11) + _f(1) + _u(2))
|
|
assert j["converged"] == 12 and j["unconverged"] == 2 and j["total"] == 14
|
|
assert j["passRate"] == 0.917 and j["meets"] is True
|
|
assert j["rawPassRate"] == 0.786 # 含未收敛、贴近生产真实交付率、不作判据
|
|
|
|
|
|
def test_judge_genre_all_unconverged_is_insufficient():
|
|
"""全未收敛 → 无收敛样本可判质量 → insufficient(不误判达标)。"""
|
|
j = B.judge_genre(_u(5))
|
|
assert j["status"] == "insufficient" and j["meets"] is False and j["converged"] == 0
|
|
|
|
|
|
# ───────────────────────── 整体聚合 ─────────────────────────
|
|
|
|
def test_aggregate_all_meet():
|
|
gp = {"a": _p(5), "b": _p(4) + _f(1), "c": _p(5)}
|
|
r = B.aggregate(gp, ["a", "b", "c"])
|
|
assert r["overallMeets"] is True and not r["belowGenres"] and not r["missingGenres"]
|
|
|
|
|
|
def test_aggregate_one_below_not_averaged():
|
|
"""假绿防护:c 全挂(0/5),a/b 全过 → 整体未达标(不被 a/b 平均掩盖)。"""
|
|
gp = {"a": _p(5), "b": _p(5), "c": _f(5)}
|
|
r = B.aggregate(gp, ["a", "b", "c"])
|
|
assert r["overallMeets"] is False and "c" in r["belowGenres"]
|
|
|
|
|
|
def test_aggregate_missing_genre_is_insufficient():
|
|
"""覆盖不足:c 缺样本 → missing、整体未达标(即便 a/b 全过)。"""
|
|
gp = {"a": _p(5), "b": _p(5)}
|
|
r = B.aggregate(gp, ["a", "b", "c"])
|
|
assert r["overallMeets"] is False and "c" in r["missingGenres"]
|
|
|
|
|
|
def test_aggregate_partial_below_boundary():
|
|
"""b 恰 0.8 达线、a/c 全过 → 整体达标(边界不误判)。"""
|
|
gp = {"a": _p(5), "b": _p(4) + _f(1), "c": _p(5)}
|
|
r = B.aggregate(gp, ["a", "b", "c"])
|
|
assert r["overallMeets"] is True
|
|
|
|
|
|
def test_aggregate_unconverged_total_surfaced():
|
|
"""编排未收敛跨品类汇总 unconvergedTotal、单列归 M2,不影响质量达标判定。"""
|
|
gp = {"a": _p(5) + _u(1), "b": _p(5), "c": _p(5) + _u(2)}
|
|
r = B.aggregate(gp, ["a", "b", "c"])
|
|
assert r["overallMeets"] is True and r["unconvergedTotal"] == 3 and r["sampleTotal"] == 18
|
|
|
|
|
|
# ───────────────────────── 丰富度分布(U-B1 · additive · 与达标正交)─────────────────────────
|
|
|
|
def _pr(n, score):
|
|
"""n 款过九门 + richness 评分 score(非降级)。"""
|
|
return [{"passed": True, "finished": True,
|
|
"richness": {"score": score, "max": 8, "degraded": False}} for _ in range(n)]
|
|
|
|
|
|
def _pd(n):
|
|
"""n 款过九门 + richness 降级(LLM 评分失败/超时,score=None)。"""
|
|
return [{"passed": True, "finished": True,
|
|
"richness": {"score": None, "max": 8, "degraded": True}} for _ in range(n)]
|
|
|
|
|
|
def test_richness_dist_basic():
|
|
"""丰富度分布:非降级款收集 score 算均分/分布,降级款单列计数(纯报告)。"""
|
|
d = B.richness_dist(_pr(2, 6) + _pr(1, 8) + _pd(1))
|
|
assert d["scoredCount"] == 3 and d["degradedCount"] == 1
|
|
assert d["meanScore"] == round((6 + 6 + 8) / 3, 2) and d["scores"] == [6, 6, 8] and d["max"] == 8
|
|
|
|
|
|
def test_richness_dist_all_degraded_or_missing():
|
|
"""全降级 / 无 richness 字段(对照路/老产物)→ meanScore=None、scoredCount=0,不报错。"""
|
|
d = B.richness_dist(_pd(2) + _p(1)) # _p 无 richness 字段
|
|
assert d["scoredCount"] == 0 and d["meanScore"] is None and d["degradedCount"] == 2
|
|
|
|
|
|
def test_aggregate_richness_additive_does_not_change_meets():
|
|
"""红线:richness 是 additive 报告 —— 加 richness 后 overallMeets / perGenre 达标判定与无 richness 时完全一致。"""
|
|
bare = {"a": _p(4) + _f(1)} # 无 richness
|
|
rich = {"a": _pr(4, 7) + _f(1)} # 4 过门款带 richness=7、1 质量挂
|
|
rb = B.aggregate(bare, ["a"])
|
|
rr = B.aggregate(rich, ["a"])
|
|
# 达标判定完全不受 richness 影响(判据只看 verdict.pass,与 richness 正交)
|
|
assert rr["overallMeets"] == rb["overallMeets"] is True
|
|
assert rr["perGenre"]["a"]["passRate"] == rb["perGenre"]["a"]["passRate"] == 0.8
|
|
assert rr["perGenre"]["a"]["meets"] == rb["perGenre"]["a"]["meets"] is True
|
|
# richness 分布 additive 出现,不混进达标判据
|
|
assert rr["richnessByGenre"]["a"]["meanScore"] == 7.0
|
|
assert rb["richnessByGenre"]["a"]["scoredCount"] == 0 # 无 richness 款 → 空分布,不报错
|
|
|
|
|
|
# ───────────────────────── 并发上限 / 零 LLM ─────────────────────────
|
|
|
|
def test_clamp_conc_upper_cap():
|
|
assert B._clamp_conc(100) == 15 # 不可超 15
|
|
assert B._clamp_conc(16) == 15
|
|
|
|
|
|
def test_clamp_conc_lower_and_bad():
|
|
assert B._clamp_conc(0) == 1
|
|
assert B._clamp_conc(3) == 3
|
|
assert B._clamp_conc("x") == 1 # 脏值兜底
|
|
|
|
|
|
def test_judge_is_deterministic_zero_llm():
|
|
"""判定全程纯函数(吃 run dict 出确定结论),不含任何模型调用 —— 同输入恒同输出。"""
|
|
gp = {"a": _p(4) + _f(1)}
|
|
r1 = B.aggregate(gp, ["a"])
|
|
r2 = B.aggregate(gp, ["a"])
|
|
assert r1["perGenre"]["a"] == r2["perGenre"]["a"]
|
|
assert r1["perGenre"]["a"]["passRate"] == 0.8 and r1["overallMeets"] is True
|
|
|
|
|
|
# ───────────────────────── brief 源缺失 fail-fast(工单 i · 空过陷阱兜底)─────────────────────────
|
|
|
|
def _fake_run_one(genre_key, brief, k, port, cdp):
|
|
"""mock 掉真跑(run_studio→M3/chrome/build):每款都收敛且过九门,模拟正常生产路的 run dict。"""
|
|
return {"gid": f"bake-{genre_key}-{k}", "passed": True, "finished": True,
|
|
"failedGates": None, "breakerKind": None, "costRmb": 0.3, "richness": None}
|
|
|
|
|
|
def test_run_bakeoff_all_briefs_missing_fails_fast(monkeypatch, capsys):
|
|
"""全缺:所有必需品类都无 brief → 立即非零退出(SystemExit code≠0),stderr 打印缺失清单(品类名+期望路径),
|
|
绝不进入 aggregate。守住空过陷阱:现状会在 0 款上打「达标 ✅」exit 0(S5b 第一跑生产实录的无声截断)。"""
|
|
monkeypatch.setattr(C, "load_brief", lambda genre: "") # 全部品类 brief 源为空
|
|
with pytest.raises(SystemExit) as ei:
|
|
asyncio.run(B.run_bakeoff(["click-score", "whack-mole", "shop-serve"], n=1, conc=1))
|
|
assert ei.value.code != 0, "空样本必须非零退出(区别于达标 exit 0)"
|
|
err = capsys.readouterr().err
|
|
assert "无任何可判样本" in err, "应拒绝在 0 款判达标、响亮失败"
|
|
for key in ("click-score", "whack-mole", "shop-serve"):
|
|
assert key in err, f"缺失清单应含品类名 {key}"
|
|
assert "run-summary.json" in err, "缺失清单应含期望 brief 路径(哪份 run-summary 缺)"
|
|
|
|
|
|
def test_run_bakeoff_normal_path_unchanged(tmp_path, monkeypatch):
|
|
"""正常路径(brief 齐全)行为不变:不 fail-fast、走既有 gather→aggregate→报告→返回,
|
|
overallMeets 仍由九门 verdict 决定(判定阈值/口径不改)。mock 掉真跑与报告落盘,只验判定装配。"""
|
|
monkeypatch.setattr(C, "load_brief", lambda genre: "一个点击得分小游戏") # brief 齐全
|
|
monkeypatch.setattr(B, "run_one_sync", _fake_run_one)
|
|
monkeypatch.setattr(B, "_next_report_path", lambda: tmp_path / "bake-off-test.json")
|
|
rep = asyncio.run(B.run_bakeoff(["click-score", "whack-mole"], n=2, conc=1)) # 不抛 SystemExit
|
|
assert rep["overallMeets"] is True # 两品类各 2/2 过 → 达标(判定口径不变)
|
|
assert rep["missingGenres"] == [] and rep["belowGenres"] == []
|
|
assert set(rep["perGenre"].keys()) == {"click-score", "whack-mole"}
|
|
assert rep["perGenre"]["click-score"]["passRate"] == 1.0
|
|
|
|
|
|
def test_run_bakeoff_partial_missing_continues(tmp_path, monkeypatch, capsys):
|
|
"""有样本但部分缺:click-score 有 brief、whack-mole 无 → 不升级为全停,打印缺失后按既有逻辑只判 click-score
|
|
(别把部分缺升级成全停,判定口径不改)。"""
|
|
monkeypatch.setattr(C, "load_brief",
|
|
lambda genre: "一个点击得分小游戏" if genre["key"] == "click-score" else "")
|
|
monkeypatch.setattr(B, "run_one_sync", _fake_run_one)
|
|
monkeypatch.setattr(B, "_next_report_path", lambda: tmp_path / "bake-off-partial.json")
|
|
rep = asyncio.run(B.run_bakeoff(["click-score", "whack-mole"], n=1, conc=1)) # 不抛 SystemExit
|
|
err = capsys.readouterr().err
|
|
assert "whack-mole" in err and "run-summary.json" in err, "缺失部分被逐条告警(品类+期望路径)"
|
|
assert set(rep["perGenre"].keys()) == {"click-score"}, "只判有 brief 的品类(既有逻辑)"
|
|
assert rep["overallMeets"] is True
|
|
|
|
|
|
def test_resolve_briefs_splits_valid_and_missing(monkeypatch):
|
|
"""_resolve_briefs 纯解析:有 brief→valid;无 brief→missing 带期望路径;未知品类→missing 带原因。"""
|
|
monkeypatch.setattr(C, "load_brief",
|
|
lambda genre: "brief" if genre["key"] == "click-score" else "")
|
|
valid, missing = B._resolve_briefs(["click-score", "whack-mole", "no-such-genre"])
|
|
assert [k for k, _ in valid] == ["click-score"]
|
|
missing_map = dict(missing)
|
|
assert "run-summary.json" in missing_map["whack-mole"] # 缺 brief → 期望路径
|
|
assert "未知品类" in missing_map["no-such-genre"] # 未知品类 → 原因
|
|
|
|
|
|
if __name__ == "__main__":
|
|
_fns = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)]
|
|
_failed = 0
|
|
for _fn in _fns:
|
|
try:
|
|
_fn()
|
|
print(f" PASS {_fn.__name__}")
|
|
except Exception as e: # noqa: BLE001
|
|
_failed += 1
|
|
print(f" FAIL {_fn.__name__}: {type(e).__name__}: {e}")
|
|
print(f"\n{len(_fns) - _failed}/{len(_fns)} passed")
|
|
sys.exit(1 if _failed else 0)
|