From 423b0b6df3e8c8b354e532108fafc13a59bd03b8 Mon Sep 17 00:00:00 2001 From: lili Date: Sat, 27 Jun 2026 08:41:46 -0700 Subject: [PATCH] =?UTF-8?q?feat(cheap-worker):=20auto-vs-golden=20?= =?UTF-8?q?=E9=80=90=E9=97=A8=20delta=20=E5=88=A4=E6=8D=AE=20+=20double-lo?= =?UTF-8?q?w=20=E5=AE=88=E5=8D=AB(M2=20U1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按品类逐门比 auto 过门率 vs golden 过门率,关键门 {E_live,H_progress,G_input} 容差 0(delta≥0)、其余门容差 -1/N。方向单边查「自动不比金标驱得差」(delta<0 退化),非查更松。double-low 守卫:关键门两路都 <0.5 → inconclusive(红),防 delta=0 假象(承 compare_node.doubleLowRed 同源风险)。aggregate_delta 三品类 全 aligned 才整体 meets、卡死品类不被平均掩盖。纯逻辑零 LLM。单测 11/11。 Co-Authored-By: Claude Opus 4.8 (1M context) --- cheap-worker/auto_vs_golden.py | 93 +++++++++++++++ cheap-worker/tests/test_auto_vs_golden.py | 132 ++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 cheap-worker/auto_vs_golden.py create mode 100644 cheap-worker/tests/test_auto_vs_golden.py diff --git a/cheap-worker/auto_vs_golden.py b/cheap-worker/auto_vs_golden.py new file mode 100644 index 00000000..fbcfe350 --- /dev/null +++ b/cheap-worker/auto_vs_golden.py @@ -0,0 +1,93 @@ +""" +auto_vs_golden.py — M2 auto-vs-golden delta 门(Node 退役授权第三条 · 按品类 · 零 LLM)。 + +对同一款生成产物,在同一份 staged src/ 上先用生产自动 spec(ensure_play_spec)、后用金标 spec +(inject_golden)各驱动一次九门,逐门比过门率。方向单边(承 plan① M2「关键门不得低于金标基线」): +查的是自动 spec 是否比金标**驱得差**(delta<0 退化),不是查自动比金标松——后者对断言同质的 +tap-targets 不成立(M1 已把自动 spec 断言加厚到与金标逐字段一致)。 + +这道门在 tap-targets 覆盖面内鉴别力有限(delta≈0 几必然),诚实定性为「不退化确认 + 逐款驱动器 +推断不失效」,非决定性独立闸;Node 退役授权的强证据是 M1 绝对达标 + 002 两路等价(见 U3 三条齐)。 + +U1 逐门 delta 判据(本段,纯逻辑可单测);U2 同游戏双驱动 play 真跑;U3 三条齐退役授权判定。 +""" + +# 九门里唯一对驱动器敏感的三门(未驱动时 E_live/H_progress 降 advisory、G_input 无输入 skip): +# 「这游戏到底能不能玩」的判据,自动低于金标即驱动更差,容差 0。 +_KEY_GATES = frozenset({"E_live", "H_progress", "G_input"}) +_DOUBLE_LOW = 0.5 # 关键门 auto 与 golden 过门率都 <此 → 两路同挂、delta=0 假象 → inconclusive(红)。 +_EPS = 1e-9 + + +def _rate(runs: list, side: str, gate: str) -> float: + """该侧(autoGates/goldenGates)某门在 N 款里的过门率。""" + total = len(runs) + if total == 0: + return 0.0 + return round(sum(1 for r in runs if (r.get(side) or {}).get(gate)) / total, 3) + + +def judge_genre_delta(runs: list) -> dict: + """单品类逐门 delta 判据。 + + runs = [{"autoGates": {门: pass_bool}, "goldenGates": {门: pass_bool}}, ...]。 + 逐门 delta = auto 过门率 - golden 过门率;关键门容差 0(delta≥0)、其余门容差 -1/N(单款 flake)。 + double-low 守卫:关键门两路都 <0.5 → inconclusive(红),不被 delta=0 误判 aligned。 + status:aligned(不退化)/ regressed(自动驱得差)/ inconclusive(关键门双低)/ insufficient(空样本)。 + """ + total = len(runs) + if total == 0: + return {"total": 0, "perGate": {}, "regressedGates": [], "inconclusiveGates": [], + "status": "insufficient", "meets": False} + + # 门集 = 各款 autoGates 键的并集(稳健于个别款缺门)。 + gate_names = sorted({g for r in runs for g in (r.get("autoGates") or {})}) + nonkey_tol = -1.0 / total + + per_gate, inconclusive, regressed = {}, [], [] + for g in gate_names: + auto_rate = _rate(runs, "autoGates", g) + golden_rate = _rate(runs, "goldenGates", g) + delta = round(auto_rate - golden_rate, 3) + per_gate[g] = {"autoRate": auto_rate, "goldenRate": golden_rate, "delta": delta} + is_key = g in _KEY_GATES + # double-low:关键门两路同低 → 红、不计入 aligned(不论 delta)。 + if is_key and auto_rate < _DOUBLE_LOW and golden_rate < _DOUBLE_LOW: + inconclusive.append(g) + continue + tol = 0.0 if is_key else nonkey_tol + if delta < tol - _EPS: + regressed.append(g) + + if inconclusive: + status = "inconclusive" + elif regressed: + status = "regressed" + else: + status = "aligned" + return {"total": total, "perGate": per_gate, "regressedGates": regressed, + "inconclusiveGates": inconclusive, "status": status, "meets": status == "aligned"} + + +def aggregate_delta(genre_runs: dict, required_genres: list) -> dict: + """按品类聚合 auto-vs-golden delta 判据。 + + 整体 meets = 所有 required 品类都有样本 且 逐品类 aligned(不退化、无关键门双低)。 + 任一缺样本(missing)/ regressed / inconclusive → 整体 meets=False,卡死品类不被平均掩盖。 + """ + per_genre = {k: judge_genre_delta(genre_runs.get(k, [])) for k in required_genres} + missing = [k for k in required_genres if per_genre[k]["total"] == 0] + regressed = [k for k in required_genres if per_genre[k]["status"] == "regressed"] + inconclusive = [k for k in required_genres if per_genre[k]["status"] == "inconclusive"] + meets = (not missing) and (not regressed) and (not inconclusive) + return { + "perGenre": per_genre, + "requiredGenres": required_genres, + "missingGenres": missing, + "regressedGenres": regressed, + "inconclusiveGenres": inconclusive, + "overallMeets": meets, + "meets": meets, + "note": "auto-vs-golden delta 门(同款双驱动 · 关键门容差 0 · 零 LLM);" + "覆盖面内鉴别力有限、定性为不退化确认,退役强证据=M1 达标+002 等价(见 U3)", + } diff --git a/cheap-worker/tests/test_auto_vs_golden.py b/cheap-worker/tests/test_auto_vs_golden.py new file mode 100644 index 00000000..763f3629 --- /dev/null +++ b/cheap-worker/tests/test_auto_vs_golden.py @@ -0,0 +1,132 @@ +""" +test_auto_vs_golden.py — M2 auto-vs-golden 门单测(U1 逐门 delta 判据 + U2 编排 mock + U3 退役授权)。 + +守的不变量(U1): + · 逐门 delta = auto 过门率 - golden 过门率;关键门 {E_live,H_progress,G_input} 容差 0(delta≥0)、 + 其余门容差 -1/N(允许单款 flake)。 + · 方向单边:查「自动不比金标驱得差」(delta<0 退化),非查「自动比金标松」。 + · double-low 守卫:关键门 auto 与 golden 过门率都 <0.5(两路同挂)→ inconclusive(红)、不误判 aligned。 + · 按品类聚合:三品类全 aligned 才整体 meets;任一 regressed/inconclusive/缺样本 → meets=False。 + · 零 LLM:纯函数、同输入恒同输出。 + +跑:cheap-worker/.venv/bin/python cheap-worker/tests/test_auto_vs_golden.py +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # → cheap-worker/ +import auto_vs_golden as A # noqa: E402 + +_ALL_GATES = ["A_boot", "B_uncaught", "C_frame", "D_render", "E_live", + "F_wiring", "G_input", "H_progress", "I_control"] + + +def _gates(fail=()): + """九门全过,fail 里的门置 False。""" + return {g: (g not in fail) for g in _ALL_GATES} + + +def _run(auto_fail=(), golden_fail=()): + """一款:auto/golden 两组逐门(默认全过,指定门挂)。""" + return {"autoGates": _gates(auto_fail), "goldenGates": _gates(golden_fail)} + + +# ───────────────────────── U1 逐门 delta 判据 ───────────────────────── + +def test_delta_all_equal_aligned(): + """auto 每门 = golden(delta 全 0)→ aligned、meets。""" + j = A.judge_genre_delta([_run() for _ in range(5)]) + assert j["status"] == "aligned" and j["meets"] is True + assert j["perGate"]["H_progress"]["delta"] == 0.0 + + +def test_delta_key_gate_lower_one_sample_regressed(): + """关键门 H_progress auto 比 golden 低一款(delta=-0.2)→ regressed(关键门容差 0)、meets=False。""" + runs = [_run(auto_fail=("H_progress",))] + [_run() for _ in range(4)] + j = A.judge_genre_delta(runs) + assert j["status"] == "regressed" and j["meets"] is False + assert "H_progress" in j["regressedGates"] + assert j["perGate"]["H_progress"]["delta"] == -0.2 + + +def test_delta_nonkey_gate_lower_one_sample_within_tolerance(): + """非关键门 D_render auto 低一款(delta=-1/5=-0.2)→ 在容差内 → aligned。""" + runs = [_run(auto_fail=("D_render",))] + [_run() for _ in range(4)] + j = A.judge_genre_delta(runs) + assert j["status"] == "aligned" and j["meets"] is True + + +def test_delta_nonkey_gate_lower_two_samples_regressed(): + """非关键门 D_render auto 低两款(delta=-0.4 < -1/5)→ regressed。""" + runs = [_run(auto_fail=("D_render",)), _run(auto_fail=("D_render",))] + [_run() for _ in range(3)] + j = A.judge_genre_delta(runs) + assert j["status"] == "regressed" and "D_render" in j["regressedGates"] + + +def test_delta_auto_higher_than_golden_not_regressed(): + """方向单边:auto 关键门比 golden 高(golden 挂、auto 过,delta=+0.2)→ 不算退化、aligned + (门查的是 auto 不比 golden 差,不查 auto 比 golden 松)。""" + runs = [_run(golden_fail=("H_progress",))] + [_run() for _ in range(4)] + j = A.judge_genre_delta(runs) + assert j["status"] == "aligned" and j["meets"] is True + assert j["perGate"]["H_progress"]["delta"] == 0.2 + + +def test_delta_double_low_key_gate_inconclusive(): + """double-low:关键门 E_live auto 与 golden 过门率都 <0.5(5 款里 3 款两路同挂)→ inconclusive(红)、 + meets=False,不被 delta=0 误判 aligned。""" + runs = [_run(auto_fail=("E_live",), golden_fail=("E_live",)) for _ in range(3)] + [_run() for _ in range(2)] + j = A.judge_genre_delta(runs) + assert j["status"] == "inconclusive" and j["meets"] is False + assert "E_live" in j["inconclusiveGates"] + + +def test_delta_insufficient_empty(): + j = A.judge_genre_delta([]) + assert j["status"] == "insufficient" and j["meets"] is False and j["total"] == 0 + + +def test_delta_deterministic_zero_llm(): + """纯函数、同输入恒同输出。""" + runs = [_run(auto_fail=("D_render",))] + [_run() for _ in range(4)] + assert A.judge_genre_delta(runs) == A.judge_genre_delta(runs) + + +# ───────────────────────── U1 按品类聚合 ───────────────────────── + +def test_aggregate_all_aligned_meets(): + gr = {"click-score": [_run() for _ in range(5)], + "whack-mole": [_run() for _ in range(5)], + "shop-serve": [_run() for _ in range(5)]} + r = A.aggregate_delta(gr, ["click-score", "whack-mole", "shop-serve"]) + assert r["meets"] is True and not r["regressedGenres"] and not r["missingGenres"] + + +def test_aggregate_one_regressed_fails(): + """某品类关键门退化 → 整体 meets=False、不被其余品类平均掩盖。""" + gr = {"click-score": [_run() for _ in range(5)], + "whack-mole": [_run(auto_fail=("H_progress",))] + [_run() for _ in range(4)], + "shop-serve": [_run() for _ in range(5)]} + r = A.aggregate_delta(gr, ["click-score", "whack-mole", "shop-serve"]) + assert r["meets"] is False and "whack-mole" in r["regressedGenres"] + + +def test_aggregate_missing_genre_fails(): + gr = {"click-score": [_run() for _ in range(5)], "whack-mole": [_run() for _ in range(5)]} + r = A.aggregate_delta(gr, ["click-score", "whack-mole", "shop-serve"]) + assert r["meets"] is False and "shop-serve" in r["missingGenres"] + + +if __name__ == "__main__": + _fns = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] + _failed = 0 + for _fn in _fns: + try: + _fn() + print(f" PASS {_fn.__name__}") + except Exception as e: # noqa: BLE001 + _failed += 1 + print(f" FAIL {_fn.__name__}: {type(e).__name__}: {e}") + print(f"\n{len(_fns) - _failed}/{len(_fns)} passed") + sys.exit(1 if _failed else 0)