games-development-ai/cheap-worker/bake_off.py

"""
bake_off.py — 便宜档 ≥80% 达标门(按品类 · 零 LLM)。

代表样本(tap-targets occupied 三品类:点击得分/打地鼠/经营点客)各 n 款,每款:
  run_studio(run_gates=True) 生成 → 自动 spec(U2 补 prefixes)驱动九门 → 取 verdict.pass。
按品类聚合 pass_count/total,逐品类 ≥0.8 判达标;任一品类缺样本报「覆盖不足」、任一 <0.8 整体未达标
——绝不用总体平均掩盖卡死品类。判定纯走确定性九门 verdict.pass,不调任何 LLM 当裁判(防 Goodhart + 可复现)。

真跑前台进程内有界并发(端口池 + 线程,复用 compare_node)、conc≤15、gameId 品类前缀隔离、报告不覆盖。
小批(每品类 n=1–2)先联调聚合/判定逻辑;M3 主模型 20 款实测(约每品类 7)为 M1 收口硬判据(plan U3),
小批不替代收口。本机/实验室口径,生产真实分布达标在 M3 复验。

CLI:cheap-worker/.venv/bin/python cheap-worker/bake_off.py [--genres click-score,whack-mole,shop-serve] [--n 7] [--conc 3]
"""

import asyncio
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))  # → cheap-worker/
import _bootstrap  # noqa: E402,F401  跨包 sys.path + key 兜底
import cheap_studio  # noqa: E402
import compare_node as C  # noqa: E402  复用 GENRES / load_brief / _port_pool / _next_index / _GENRE_BY_KEY

_RESULTS_DIR = Path(__file__).resolve().parent / "results"
_PASS_THRESHOLD = 0.8   # 逐品类达标线(plan R3 ≥80%)
_CONC_CAP = 15          # 并发硬上限(plan R5;便宜档容量画像:瓶颈是 Chrome/esbuild 的 CPU 突发)


# ───────────────────────── 纯逻辑(可单测,不依赖真跑、零 LLM)─────────────────────────

def judge_genre(runs: list) -> dict:
    """单品类达标判定(质量口径,创始人 2026-06-27 定)。

    runs = [{"passed": bool, "finished": bool}, ...]。区分两类失败:
      · **编排未收敛**(finished=False:M3 撞 timeout/step_cap,根本没产出可玩游戏、无九门 verdict)
        —— 属生成稳定性,有独立编排闸在拦,归 M2 编排硬化,**不计入质量达标率分母**、单列 unconverged。
      · **九门质量挂**(finished=True 但 verdict.pass=False)—— 真·游戏不可玩,计入失败。

    `passRate` = 质量口径 = passed / converged(分母只数 finished 款),据它判 ≥0.8。
    `rawPassRate` = 原始口径 = passed / total(含未收敛,贴近生产真实交付率,供对照、不作判据)。
    converged==0(无任何款收敛产出 verdict)→ insufficient,质量无从判。
    """
    total = len(runs)
    unconverged = sum(1 for r in runs if not r.get("finished"))
    converged = total - unconverged
    passed = sum(1 for r in runs if r.get("finished") and r.get("passed"))
    raw_rate = round(passed / total, 3) if total else 0.0
    base = {"total": total, "converged": converged, "unconverged": unconverged,
            "passed": passed, "rawPassRate": raw_rate}
    if converged == 0:
        return {**base, "passRate": 0.0, "status": "insufficient", "meets": False}
    rate = round(passed / converged, 3)
    meets = rate >= _PASS_THRESHOLD
    return {**base, "passRate": rate, "status": "meets" if meets else "below", "meets": meets}


def aggregate(genre_runs: dict, required_genres: list) -> dict:
    """按品类聚合 + 整体达标判定(质量口径)。

    genre_runs = {key: [{"passed","finished"}, ...]}。整体达标 = 所有 required 核心品类
    都有收敛样本(converged>0)且逐品类质量达标率 ≥0.8。任一无收敛样本(missing)或任一 <0.8(below)
    即整体未达标 —— 卡死品类不被其余品类平均掩盖(plan R3 防自欺)。

    编排未收敛(timeout/step_cap)按质量口径剔出分母、单列 `unconvergedTotal` 作 M2 编排稳定性指标
    (创始人 2026-06-27 定:M1 达标门只衡量九门质量地板,未收敛归 M2 编排硬化)。
    """
    per_genre = {k: judge_genre(genre_runs.get(k, [])) for k in required_genres}
    missing = [k for k in required_genres if per_genre[k]["converged"] == 0]
    below = [k for k in required_genres if per_genre[k]["converged"] > 0 and not per_genre[k]["meets"]]
    overall = (not missing) and (not below)
    unconverged_total = sum(per_genre[k]["unconverged"] for k in required_genres)
    sample_total = sum(per_genre[k]["total"] for k in required_genres)
    return {
        "perGenre": per_genre,
        "requiredGenres": required_genres,
        "missingGenres": missing,
        "belowGenres": below,
        "overallMeets": overall,
        "threshold": _PASS_THRESHOLD,
        "unconvergedTotal": unconverged_total,   # 编排未收敛款数(M2 follow-up,不计入质量达标率)
        "sampleTotal": sample_total,
        "note": "质量口径(分母=收敛款,编排未收敛剔出、单列 unconvergedTotal 归 M2);"
                "本机/实验室口径,生产真实分布达标在 M3 复验;判定零 LLM(纯确定性九门 verdict.pass)",
    }


def _clamp_conc(conc) -> int:
    """并发夹到 [1, 15](plan R5 上限纪律:不可超 15)。"""
    try:
        conc = int(conc)
    except (TypeError, ValueError):
        conc = 1
    return max(1, min(_CONC_CAP, conc))


def _next_report_path() -> Path:
    """报告分批次文件、不覆盖历史(plan U3)。"""
    _RESULTS_DIR.mkdir(exist_ok=True)
    idx = C._next_index([p.stem for p in _RESULTS_DIR.glob("bake-off-*.json")])
    return _RESULTS_DIR / f"bake-off-{idx}.json"


# ───────────────────────── 真跑单款(run_studio 生成 + 自动 spec 九门)─────────────────────────

def run_one_sync(genre_key: str, brief: str, k: int, port: int, cdp: int) -> dict:
    """一款达标样本(同步,由 run_bakeoff 经 to_thread 并发调度):生成→自动 spec→九门,取 verdict.pass。

    走生产形态(run_gates=True:run_studio 内部 ensure_play_spec 自动 spec + 九门 play),非金标注入
    —— 达标门量的就是「生产自动 spec 驱动能不能稳定过九门」。
    """
    gid = f"bake-{genre_key}-{k}"
    summary = asyncio.run(cheap_studio.run_studio(gid, brief, run_gates=True, port=port, cdp_port=cdp))
    v = summary.get("verdict")  # _verdict_brief: {pass, failedGates} 或 None
    breaker = summary.get("breaker")  # 未收敛时 {kind, reason}:timeout / step_cap / budget
    return {"gid": gid, "passed": bool(v and v.get("pass")), "finished": summary.get("finished"),
            "failedGates": (v or {}).get("failedGates"),
            "breakerKind": (breaker or {}).get("kind"),  # 未收敛原因(归 M2 编排稳定性诊断)
            "costRmb": summary.get("costRmb")}


async def run_bakeoff(genre_keys: list, n: int, conc: int = 1, offset: int = 0,
                      base_port: int = 4320, base_cdp: int = 9222) -> dict:
    """三品类 × n 达标批跑:逐款 run_studio(自动 spec 九门),按品类聚合 ≥80%。

    前台进程内有界并发:端口池(每槽独立 port/cdp)+ 信号量限并发(≤conc),每款经 to_thread 独立线程跑
    (线程内 run_studio 自带 event loop)。禁后台子代理 / monitor / 自我唤醒重试。
    """
    conc = _clamp_conc(conc)
    n = max(1, int(n))
    pool = asyncio.Queue()  # 端口池:size=conc,get 空则阻塞 → 天然限并发到 ≤conc
    for pp in C._port_pool(conc, base_port, base_cdp):
        pool.put_nowait(pp)

    valid = []
    genre_runs = {}
    for key in genre_keys:
        genre = C._GENRE_BY_KEY.get(key)
        if genre is None:
            print(f"[skip] 未知品类 {key}", file=sys.stderr)
            continue
        brief = C.load_brief(genre)
        if not brief:
            print(f"[skip] {key} 无 brief(base {genre['base']} run-summary 缺失)", file=sys.stderr)
            continue
        valid.append((key, brief))
        genre_runs[key] = []

    async def one(key, brief, k):
        port, cdp = await pool.get()
        try:
            print(f"[{key} {k + 1}/{n}] port={port}/{cdp} 生成→自动spec→九门 …", file=sys.stderr)
            r = await asyncio.to_thread(run_one_sync, key, brief, offset + k, port, cdp)
            genre_runs[key].append(r)
            print(f"[{key} {k + 1}/{n}] passed={r['passed']} finished={r['finished']} cost={r['costRmb']}",
                  file=sys.stderr)
        except Exception as e:  # noqa: BLE001  单款失败不拖垮整批,记录后继续
            print(f"[{key} {k + 1}/{n}] 异常:{type(e).__name__}: {e}", file=sys.stderr)
        finally:
            pool.put_nowait((port, cdp))  # 归还端口

    tasks = [one(key, brief, k) for (key, brief) in valid for k in range(n)]
    print(f"[bake-off] {len(valid)} 品类 × n={n} = {len(tasks)} 款,conc={conc}(端口段 {base_port}.. / {base_cdp}..)",
          file=sys.stderr)
    await asyncio.gather(*tasks)

    report = aggregate(genre_runs, [k for (k, _) in valid])  # 质量口径:judge_genre 直吃 run dict(剔未收敛)
    report["runs"] = genre_runs
    out = _next_report_path()
    out.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
    print(f"[report] → {out}", file=sys.stderr)
    print_report(report)
    return report


def print_report(report: dict) -> None:
    print("\n=== 便宜档 ≥80% 达标门(按品类 · 自动 spec 驱动九门 · 零 LLM · 质量口径)===", file=sys.stderr)
    for k, j in report["perGenre"].items():
        unc = f"  +{j['unconverged']} 未收敛(剔出)" if j.get("unconverged") else ""
        print(f"[{k}] 质量 {j['passed']}/{j['converged']} = {j['passRate']}  {j['status']}"
              f"  (原始 {j['rawPassRate']}/{j['total']} 款){unc}", file=sys.stderr)
    flag = "达标 ✅" if report["overallMeets"] else "未达标 ❌"
    print(f">>> 整体:{flag}(逐品类阈值 {report['threshold']})  缺样本:{report['missingGenres']}  "
          f"不达标:{report['belowGenres']}", file=sys.stderr)
    print(f">>> 编排未收敛:{report.get('unconvergedTotal', 0)}/{report.get('sampleTotal', 0)} 款"
          f"(timeout/step_cap,归 M2 编排硬化、不计质量分母)", file=sys.stderr)
    print(f">>> 注:{report['note']}\n", file=sys.stderr)


if __name__ == "__main__":
    import argparse

    ap = argparse.ArgumentParser(description="便宜档 ≥80% 达标门(按品类 · 零 LLM)")
    ap.add_argument("--genres", default="click-score,whack-mole,shop-serve",
                    help="逗号分隔品类键(默认三 tap-targets 代表品类)")
    ap.add_argument("--n", type=int, default=7, help="每品类款数(20 款收口约每品类 7;小批联调用 1–2)")
    ap.add_argument("--conc", type=int, default=1, help="并发款数(端口池+线程,≤15;本机 Chrome 建议 ≤4)")
    ap.add_argument("--offset", type=int, default=0, help="gameId 起始索引(补跑用,避免覆盖既有 bake-<key>-<k>)")
    a = ap.parse_args()
    keys = [k.strip() for k in a.genres.split(",") if k.strip()]
    rep = asyncio.run(run_bakeoff(keys, a.n, conc=a.conc, offset=a.offset))
    # 退出码:整体达标 0、未达标 1。
    sys.exit(0 if rep["overallMeets"] else 1)