"""hard_genre_batch.py — 便宜档【5 已覆盖品类】n=5 重测基线批跑(W-AXIS 收尾)。 2026-07-09 换轴后重锚:验收权威 = accepted = 机械九门预筛 ∧ 独立模型玩法判定(glm-5.2 看真玩证据), verdictPass 只是预筛地板、finished 只是模型自称收敛。本脚本作废换轴前那批全绿的污染数字(旧口径把 「九门 pass」当验收通过),在新链路(turns.jsonl 真相层 + per-run 归档 + 三层归因)与新语义下重测。 品类 = 5 个【有 per-genre 模板】的已覆盖品类(heritage/trpg/sim-business/puzzle/narrative);idle/action 无模板品类不入本轮(归 W-GENRE)。每品类各跑 n=5 局(distinct gid `hard--r`,绝不同 gid 重跑 洗数字——同 gid 会触发归档覆盖)。串行跑(本机 CPU+chrome 负载,别开并发)。 跑法(分级防无人值守烧钱): 冒烟 n=1×5: cheap-worker/.venv/bin/python cheap-worker/hard_genre_batch.py 1 1 全量补 r2-5: cheap-worker/.venv/bin/python cheap-worker/hard_genre_batch.py 4 2 参数:argv[1]=本次跑几轮(默认 1) argv[2]=起始轮号(默认 1);gid 后缀 = r<轮号>。 env:NO_PROXY(网关直连绕代理) + NEWAPI_KEY(_bootstrap 自动从凭据档解析)。 每局逐行追加 results/wax-baseline.jsonl(断点续跑不丢已完成局);累计成本超 ¥200 硬停(PARTIAL)。 """ import asyncio import json import os import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import cheap_studio # noqa: E402 from cheap_genre_route import route_genre # noqa: E402 brief→per-genre 黄金模板路由(对齐生产 create 路) from cheap_roles import SCAFFOLD_DESC_BY_TEMPLATE # noqa: E402 per-genre 起点一句话描述 from worker import genconfig # noqa: E402 与生产同读 genconfig(max_tokens 单一源) # max_tokens 与生产 create 路(cheap_service_driver)同源 genconfig.model.max_tokens(2026-07-09 调 32K:16K 是自设限、 # 实测 M3 单轮能输出 17000+ tokens,inline thinking 吃额度致大文件整写截断——给 thinking+代码留足)。别用 run_studio 的 16K 默认。 _MAXTOK = genconfig.get("model", "max_tokens", 16000) # 累计成本硬停线(¥):无人值守跑 25 局,任何一轮累计越线立即停机 PARTIAL(单局另有 ¥10 软/¥15 硬 breaker,此为跨局兜底)。 _HARD_STOP_RMB = 200.0 # ── W-AXIS-V2 波3 批次隔离旋钮(env-gated,默认保持 07-09 hard 基线行为完全不变)── # WAX_GID_PREFIX:gid 前缀(默认 hard);波3 传 wax2 → gid=wax2--r,与 07-09 基线 distinct、不撞归档。 # WAX_JSONL_NAME:落盘文件名(默认 wax-baseline.jsonl);波3 传 wax2-baseline.jsonl,两套基线干净分离可并排对账。 # WAX_ONLY_GENRE:只跑单品类(逐局单进程调用,适配 Bash 600s/局硬约束);默认空=全 5 品类。 _GID_PREFIX = os.environ.get("WAX_GID_PREFIX", "hard").strip() or "hard" _JSONL_NAME = os.environ.get("WAX_JSONL_NAME", "wax-baseline.jsonl").strip() or "wax-baseline.jsonl" _ONLY_GENRE = os.environ.get("WAX_ONLY_GENRE", "").strip() # 结果逐行落盘(断点续跑不丢):每完成一局即 append 一行 JSON。 _JSONL = Path(__file__).resolve().parent / "results" / _JSONL_NAME def _session_result_path(jsonl_path: Path, start_round: int, n_rounds: int) -> Path: """由逐局真相文件名派生本次聚合名,保证 wax/wax2 等批次命名空间不互相覆盖。""" end_round = start_round + n_rounds - 1 return jsonl_path.with_name(f"{jsonl_path.stem}-r{start_round}-{end_round}.json") def _aggregate_rows(jsonl_path: Path, start_round: int, n_rounds: int) -> list[dict]: """从逐行真相账重建指定轮次聚合,断点重入和分品类进程都不会覆盖既有结果。""" if not jsonl_path.is_file(): return [] end_round = start_round + n_rounds - 1 rows = [] for line in jsonl_path.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line: continue try: row = json.loads(line) except json.JSONDecodeError: continue round_no = row.get("round") if isinstance(round_no, int) and start_round <= round_no <= end_round: rows.append(row) return rows # 5 个【已覆盖品类】brief——各命中一个 per-genre 黄金模板(route_genre 关键词确定性路由),取自批跑器既有 brief 池: # narrative/trpg/heritage 用同题 brief(HARD 池);puzzle/sim-business 用跨主题 brief(XTHEME 池,该两类同题无 brief)。 # 路由核对:文字冒险/分支→story、掷骰→trpg、非遗→feiyi、华容道/解谜→puzzle、经营/奶茶店→shop。 COVERED_BRIEFS = [ ("narrative", "一个分支选择推进的互动文字冒险:每屏一段剧情+2-3个选项,选择影响走向,最终3个不同结局,有结局图鉴"), ("trpg", "一个掷骰战斗的地牢爬塔:每层遇一个敌人,点击掷骰(可见骰点)按攻防结算,击败进下一层,失败结算,每3层升级选天赋"), ("heritage", "一个还原传统榫卯木工的非遗小游戏:按工序节拍(选料→画线→凿卯→修整)逐步操作,每步有节奏判定,完成得成品评分"), ("puzzle", "一个数字华容道解谜:滑动数字块把它们按序归位,规则逐关递进,卡壳给提示,通关计步数炫耀成绩"), ("sim-business", "一个经营奶茶店:顾客排队点单,按配方调对饮品上对得分,收银攒钱升级解锁新品与自动出杯设备"), ] def _row_total_cost(row: dict) -> float: """一局的总成本 = 生成 costRmb + 判定 judge.costRmb + 测试员 playtest.costRmb(W-AXIS-V2 波2)。 v2 模式下 judge 段镜像 playtest 裁决、两者 costRmb 同值——取 max 只计一次(防同一笔钱双计); shadow 模式下 judge 是判定器成本、playtest 是测试员成本,两笔真实分立——相加。据 acceptanceVersion 分辨。 """ # v3 的 parentChainCostRmb 已含每轮 writer 与验收增量,是跨 repair 不重不漏的权威总成本。 if row.get("acceptanceVersion") in ("v3", "v3_shadow"): try: return float(row.get("parentChainCostRmb") or 0.0) except (TypeError, ValueError): return 0.0 gen = row.get("costRmb") or 0.0 j = row.get("judge") or {} pt = row.get("playtest") or {} jc = j.get("costRmb") or 0.0 pc = pt.get("costRmb") or 0.0 try: if row.get("acceptanceVersion") == "v2": extra = max(float(jc or 0.0), float(pc or 0.0)) # v2:judge 段镜像 playtest,同一笔钱别双计 else: extra = float(jc or 0.0) + float(pc or 0.0) # shadow/v1:判定器与测试员各自真花,分立累计 return float(gen) + extra except (TypeError, ValueError): return 0.0 def _v3_batch_metrics(summary: dict) -> dict: """从完整封存对象提取 factual 质量账;兼容镜像不参与质量分子。""" final = summary.get("acceptanceV3") if isinstance(summary.get("acceptanceV3"), dict) else None if final is None: return {} first = (summary.get("acceptanceV3FirstPass") if isinstance(summary.get("acceptanceV3FirstPass"), dict) else final) decision = final.get("decision") if isinstance(final.get("decision"), dict) else {} first_decision = first.get("decision") if isinstance(first.get("decision"), dict) else {} merge = final.get("merge") if isinstance(final.get("merge"), dict) else {} obligations = [row for row in final.get("proofObligations") or [] if isinstance(row, dict)] required = [row for row in obligations if row.get("required") is True] legacy_compatibility = final.get("compatibility") if isinstance(final.get("compatibility"), dict) else {} legacy_playtest = (legacy_compatibility.get("playtest") if isinstance(legacy_compatibility.get("playtest"), dict) else {}) factual_accepted = final.get("outcome") == "accept" and decision.get("accepted") is True first_accepted = first.get("outcome") == "accept" and first_decision.get("accepted") is True repair_attempted = isinstance(summary.get("acceptanceV3FirstPass"), dict) return { "accepted": factual_accepted, "firstPassAccepted": first_accepted, "acceptedAfterRepair": bool(repair_attempted and factual_accepted), "repairAttempted": repair_attempted, "outcome": final.get("outcome"), "rescuedByRoll": decision.get("rescuedByRoll") or merge.get("rescuedByRoll"), "publishFrozen": decision.get("publishFrozen"), "proofComplete": (bool(required) and all(row.get("status") == "satisfied" for row in required)) if final.get("schemaVersion") == "playtest/3" else legacy_playtest.get("proofComplete"), "acceptanceCostRmb": final.get("costRmb"), "parentChainCostRmb": decision.get("parentChainCostRmb"), "acceptanceVersion": final.get("acceptanceMode"), } def _prior_cumulative() -> float: """读已落盘 JSONL 里所有历史局的累计成本(跨多次调用续跑时,¥200 兜底要含之前轮次已花)。""" if not _JSONL.is_file(): return 0.0 total = 0.0 for line in _JSONL.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line: continue try: total += _row_total_cost(json.loads(line)) except json.JSONDecodeError: continue return total async def _run_one(genre, brief, port, cdp, round_no): """跑一局并返回批次账行(串行,无 semaphore)。gid=hard--r,distinct、绝不同 gid 重跑。""" gid = f"{_GID_PREFIX}-{genre}-r{round_no}" t0 = time.time() # 对齐生产 create 路:brief→route_genre 选 per-genre 黄金模板作起点 + scaffold_desc 描述引导。 scaffold_template, routed_genre = route_genre(brief) # 不写锁(2026-07-08/09 创始人纠偏,W-AXIS 波2 拆 reskin 管路):game-logic/core/render/assets 全放开, # 只锁绝对通用 plumbing=L1_FIXED;写坏靠 harness 门 + 独立判定兜、不靠限死模型的手。 write_whitelist = None scaffold_desc = SCAFFOLD_DESC_BY_TEMPLATE.get(scaffold_template) if scaffold_template else None try: summary = await cheap_studio.run_studio( gid, brief, run_gates=True, port=port, cdp_port=cdp, scaffold_template=scaffold_template, scaffold_desc=scaffold_desc, write_whitelist=write_whitelist, genre=routed_genre, max_tokens=_MAXTOK) except Exception as e: # noqa: BLE001 harness 级异常也原样计入基线(失败也是基线的一部分) return {"genre": genre, "gid": gid, "round": round_no, "accepted": False, "floorPass": None, "acceptanceVersion": None, "verdictPass": None, "finished": False, "ok": False, "judge": None, "fail": f"{type(e).__name__}: {e}", "wallSec": round(time.time() - t0, 1)} v = summary.get("verdict") or {} floor = summary.get("floor") or {} # 四门投影(run_acceptance 落;缺则空 → floorPass=None) rich = summary.get("richness") or {} layer = summary.get("failureLayer") or {} # classify_cheap_failure_layer 返回 {layer,reason,failedGates} j = summary.get("judge") or {} # run_acceptance 写入(仅 run_gates∧finished 时有) pt = summary.get("playtest") or {} # 测试员段(W-AXIS-V2 波2:v2/shadow 时有,批账观测口径) v3_metrics = _v3_batch_metrics(summary) return { "genre": genre, "gid": gid, "round": round_no, "template": scaffold_template or "_template(通用)", # ── 验收信号分开记(W-AXIS-V2 波1 语义,别混): # v3 用 factual decision 统计质量;v3_shadow 的 compatibility.accepted=false 仅表示冻结发布,不代表真玩失败。 "accepted": v3_metrics.get("accepted", bool(summary.get("accepted"))), "firstPassAccepted": v3_metrics.get("firstPassAccepted"), "acceptedAfterRepair": v3_metrics.get("acceptedAfterRepair"), "repairAttempted": v3_metrics.get("repairAttempted"), "outcome": v3_metrics.get("outcome"), "rescuedByRoll": v3_metrics.get("rescuedByRoll"), "publishFrozen": v3_metrics.get("publishFrozen"), "proofComplete": v3_metrics.get("proofComplete"), "acceptanceCostRmb": v3_metrics.get("acceptanceCostRmb"), "parentChainCostRmb": v3_metrics.get("parentChainCostRmb"), "floorPass": floor.get("pass"), # ★预筛权威 = 四门投影(A/B/C/D),取代 verdictPass 作预筛口径 "acceptanceVersion": v3_metrics.get("acceptanceVersion", summary.get("acceptanceVersion")), "verdictPass": v.get("pass"), # harness 原生九门/七门 AND,窗口内仅对照打印、【不作判定输入】 "finished": bool(summary.get("finished")), # 模型自称收敛(check+build 绿),不是验收 "ok": bool(summary.get("ok")), # run_studio 收口 ok(blocking 下 = accepted) # ── 判定段(verdict/rejectClasses/degraded/costRmb + 诊断): "judge": { "ran": j.get("ran"), "verdict": j.get("verdict"), "rejectClasses": j.get("rejectClasses"), "degraded": j.get("degraded"), "costRmb": j.get("costRmb"), "reason": j.get("reason"), "model": j.get("model"), "frames": j.get("frames"), } if j else None, # ── 测试员段(W-AXIS-V2 波2 §4:裁决/degraded/成本/rolls/首局三字段——批账观测口径): "playtest": { "accepted": pt.get("accepted"), "verdict": pt.get("verdict"), "degraded": pt.get("degraded"), "imageBlind": pt.get("imageBlind"), "rollCount": pt.get("rollCount"), "costRmb": pt.get("costRmb"), "reason": pt.get("reason"), "summary": pt.get("summary"), "firstPlay": pt.get("firstPlay"), "model": pt.get("model"), } if pt else None, # ── 失败归因三层(driver_contract/mechanical/gameplay/none)+ 归档指针: # harness 原生失败门只作降观测对照,不参与 accepted/failureLayer/failureReason 最终归因。 "failedGates": v.get("failedGates"), # 兼容旧批账消费者,一个版本窗口后停读 "observedFailedGates": v.get("failedGates"), # 明确观测语义,保留 E/G/H 等证据但不误称最终失败 "failureLayer": layer.get("layer"), "failureReason": layer.get("reason"), "archivedPriorRunTo": summary.get("archivedPriorRunTo"), # distinct gid 首跑恒 None "attempts": summary.get("attempts"), "breaker": summary.get("breaker"), "costRmb": summary.get("costRmb"), # 生成成本(不含判定) "richness": rich.get("score"), "richMax": rich.get("max"), "wallSec": round(time.time() - t0, 1), } def _mark(r: dict) -> str: """一局的达标标记(accepted 口径,W-AXIS-V2 波1): ✅ accepted=True(最终权威过) ⚠️测试员拒 四门过(floorPass=True)但测试员拒(accepted=False)——机械门放行、测试员逮住的坏游戏 finished-only 模型自称收敛(finished=True)但四门未过 ❌ 未收敛/harness 挂 """ if r.get("accepted"): return "✅" if r.get("floorPass") is True: return "⚠️测试员拒" if r.get("finished"): return "finished-only" return "❌" async def main(): n_rounds = int(sys.argv[1]) if len(sys.argv) > 1 else 1 start_round = int(sys.argv[2]) if len(sys.argv) > 2 else 1 _JSONL.parent.mkdir(exist_ok=True) cumulative = _prior_cumulative() # 断点续跑:已落盘 gid 直接跳过——防 Bash 600s 超时重入时同 gid 重跑/双写,守零重跑纪律。 done_gids = set() if _JSONL.is_file(): for line in _JSONL.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line: continue try: done_gids.add(json.loads(line).get("gid")) except json.JSONDecodeError: continue # 品类过滤(WAX_ONLY_GENRE 逐局单跑);保留原 index 稳定 port 映射,避免残留 chrome 端口撞。 briefs = [(i, g, b) for i, (g, b) in enumerate(COVERED_BRIEFS) if not _ONLY_GENRE or g == _ONLY_GENRE] print(f">>> W-AXIS 重测基线:前缀={_GID_PREFIX} 落盘={_JSONL.name} 品类={[g for _, g, _ in briefs]} " f"本次跑 {n_rounds} 轮(r{start_round}..r{start_round + n_rounds - 1}),串行;" f"已落盘累计成本 ¥{cumulative:.2f}(硬停线 ¥{_HARD_STOP_RMB:.0f})") session_rows = [] stopped = False for round_no in range(start_round, start_round + n_rounds): if stopped: break for cat_i, genre, brief in briefs: gid = f"{_GID_PREFIX}-{genre}-r{round_no}" if gid in done_gids: print(f" ⏭ {gid} 已在 JSONL,跳过(零重跑)") continue # 串行:每局固定 port(同 index),前一局 chrome 已收才起下一局。 r = await _run_one(genre, brief, 4340 + cat_i * 2, 9242 + cat_i * 2, round_no) session_rows.append(r) # 逐行落盘(断点续跑不丢)。 with _JSONL.open("a", encoding="utf-8") as f: f.write(json.dumps(r, ensure_ascii=False) + "\n") cumulative += _row_total_cost(r) j = r.get("judge") or {} pt = r.get("playtest") or {} print(f"{_mark(r):>13s} r{round_no} {genre:14s} tpl={r.get('template','?')} " f"accepted={r.get('accepted')} floorPass={r.get('floorPass')} ver={r.get('acceptanceVersion')} " f"verdictPass(对照)={r.get('verdictPass')} finished={r.get('finished')} " f"playtest={pt.get('verdict')}/rolls={pt.get('rollCount')} ptDegraded={pt.get('degraded')} " f"judge={j.get('verdict')}/{j.get('rejectClasses')} degraded={j.get('degraded')} " f"layer={r.get('failureLayer')} attempts={r.get('attempts')} " f"genCost=¥{r.get('costRmb')} judgeCost=¥{j.get('costRmb')} ptCost=¥{pt.get('costRmb')} wall={r.get('wallSec')}s " f"累计=¥{cumulative:.2f} {('FAIL='+r['fail']) if r.get('fail') else ''}") # ¥200 跨局硬停:立即停机,已完成局全在 JSONL,报 PARTIAL。 if cumulative > _HARD_STOP_RMB: print(f"\n!!! 累计成本 ¥{cumulative:.2f} 越硬停线 ¥{_HARD_STOP_RMB:.0f} → 停机 PARTIAL(已完成 {len(session_rows)} 局本次)") stopped = True break # ── 本次汇总(accepted 口径为主,floorPass/finished 并列;verdictPass 仅对照、不作判定输入)── n = len(session_rows) acc = sum(1 for r in session_rows if r.get("accepted")) fp = sum(1 for r in session_rows if r.get("floorPass") is True) # 预筛权威 = 四门投影 vp = sum(1 for r in session_rows if r.get("verdictPass") is True) # harness 原生九门/七门,仅对照 fin = sum(1 for r in session_rows if r.get("finished")) deg = sum(1 for r in session_rows if (r.get("judge") or {}).get("degraded") or (r.get("playtest") or {}).get("degraded")) print("\n========== 本次汇总(W-AXIS-V2 验收语义)==========") print(f">>> 真实率(accepted 最终权威):{acc}/{n} = {acc / n:.0%}" if n else ">>> 无完成局") if n: print(f">>> 预筛(floorPass 四门投影):{fp}/{n}={fp / n:.0%} | 收敛(finished){fin}/{n}={fin / n:.0%} | degraded {deg}/{n}") print(f">>> 对照(harness 原生 verdictPass,不作判定输入):{vp}/{n}={vp / n:.0%}") print(f">>> 差额 = 四门过但测试员拒 {fp - acc} 局(机械门放行、测试员逮住)") outcomes = {name: sum(1 for r in session_rows if r.get("outcome") == name) for name in ("accept", "reject", "inconclusive", "tester_error")} repaired = sum(1 for r in session_rows if r.get("acceptedAfterRepair")) rescued = sum(1 for r in session_rows if r.get("rescuedByRoll") == 2) frozen = sum(1 for r in session_rows if r.get("publishFrozen") is True) proof = sum(1 for r in session_rows if r.get("proofComplete") is True) print(f">>> v3 四态={outcomes} | repair 后接受={repaired} | 二掷救回={rescued} | 冻结={frozen} | 硬证完整={proof}") print(f">>> 累计成本(本次+历史)= ¥{cumulative:.2f}") # 聚合从权威 JSONL 按轮次重建:断点重入跳过旧 gid 时仍保留历史行,分品类进程也只会增量汇合。 out = _session_result_path(_JSONL, start_round, n_rounds) out.write_text(json.dumps(_aggregate_rows(_JSONL, start_round, n_rounds), ensure_ascii=False, indent=2), encoding="utf-8") print(f">>> 本次结果 → {out};逐局全量 → {_JSONL}") if __name__ == "__main__": asyncio.run(main())