games-development-ai/cheap-worker/hard_genre_batch.py
lili cbfd4d871b
Some checks failed
contract-gates / contract-gates (push) Has been cancelled
docs-gate / docs-gate (push) Has been cancelled
feat(acceptance): 闭合 playtest v3 与 A+ 可信消费链
固化 Match-3 生产者、视觉、音频与双 Judge 证据闭包。

将《山海行纪》r1.1 绑定新的不可变 release,并以生产预检现场核验 bundle、Registry/2 和 25 项 Writer 快照。

同步地图1平衡锁值、跨游戏回归修复、验收契约与 SoT 证据。
2026-07-28 20:16:13 -07:00

344 lines
21 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""hard_genre_batch.py — 便宜档【5 已覆盖品类】n=5 重测基线批跑W-AXIS 收尾)。
2026-07-09 换轴后重锚:验收权威 = accepted = 机械九门预筛 ∧ 独立模型玩法判定(glm-5.2 看真玩证据),
verdictPass 只是预筛地板、finished 只是模型自称收敛。本脚本作废换轴前那批全绿的污染数字(旧口径把
「九门 pass」当验收通过),在新链路(turns.jsonl 真相层 + per-run 归档 + 三层归因)与新语义下重测。
品类 = 5 个【有 per-genre 模板】的已覆盖品类(heritage/trpg/sim-business/puzzle/narrative);idle/action
无模板品类不入本轮(归 W-GENRE)。每品类各跑 n=5 局(distinct gid `hard-<genre>-r<round>`,绝不同 gid 重跑
洗数字——同 gid 会触发归档覆盖)。串行跑(本机 CPU+chrome 负载,别开并发)。
跑法(分级防无人值守烧钱):
冒烟 n=1×5: cheap-worker/.venv/bin/python cheap-worker/hard_genre_batch.py 1 1
全量补 r2-5: cheap-worker/.venv/bin/python cheap-worker/hard_genre_batch.py 4 2
参数:argv[1]=本次跑几轮(默认 1) argv[2]=起始轮号(默认 1);gid 后缀 = r<轮号>。
env:NO_PROXY(网关直连绕代理) + NEWAPI_KEY(_bootstrap 自动从凭据档解析)。
每局逐行追加 results/wax-baseline.jsonl(断点续跑不丢已完成局);累计成本超 ¥200 硬停(PARTIAL)。
"""
import asyncio
import json
import os
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import cheap_studio # noqa: E402
from cheap_genre_route import route_genre # noqa: E402 brief→per-genre 黄金模板路由(对齐生产 create 路)
from cheap_roles import SCAFFOLD_DESC_BY_TEMPLATE # noqa: E402 per-genre 起点一句话描述
from worker import genconfig # noqa: E402 与生产同读 genconfigmax_tokens 单一源)
# max_tokens 与生产 create 路cheap_service_driver同源 genconfig.model.max_tokens2026-07-09 调 32K16K 是自设限、
# 实测 M3 单轮能输出 17000+ tokensinline thinking 吃额度致大文件整写截断——给 thinking+代码留足)。别用 run_studio 的 16K 默认。
_MAXTOK = genconfig.get("model", "max_tokens", 16000)
# 累计成本硬停线(¥):无人值守跑 25 局,任何一轮累计越线立即停机 PARTIAL(单局另有 ¥10 软/¥15 硬 breaker,此为跨局兜底)。
_HARD_STOP_RMB = 200.0
# ── W-AXIS-V2 波3 批次隔离旋钮(env-gated,默认保持 07-09 hard 基线行为完全不变)──
# WAX_GID_PREFIX:gid 前缀(默认 hard);波3 传 wax2 → gid=wax2-<genre>-r<N>,与 07-09 基线 distinct、不撞归档。
# WAX_JSONL_NAME:落盘文件名(默认 wax-baseline.jsonl);波3 传 wax2-baseline.jsonl,两套基线干净分离可并排对账。
# WAX_ONLY_GENRE:只跑单品类(逐局单进程调用,适配 Bash 600s/局硬约束);默认空=全 5 品类。
_GID_PREFIX = os.environ.get("WAX_GID_PREFIX", "hard").strip() or "hard"
_JSONL_NAME = os.environ.get("WAX_JSONL_NAME", "wax-baseline.jsonl").strip() or "wax-baseline.jsonl"
_ONLY_GENRE = os.environ.get("WAX_ONLY_GENRE", "").strip()
# 结果逐行落盘(断点续跑不丢):每完成一局即 append 一行 JSON。
_JSONL = Path(__file__).resolve().parent / "results" / _JSONL_NAME
def _session_result_path(jsonl_path: Path, start_round: int, n_rounds: int) -> Path:
"""由逐局真相文件名派生本次聚合名,保证 wax/wax2 等批次命名空间不互相覆盖。"""
end_round = start_round + n_rounds - 1
return jsonl_path.with_name(f"{jsonl_path.stem}-r{start_round}-{end_round}.json")
def _aggregate_rows(jsonl_path: Path, start_round: int, n_rounds: int) -> list[dict]:
"""从逐行真相账重建指定轮次聚合,断点重入和分品类进程都不会覆盖既有结果。"""
if not jsonl_path.is_file():
return []
end_round = start_round + n_rounds - 1
rows = []
for line in jsonl_path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except json.JSONDecodeError:
continue
round_no = row.get("round")
if isinstance(round_no, int) and start_round <= round_no <= end_round:
rows.append(row)
return rows
# 5 个【已覆盖品类】brief——各命中一个 per-genre 黄金模板(route_genre 关键词确定性路由),取自批跑器既有 brief 池:
# narrative/trpg/heritage 用同题 brief(HARD 池);puzzle/sim-business 用跨主题 brief(XTHEME 池,该两类同题无 brief)。
# 路由核对:文字冒险/分支→story、掷骰→trpg、非遗→feiyi、华容道/解谜→puzzle、经营/奶茶店→shop。
COVERED_BRIEFS = [
("narrative", "一个分支选择推进的互动文字冒险:每屏一段剧情+2-3个选项,选择影响走向,最终3个不同结局,有结局图鉴"),
("trpg", "一个掷骰战斗的地牢爬塔:每层遇一个敌人,点击掷骰(可见骰点)按攻防结算,击败进下一层,失败结算,每3层升级选天赋"),
("heritage", "一个还原传统榫卯木工的非遗小游戏:按工序节拍(选料→画线→凿卯→修整)逐步操作,每步有节奏判定,完成得成品评分"),
("puzzle", "一个数字华容道解谜:滑动数字块把它们按序归位,规则逐关递进,卡壳给提示,通关计步数炫耀成绩"),
("sim-business", "一个经营奶茶店:顾客排队点单,按配方调对饮品上对得分,收银攒钱升级解锁新品与自动出杯设备"),
]
def _row_total_cost(row: dict) -> float:
"""一局的总成本 = 生成 costRmb + 判定 judge.costRmb + 测试员 playtest.costRmb(W-AXIS-V2 波2)。
v2 模式下 judge 段镜像 playtest 裁决、两者 costRmb 同值——取 max 只计一次(防同一笔钱双计);
shadow 模式下 judge 是判定器成本、playtest 是测试员成本,两笔真实分立——相加。据 acceptanceVersion 分辨。
"""
# v3 的 parentChainCostRmb 已含每轮 writer 与验收增量,是跨 repair 不重不漏的权威总成本。
if row.get("acceptanceVersion") in ("v3", "v3_shadow"):
try:
return float(row.get("parentChainCostRmb") or 0.0)
except (TypeError, ValueError):
return 0.0
gen = row.get("costRmb") or 0.0
j = row.get("judge") or {}
pt = row.get("playtest") or {}
jc = j.get("costRmb") or 0.0
pc = pt.get("costRmb") or 0.0
try:
if row.get("acceptanceVersion") == "v2":
extra = max(float(jc or 0.0), float(pc or 0.0)) # v2:judge 段镜像 playtest,同一笔钱别双计
else:
extra = float(jc or 0.0) + float(pc or 0.0) # shadow/v1:判定器与测试员各自真花,分立累计
return float(gen) + extra
except (TypeError, ValueError):
return 0.0
def _v3_batch_metrics(summary: dict) -> dict:
"""从完整封存对象提取 factual 质量账;兼容镜像不参与质量分子。"""
final = summary.get("acceptanceV3") if isinstance(summary.get("acceptanceV3"), dict) else None
if final is None:
return {}
first = (summary.get("acceptanceV3FirstPass")
if isinstance(summary.get("acceptanceV3FirstPass"), dict) else final)
decision = final.get("decision") if isinstance(final.get("decision"), dict) else {}
first_decision = first.get("decision") if isinstance(first.get("decision"), dict) else {}
merge = final.get("merge") if isinstance(final.get("merge"), dict) else {}
obligations = [row for row in final.get("proofObligations") or [] if isinstance(row, dict)]
required = [row for row in obligations if row.get("required") is True]
legacy_compatibility = final.get("compatibility") if isinstance(final.get("compatibility"), dict) else {}
legacy_playtest = (legacy_compatibility.get("playtest")
if isinstance(legacy_compatibility.get("playtest"), dict) else {})
factual_accepted = final.get("outcome") == "accept" and decision.get("accepted") is True
first_accepted = first.get("outcome") == "accept" and first_decision.get("accepted") is True
repair_attempted = isinstance(summary.get("acceptanceV3FirstPass"), dict)
return {
"accepted": factual_accepted,
"firstPassAccepted": first_accepted,
"acceptedAfterRepair": bool(repair_attempted and factual_accepted),
"repairAttempted": repair_attempted,
"outcome": final.get("outcome"),
"rescuedByRoll": decision.get("rescuedByRoll") or merge.get("rescuedByRoll"),
"publishFrozen": decision.get("publishFrozen"),
"proofComplete": (bool(required) and all(row.get("status") == "satisfied" for row in required))
if final.get("schemaVersion") == "playtest/3"
else legacy_playtest.get("proofComplete"),
"acceptanceCostRmb": final.get("costRmb"),
"parentChainCostRmb": decision.get("parentChainCostRmb"),
"acceptanceVersion": final.get("acceptanceMode"),
}
def _prior_cumulative() -> float:
"""读已落盘 JSONL 里所有历史局的累计成本(跨多次调用续跑时,¥200 兜底要含之前轮次已花)。"""
if not _JSONL.is_file():
return 0.0
total = 0.0
for line in _JSONL.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
total += _row_total_cost(json.loads(line))
except json.JSONDecodeError:
continue
return total
async def _run_one(genre, brief, port, cdp, round_no):
"""跑一局并返回批次账行(串行,无 semaphore)。gid=hard-<genre>-r<round>,distinct、绝不同 gid 重跑。"""
gid = f"{_GID_PREFIX}-{genre}-r{round_no}"
t0 = time.time()
# 对齐生产 create 路:brief→route_genre 选 per-genre 黄金模板作起点 + scaffold_desc 描述引导。
scaffold_template, routed_genre = route_genre(brief)
# 不写锁(2026-07-08/09 创始人纠偏,W-AXIS 波2 拆 reskin 管路):game-logic/core/render/assets 全放开,
# 只锁绝对通用 plumbing=L1_FIXED;写坏靠 harness 门 + 独立判定兜、不靠限死模型的手。
write_whitelist = None
scaffold_desc = SCAFFOLD_DESC_BY_TEMPLATE.get(scaffold_template) if scaffold_template else None
try:
summary = await cheap_studio.run_studio(
gid, brief, run_gates=True, port=port, cdp_port=cdp,
scaffold_template=scaffold_template, scaffold_desc=scaffold_desc,
write_whitelist=write_whitelist, genre=routed_genre, max_tokens=_MAXTOK)
except Exception as e: # noqa: BLE001 harness 级异常也原样计入基线(失败也是基线的一部分)
return {"genre": genre, "gid": gid, "round": round_no,
"accepted": False, "floorPass": None, "acceptanceVersion": None,
"verdictPass": None, "finished": False, "ok": False,
"judge": None, "fail": f"{type(e).__name__}: {e}", "wallSec": round(time.time() - t0, 1)}
v = summary.get("verdict") or {}
floor = summary.get("floor") or {} # 四门投影(run_acceptance 落;缺则空 → floorPass=None)
rich = summary.get("richness") or {}
layer = summary.get("failureLayer") or {} # classify_cheap_failure_layer 返回 {layer,reason,failedGates}
j = summary.get("judge") or {} # run_acceptance 写入(仅 run_gates∧finished 时有)
pt = summary.get("playtest") or {} # 测试员段(W-AXIS-V2 波2:v2/shadow 时有,批账观测口径)
v3_metrics = _v3_batch_metrics(summary)
return {
"genre": genre, "gid": gid, "round": round_no,
"template": scaffold_template or "_template(通用)",
# ── 验收信号分开记(W-AXIS-V2 波1 语义,别混):
# v3 用 factual decision 统计质量v3_shadow 的 compatibility.accepted=false 仅表示冻结发布,不代表真玩失败。
"accepted": v3_metrics.get("accepted", bool(summary.get("accepted"))),
"firstPassAccepted": v3_metrics.get("firstPassAccepted"),
"acceptedAfterRepair": v3_metrics.get("acceptedAfterRepair"),
"repairAttempted": v3_metrics.get("repairAttempted"),
"outcome": v3_metrics.get("outcome"),
"rescuedByRoll": v3_metrics.get("rescuedByRoll"),
"publishFrozen": v3_metrics.get("publishFrozen"),
"proofComplete": v3_metrics.get("proofComplete"),
"acceptanceCostRmb": v3_metrics.get("acceptanceCostRmb"),
"parentChainCostRmb": v3_metrics.get("parentChainCostRmb"),
"floorPass": floor.get("pass"), # ★预筛权威 = 四门投影(A/B/C/D),取代 verdictPass 作预筛口径
"acceptanceVersion": v3_metrics.get("acceptanceVersion", summary.get("acceptanceVersion")),
"verdictPass": v.get("pass"), # harness 原生九门/七门 AND,窗口内仅对照打印、【不作判定输入】
"finished": bool(summary.get("finished")), # 模型自称收敛(check+build 绿),不是验收
"ok": bool(summary.get("ok")), # run_studio 收口 ok(blocking 下 = accepted)
# ── 判定段(verdict/rejectClasses/degraded/costRmb + 诊断):
"judge": {
"ran": j.get("ran"), "verdict": j.get("verdict"),
"rejectClasses": j.get("rejectClasses"), "degraded": j.get("degraded"),
"costRmb": j.get("costRmb"), "reason": j.get("reason"),
"model": j.get("model"), "frames": j.get("frames"),
} if j else None,
# ── 测试员段(W-AXIS-V2 波2 §4:裁决/degraded/成本/rolls/首局三字段——批账观测口径):
"playtest": {
"accepted": pt.get("accepted"), "verdict": pt.get("verdict"),
"degraded": pt.get("degraded"), "imageBlind": pt.get("imageBlind"),
"rollCount": pt.get("rollCount"), "costRmb": pt.get("costRmb"),
"reason": pt.get("reason"), "summary": pt.get("summary"),
"firstPlay": pt.get("firstPlay"), "model": pt.get("model"),
} if pt else None,
# ── 失败归因三层(driver_contract/mechanical/gameplay/none)+ 归档指针:
# harness 原生失败门只作降观测对照,不参与 accepted/failureLayer/failureReason 最终归因。
"failedGates": v.get("failedGates"), # 兼容旧批账消费者,一个版本窗口后停读
"observedFailedGates": v.get("failedGates"), # 明确观测语义,保留 E/G/H 等证据但不误称最终失败
"failureLayer": layer.get("layer"),
"failureReason": layer.get("reason"),
"archivedPriorRunTo": summary.get("archivedPriorRunTo"), # distinct gid 首跑恒 None
"attempts": summary.get("attempts"),
"breaker": summary.get("breaker"),
"costRmb": summary.get("costRmb"), # 生成成本(不含判定)
"richness": rich.get("score"), "richMax": rich.get("max"),
"wallSec": round(time.time() - t0, 1),
}
def _mark(r: dict) -> str:
"""一局的达标标记(accepted 口径,W-AXIS-V2 波1):
✅ accepted=True(最终权威过)
⚠️测试员拒 四门过(floorPass=True)但测试员拒(accepted=False)——机械门放行、测试员逮住的坏游戏
finished-only 模型自称收敛(finished=True)但四门未过
❌ 未收敛/harness 挂
"""
if r.get("accepted"):
return ""
if r.get("floorPass") is True:
return "⚠️测试员拒"
if r.get("finished"):
return "finished-only"
return ""
async def main():
n_rounds = int(sys.argv[1]) if len(sys.argv) > 1 else 1
start_round = int(sys.argv[2]) if len(sys.argv) > 2 else 1
_JSONL.parent.mkdir(exist_ok=True)
cumulative = _prior_cumulative()
# 断点续跑:已落盘 gid 直接跳过——防 Bash 600s 超时重入时同 gid 重跑/双写,守零重跑纪律。
done_gids = set()
if _JSONL.is_file():
for line in _JSONL.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
done_gids.add(json.loads(line).get("gid"))
except json.JSONDecodeError:
continue
# 品类过滤(WAX_ONLY_GENRE 逐局单跑);保留原 index 稳定 port 映射,避免残留 chrome 端口撞。
briefs = [(i, g, b) for i, (g, b) in enumerate(COVERED_BRIEFS) if not _ONLY_GENRE or g == _ONLY_GENRE]
print(f">>> W-AXIS 重测基线:前缀={_GID_PREFIX} 落盘={_JSONL.name} 品类={[g for _, g, _ in briefs]} "
f"本次跑 {n_rounds} 轮(r{start_round}..r{start_round + n_rounds - 1}),串行;"
f"已落盘累计成本 ¥{cumulative:.2f}(硬停线 ¥{_HARD_STOP_RMB:.0f})")
session_rows = []
stopped = False
for round_no in range(start_round, start_round + n_rounds):
if stopped:
break
for cat_i, genre, brief in briefs:
gid = f"{_GID_PREFIX}-{genre}-r{round_no}"
if gid in done_gids:
print(f"{gid} 已在 JSONL,跳过(零重跑)")
continue
# 串行:每局固定 port(同 index),前一局 chrome 已收才起下一局。
r = await _run_one(genre, brief, 4340 + cat_i * 2, 9242 + cat_i * 2, round_no)
session_rows.append(r)
# 逐行落盘(断点续跑不丢)。
with _JSONL.open("a", encoding="utf-8") as f:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
cumulative += _row_total_cost(r)
j = r.get("judge") or {}
pt = r.get("playtest") or {}
print(f"{_mark(r):>13s} r{round_no} {genre:14s} tpl={r.get('template','?')} "
f"accepted={r.get('accepted')} floorPass={r.get('floorPass')} ver={r.get('acceptanceVersion')} "
f"verdictPass(对照)={r.get('verdictPass')} finished={r.get('finished')} "
f"playtest={pt.get('verdict')}/rolls={pt.get('rollCount')} ptDegraded={pt.get('degraded')} "
f"judge={j.get('verdict')}/{j.get('rejectClasses')} degraded={j.get('degraded')} "
f"layer={r.get('failureLayer')} attempts={r.get('attempts')} "
f"genCost=¥{r.get('costRmb')} judgeCost=¥{j.get('costRmb')} ptCost=¥{pt.get('costRmb')} wall={r.get('wallSec')}s "
f"累计=¥{cumulative:.2f} {('FAIL='+r['fail']) if r.get('fail') else ''}")
# ¥200 跨局硬停:立即停机,已完成局全在 JSONL,报 PARTIAL。
if cumulative > _HARD_STOP_RMB:
print(f"\n!!! 累计成本 ¥{cumulative:.2f} 越硬停线 ¥{_HARD_STOP_RMB:.0f} → 停机 PARTIAL(已完成 {len(session_rows)} 局本次)")
stopped = True
break
# ── 本次汇总(accepted 口径为主,floorPass/finished 并列;verdictPass 仅对照、不作判定输入)──
n = len(session_rows)
acc = sum(1 for r in session_rows if r.get("accepted"))
fp = sum(1 for r in session_rows if r.get("floorPass") is True) # 预筛权威 = 四门投影
vp = sum(1 for r in session_rows if r.get("verdictPass") is True) # harness 原生九门/七门,仅对照
fin = sum(1 for r in session_rows if r.get("finished"))
deg = sum(1 for r in session_rows if (r.get("judge") or {}).get("degraded")
or (r.get("playtest") or {}).get("degraded"))
print("\n========== 本次汇总(W-AXIS-V2 验收语义)==========")
print(f">>> 真实率(accepted 最终权威):{acc}/{n} = {acc / n:.0%}" if n else ">>> 无完成局")
if n:
print(f">>> 预筛(floorPass 四门投影):{fp}/{n}={fp / n:.0%} | 收敛(finished){fin}/{n}={fin / n:.0%} | degraded {deg}/{n}")
print(f">>> 对照(harness 原生 verdictPass,不作判定输入):{vp}/{n}={vp / n:.0%}")
print(f">>> 差额 = 四门过但测试员拒 {fp - acc} 局(机械门放行、测试员逮住)")
outcomes = {name: sum(1 for r in session_rows if r.get("outcome") == name)
for name in ("accept", "reject", "inconclusive", "tester_error")}
repaired = sum(1 for r in session_rows if r.get("acceptedAfterRepair"))
rescued = sum(1 for r in session_rows if r.get("rescuedByRoll") == 2)
frozen = sum(1 for r in session_rows if r.get("publishFrozen") is True)
proof = sum(1 for r in session_rows if r.get("proofComplete") is True)
print(f">>> v3 四态={outcomes} | repair 后接受={repaired} | 二掷救回={rescued} | 冻结={frozen} | 硬证完整={proof}")
print(f">>> 累计成本(本次+历史)= ¥{cumulative:.2f}")
# 聚合从权威 JSONL 按轮次重建:断点重入跳过旧 gid 时仍保留历史行,分品类进程也只会增量汇合。
out = _session_result_path(_JSONL, start_round, n_rounds)
out.write_text(json.dumps(_aggregate_rows(_JSONL, start_round, n_rounds), ensure_ascii=False, indent=2),
encoding="utf-8")
print(f">>> 本次结果 → {out};逐局全量 → {_JSONL}")
if __name__ == "__main__":
asyncio.run(main())