""" test_gameplay_judge.py — W-AXIS 波2 独立模型玩法地板判定单测(schema / fail-closed / ok 语义切换 / 金标读取器)。 三层,全部零真网络(fake model 注入 + 注入证据 + tmp 证据目录): ① parse_floor_judgment 纯函数:accept/reject 三类解析、LLM verdict 覆盖、裸 bool 容忍、缺字段/坏 JSON → fail-closed degraded。 ② judge_gameplay_floor 契约:无证据 → fail-closed;fake 模型 accept/异常/超时 → 结构正确;成本记账字段。 ③ apply_gameplay_judge ok 语义:预筛未过不跑判定;accept∧blocking→ok=accepted;reject/degraded∧blocking→fail-closed; blocking=false→观测不阻断(ok=预筛)。外加金标读取器 load_golden(五份待标注种子)与 _collect_frames 排序/采样。 跑:cheap-worker/.venv/bin/python -m pytest cheap-worker/tests/test_gameplay_judge.py -q 或:cheap-worker/.venv/bin/python cheap-worker/tests/test_gameplay_judge.py """ import asyncio import json import sys import tempfile from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # → cheap-worker/ import cheap_verify # noqa: E402 import judge_golden # noqa: E402 # ── fake 判定模型(async 可调用;默认无 usage_sum,使 judge 的成本块走 except、零网络)── class _FakeResp: def __init__(self, text): self.content = text class _FakeJudge: """返回固定 JSON / 抛异常 / 睡眠(测超时)。默认不带 usage_sum(成本块 AttributeError→跳过,零网络)。""" def __init__(self, text="", raise_exc=None, sleep=0.0): self._text, self._raise, self._sleep = text, raise_exc, sleep async def __call__(self, messages, **kw): if self._sleep: await asyncio.sleep(self._sleep) if self._raise: raise self._raise return _FakeResp(self._text) class _FakeJudgeWithUsage(_FakeJudge): """带 usage_sum + records 的 fake(测成本记账字段流转)。""" records = [(1000, 200, 100)] def usage_sum(self): return (1000, 200) def _accept_json(): return json.dumps({"broken": {"problem": False, "why": "能玩通"}, "hollow": {"problem": False, "why": "有决策"}, "offBrief": {"problem": False, "why": "切题"}, "verdict": "accept", "notes": "地板成立"}, ensure_ascii=False) def _reject_json(cls="hollow"): node = {"broken": {"problem": False, "why": "x"}, "hollow": {"problem": False, "why": "x"}, "offBrief": {"problem": False, "why": "x"}} key = {"broken": "broken", "hollow": "hollow", "off_brief": "offBrief"}[cls] node[key] = {"problem": True, "why": "有问题"} node["verdict"] = "reject" node["notes"] = "地板不成立" return json.dumps(node, ensure_ascii=False) def _tmp_evidence(with_first=True, mids=0, with_after=True): """建 tmp 证据目录 + 写占位 png(内容随意,_collect_frames 只 base64 字节)。返回目录路径。""" d = Path(tempfile.mkdtemp()) if with_first: (d / "first-paint.png").write_bytes(b"\x89PNG-first") for i in range(1, mids + 1): (d / f"midplay-{i}.png").write_bytes(b"\x89PNG-mid%d" % i) if with_after: (d / "after-play.png").write_bytes(b"\x89PNG-after") return str(d) # ───────────────────────── ① parse_floor_judgment 纯函数 ───────────────────────── def test_parse_accept(): r = cheap_verify.parse_floor_judgment(_accept_json()) assert r["degraded"] is False assert r["accepted"] is True and r["verdict"] == "accept" assert r["rejectClasses"] == [] assert len(r["checks"]) == 3 and {c["class"] for c in r["checks"]} == {"broken", "hollow", "off_brief"} def test_parse_reject_hollow(): r = cheap_verify.parse_floor_judgment(_reject_json("hollow")) assert r["degraded"] is False assert r["accepted"] is False and r["verdict"] == "reject" assert r["rejectClasses"] == ["hollow"] def test_parse_reject_broken_and_offbrief(): node = {"broken": {"problem": True, "why": "b"}, "hollow": {"problem": False, "why": "x"}, "offBrief": {"problem": True, "why": "o"}, "verdict": "reject"} r = cheap_verify.parse_floor_judgment(json.dumps(node)) assert r["accepted"] is False assert set(r["rejectClasses"]) == {"broken", "off_brief"} def test_parse_llm_verdict_reject_overrides(): """三类 problem 都 false 但 LLM verdict=reject → 取交后仍判 reject(fail-closed 偏严,任一拒信号即拒)。""" node = {"broken": {"problem": False}, "hollow": {"problem": False}, "offBrief": {"problem": False}, "verdict": "reject"} r = cheap_verify.parse_floor_judgment(json.dumps(node)) assert r["accepted"] is False and r["rejectClasses"] == [] def test_parse_bare_bool_tolerance(): """模型偷懒直接给裸 bool(非 {problem,why})也接住。""" node = {"broken": False, "hollow": True, "offBrief": False, "verdict": "reject"} r = cheap_verify.parse_floor_judgment(json.dumps(node)) assert r["accepted"] is False and r["rejectClasses"] == ["hollow"] def test_parse_garbage_degraded(): r = cheap_verify.parse_floor_judgment("完全不是 JSON") assert r["degraded"] is True and r["accepted"] is False and r["rejectClasses"] == ["degraded"] def test_parse_missing_class_degraded(): """缺任一类判定字段 → 无法确认地板 → fail-closed degraded(不静默判过)。""" r = cheap_verify.parse_floor_judgment(json.dumps({"broken": {"problem": False}, "verdict": "accept"})) assert r["degraded"] is True and r["accepted"] is False def test_parse_none_empty_degraded(): assert cheap_verify.parse_floor_judgment(None)["degraded"] is True assert cheap_verify.parse_floor_judgment("")["degraded"] is True assert cheap_verify.parse_floor_judgment(" ")["degraded"] is True def test_parse_fenced_json(): """markdown 围栏 + 赘语 → json_repair 兜住。""" text = "评定如下:\n```json\n" + _accept_json() + "\n```" r = cheap_verify.parse_floor_judgment(text) assert r["degraded"] is False and r["accepted"] is True # ───────────────────────── ② judge_gameplay_floor 契约 ───────────────────────── def test_judge_no_evidence_failclosed(): """无截图证据(注入空 frames)→ fail-closed degraded,不放行,且未调用模型。""" r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", brief="b", frames=[], model=_FakeJudge(raise_exc=RuntimeError("不该被调用")))) assert r["degraded"] is True and r["accepted"] is False assert "证据缺失" in r.get("reason", "") def test_judge_happy_accept(): """注入 frames + fake accept → 结构化 accept、model/frames 观测。""" r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", brief="b", frames=["data:image/png;base64,AAAA"], model=_FakeJudge(text=_accept_json()))) assert r["degraded"] is False and r["accepted"] is True assert r["model"] == cheap_verify._JUDGE_DEFAULT_MODEL and r["frames"] == 1 def test_judge_reject_from_model(): r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", frames=["data:image/png;base64,AAAA"], model=_FakeJudge(text=_reject_json("broken")))) assert r["accepted"] is False and r["rejectClasses"] == ["broken"] def test_judge_model_raises_degraded(): r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", frames=["data:image/png;base64,AAAA"], model=_FakeJudge(raise_exc=RuntimeError("boom")))) assert r["degraded"] is True and r["accepted"] is False def test_judge_timeout_degraded(): r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", frames=["data:image/png;base64,AAAA"], model=_FakeJudge(text=_accept_json(), sleep=0.3), timeout=0.05)) assert r["degraded"] is True and r["accepted"] is False def test_judge_cost_accounting(): """带 usage 的 fake + 打桩 _estimate_judge_cost → judgeTokens/costRmb 落进结果(零网络)。""" orig = cheap_verify._estimate_judge_cost cheap_verify._estimate_judge_cost = lambda mn, ti, to, cached: 0.087 try: r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", frames=["data:image/png;base64,AAAA"], model=_FakeJudgeWithUsage(text=_accept_json()))) finally: cheap_verify._estimate_judge_cost = orig assert r["judgeTokens"] == {"in": 1000, "out": 200, "total": 1200} assert r["costRmb"] == 0.087 # ───────────────────────── ③ apply_gameplay_judge ok 语义切换 ───────────────────────── def test_apply_prefilter_false_no_judge(): """预筛未过 → 不跑判定、不花¥;accepted=False、ok=False、judge.ran=False。""" s = {"ok": False, "gameId": "g"} out = asyncio.run(cheap_verify.apply_gameplay_judge( s, game_id="g", brief="b", model=_FakeJudge(raise_exc=RuntimeError("不该被调用")), blocking=True)) assert out["accepted"] is False and out["ok"] is False assert out["judge"]["ran"] is False def test_apply_accept_blocking_ok_true(): """预筛过 + 判定 accept + blocking → ok=accepted=True。""" ev = _tmp_evidence() s = {"ok": True, "gameId": "g"} out = asyncio.run(cheap_verify.apply_gameplay_judge( s, game_id="g", brief="b", model=_FakeJudge(text=_accept_json()), blocking=True, evidence_dir=ev)) assert out["accepted"] is True and out["ok"] is True assert out["judge"]["ran"] is True and out["judge"]["verdict"] == "accept" def test_apply_reject_blocking_ok_false(): """预筛过 + 判定 reject + blocking → ok=accepted=False(阻断放行)。""" ev = _tmp_evidence() s = {"ok": True, "gameId": "g"} out = asyncio.run(cheap_verify.apply_gameplay_judge( s, game_id="g", brief="b", model=_FakeJudge(text=_reject_json("hollow")), blocking=True, evidence_dir=ev)) assert out["accepted"] is False and out["ok"] is False assert out["judge"]["rejectClasses"] == ["hollow"] def test_apply_degraded_blocking_failclosed(): """预筛过 + 证据缺失(空目录)→ 判定 degraded → fail-closed → ok=False + failureReason 可重试。""" ev = _tmp_evidence(with_first=False, with_after=False) # 空目录:无截图 s = {"ok": True, "gameId": "g"} out = asyncio.run(cheap_verify.apply_gameplay_judge( s, game_id="g", brief="b", model=_FakeJudge(text=_accept_json()), blocking=True, evidence_dir=ev)) assert out["accepted"] is False and out["ok"] is False assert out["judge"]["degraded"] is True assert out.get("failureReason") == "llm_error" # degraded → 可重试类 def test_apply_reject_nonblocking_observe(): """预筛过 + 判定 reject + blocking=false → ok=预筛(True,不阻断);accepted 仍记 False(观测/整体回退位)。""" ev = _tmp_evidence() s = {"ok": True, "gameId": "g"} out = asyncio.run(cheap_verify.apply_gameplay_judge( s, game_id="g", brief="b", model=_FakeJudge(text=_reject_json("broken")), blocking=False, evidence_dir=ev)) assert out["ok"] is True # 观测模式:不阻断,ok 维持预筛 assert out["accepted"] is False # 判定裁决仍如实记(供观测/校准) assert out["judge"]["blocking"] is False and out["judge"]["verdict"] == "reject" # ───────────────────────── _collect_frames 排序/采样 + 金标读取器 ───────────────────────── def test_collect_frames_order_and_cap(): """顺序 first-paint → midplay-1..N → after-play;超 max_frames 保头尾均匀采样。""" ev = _tmp_evidence(mids=8) # first + 8 mid + after = 10 张 uris = cheap_verify._collect_frames(ev, max_frames=4) assert len(uris) == 4 # 头 + 2 采样 + 尾 # 头是 first-paint、尾是 after-play(解 base64 尾核对内容标记) import base64 as _b64 head = _b64.b64decode(uris[0].split(",", 1)[1]) tail = _b64.b64decode(uris[-1].split(",", 1)[1]) assert head == b"\x89PNG-first" and tail == b"\x89PNG-after" def test_collect_frames_natural_sort(): """midplay-2 排在 midplay-10 之前(自然排序,非字典序)。""" ev = _tmp_evidence(mids=12) uris = cheap_verify._collect_frames(ev, max_frames=20) # 全收 assert len(uris) == 14 # first + 12 mid + after import base64 as _b64 # 第 2 张(index1)应是 midplay-1 assert _b64.b64decode(uris[1].split(",", 1)[1]) == b"\x89PNG-mid1" def test_load_golden_five_unlabeled_seeds(): """金标 manifest:五份 score-only 样本登记为待标注种子(expectReject 全 null,不预设反例)。""" g = judge_golden.load_golden() assert g["schemaVersion"] == "judge-golden/1" gids = [s["gid"] for s in g["samples"]] assert gids == ["hard-heritage", "hard-trpg", "hard-idle-sim", "c2v-1", "c2v-3"] assert all(s["expectReject"] is None for s in g["samples"]), "五份必须都未标注(双评审明令:不预设反例)" assert judge_golden.labeled_samples(g) == [], "无已标注金标(待创始人亲玩定标)" for s in g["samples"]: assert s["brief"] and s["evidenceDir"] # 清单齐全 # ───────────────────── ⑤ 空输出有界重试 + 判定真相层落盘(W-AXIS 波2 验收补,2026-07-09)───────────────────── # 背景:w2b-verify 真跑判定 degraded「判定输出为空」——glm-5.2 带思考,思考长度随机,吃光 max_tokens 时 # content 为空;主会话同证据复现即得答案 → 修=空输出重试一次(重试前放大 max_tokens)+ 判定落盘 evidence/。 class _FakeJudgeEmptyThenOk(_FakeJudge): """第一次返回空(模拟思考吃光 max_tokens),第二次返回合法 JSON——测有界重试恢复。""" def __init__(self, ok_text): super().__init__(text=ok_text) self.calls = 0 self.max_tokens = 1500 async def __call__(self, messages, **kw): self.calls += 1 if self.calls == 1: return _FakeResp("") return _FakeResp(self._text) def test_judge_empty_retry_once_recovers(): m = _FakeJudgeEmptyThenOk(_accept_json()) r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", frames=["data:image/png;base64,AAAA"], model=m)) assert m.calls == 2, "空输出应重试一次(共两次调用)" assert r["accepted"] is True and r["degraded"] is False assert r.get("retries") == 1 assert m.max_tokens == 3000, "重试前应放大 max_tokens(思考吃光预算的对症解)" def test_judge_empty_twice_degraded(): r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", frames=["data:image/png;base64,AAAA"], model=_FakeJudge(text=""))) assert r["degraded"] is True and r["accepted"] is False assert "判定输出为空" in r["reason"] def test_judge_persists_judge_json(): ev = _tmp_evidence() r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", evidence_dir=ev, model=_FakeJudge(text=_accept_json()))) assert r["accepted"] is True jp = Path(ev) / "judge.json" assert jp.is_file(), "判定必须在证据目录落 judge.json(判定器自身真相层)" data = json.loads(jp.read_text(encoding="utf-8")) assert data["accepted"] is True and data["verdict"] == "accept" assert "rawTextHead" in data and data["ts"] > 0 def test_apply_explicit_prefilter_overrides_summary_ok(): """显式 prefilter 参数必须压过 summary.ok(CLI 路修复,2026-07-09 W3 真跑暴露): summary.ok=True(finished)但九门挂 → prefilter=False → 判定不跑、accepted=False。""" s = {"ok": True} out = asyncio.run(cheap_verify.apply_gameplay_judge( s, game_id="g", brief="b", model=_FakeJudge(raise_exc=RuntimeError("预筛未过不该调判定")), blocking=True, prefilter=False)) assert out["accepted"] is False and out["ok"] is False assert out["judge"]["ran"] is False and "预筛" in out["judge"]["reason"] def test_apply_explicit_prefilter_true_runs_judge(): """prefilter=True 显式给真(即便 summary.ok=False 的殊形)→ 判定照跑,ok=判定结果。""" s = {"ok": False} out = asyncio.run(cheap_verify.apply_gameplay_judge( s, game_id="g", brief="b", model=_FakeJudge(text=_accept_json()), blocking=True, prefilter=True, evidence_dir=_tmp_evidence())) assert out["accepted"] is True and out["ok"] is True and out["judge"]["ran"] is True def test_apply_writes_back_verdict_json(): ev = _tmp_evidence() (Path(ev) / "verdict.json").write_text( json.dumps({"pass": True, "guards": {}}), encoding="utf-8") s = {"ok": True} out = asyncio.run(cheap_verify.apply_gameplay_judge( s, game_id="g", brief="b", model=_FakeJudge(text=_reject_json("hollow")), blocking=True, evidence_dir=ev)) assert out["accepted"] is False and out["ok"] is False v = json.loads((Path(ev) / "verdict.json").read_text(encoding="utf-8")) assert v["pass"] is True, "pass 物理保留(语义=预筛通过)" assert v["accepted"] is False, "accepted=最终权威(判定拒)" assert v["judge"]["verdict"] == "reject" and v["judge"]["rejectClasses"] == ["hollow"] # ───────────────────── ⑥ 图像盲微扰重掷(n=5 基线终审补,2026-07-09)───────────────────── # 背景:网关对 glm-5.2 按请求体确定性路由,部分载荷永远落在丢图通道——判定 JSON 合法却自报看不见截图, # 把好游戏错杀成 hollow/off_brief(基线 3/20 假阴)。同载荷复判 3/3 仍盲;brief 尾加一空格微扰即换路由当场看见。 # 修:输出契约加 imagesSeen 自报字段;送帧而自报 0 → 微扰重掷一次;两掷均盲 → 仪器故障 degraded。 def _accept_json_seen(n): """带 imagesSeen 自报的 accept JSON(n=模型自称看到的截图张数)。""" d = json.loads(_accept_json()) d["imagesSeen"] = n return json.dumps(d, ensure_ascii=False) class _FakeJudgeBlindThenSeen(_FakeJudge): """第一掷自报 imagesSeen=0(模拟丢图通道),第二掷自报 6——测微扰重掷恢复;记每掷收到的 brief 头文本。""" def __init__(self): super().__init__() self.calls = 0 self.brief_heads = [] async def __call__(self, messages, **kw): self.calls += 1 # 记录 user 消息首个 text 块的头部(含 brief),供断言两掷载荷确实不同(微扰生效) parts = messages[-1]["content"] self.brief_heads.append(parts[0]["text"][:80]) return _FakeResp(_accept_json_seen(0 if self.calls == 1 else 6)) def test_parse_images_seen_passthrough(): assert cheap_verify.parse_floor_judgment(_accept_json_seen(6))["imagesSeen"] == 6 assert cheap_verify.parse_floor_judgment(_accept_json_seen(0))["imagesSeen"] == 0 assert cheap_verify.parse_floor_judgment(_accept_json())["imagesSeen"] is None, "缺字段=未知,不触发盲重掷" bad = json.loads(_accept_json()); bad["imagesSeen"] = "abc" assert cheap_verify.parse_floor_judgment(json.dumps(bad, ensure_ascii=False))["imagesSeen"] is None bad["imagesSeen"] = True # bool 不算数字(True==1 的坑),按未知处理 assert cheap_verify.parse_floor_judgment(json.dumps(bad, ensure_ascii=False))["imagesSeen"] is None def test_judge_image_blind_retry_recovers(): m = _FakeJudgeBlindThenSeen() r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", brief="掷骰爬塔", frames=["data:image/png;base64,AAAA"], model=m)) assert m.calls == 2, "自报 0 张应微扰重掷一次" assert m.brief_heads[0] != m.brief_heads[1], "第二掷载荷必须与第一掷不同(微扰换网关路由)" assert r["accepted"] is True and r["degraded"] is False assert r.get("imageBlindRetried") is True and r.get("imagesSeen") == 6 assert r.get("retries") == 1 def test_judge_image_blind_twice_degraded(): m = _FakeJudge(text=_accept_json_seen(0)) # 两掷都自报看不见 r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", brief="掷骰爬塔", frames=["data:image/png;base64,AAAA"], model=m)) assert r["degraded"] is True and r["accepted"] is False assert r["rejectClasses"] == ["degraded"] assert "看不见截图" in r["reason"], "归因必须是判定仪器故障,不是游戏证据缺失" assert r.get("imageBlindRetried") is True def test_judge_images_seen_positive_no_extra_call(): m = _FakeJudgeBlindThenSeen() m.calls = 1 # 让首掷直接返回 seen=6(复用 fake:calls 从 2 起) r = asyncio.run(cheap_verify.judge_gameplay_floor( "x", frames=["data:image/png;base64,AAAA"], model=m)) assert m.calls == 2 and len(m.brief_heads) == 1, "自报看得见 → 不重掷" assert r["accepted"] is True and r.get("imageBlindRetried") is None if __name__ == "__main__": _fns = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] _failed = 0 for _fn in _fns: try: _fn() print(f" PASS {_fn.__name__}") except AssertionError as e: _failed += 1 print(f" FAIL {_fn.__name__}: {e}") except Exception as e: # noqa: BLE001 _failed += 1 print(f" ERROR {_fn.__name__}: {type(e).__name__}: {e}") print(f"\n{len(_fns) - _failed}/{len(_fns)} passed") sys.exit(1 if _failed else 0)