feat(cheap-gen): W-AXIS-V2 波1——测试 agent 生产化+三路统一验收编排器+字段迁移
opus 执行位交付、主会话四步验收过(测试亲跑/红线 diff 亲读/硬证亲眼): - playtest.cdp.cjs 生产版(§3 八件:DPR=1 playtest/1 坐标协议/坐标尺/落点回显/ 同点硬提示/反早退/预算 14→24 随进展扩/fail 二掷显式 seed+清存档/图像自检 fail-closed exit=3)+ serve-and-playtest.sh;证据落 evidence/playtest/roll-N/ - cheap_verify:project_floor 四门投影(预筛权威唯一产地,绝不复用 harness 原生 verdict.pass)+ run_playtest(子进程/超时/记账/playtest.json 真相层/端口派生避 Chrome unsafe 5060/5061)+ run_acceptance 编排器(v2=floor∧测试员阻断/ shadow=旧口径+shadowV2Accepted 对照/v1=旧判定器;修复反馈只引现象段) - 三路接线:cheap_studio/cheap_service_driver/cheap_modify 两档全改调编排器, create 与 modify 单一验收标准 - §4 字段迁移:run-summary acceptanceVersion+playtest+floor 段、trace.playtest additive(firstPlay 三字段镜像 ReadinessScorer 口径)、批账 floorPass+accepted+ acceptanceVersion(verdictPass 仅对照);result_out 三级取值(floor→verdictFull 投影→verdict.pass 回落)、gate_judge 续修触发器切四门、xtheme 同批;判定语义单轨 - §5 Java 读侧:ReadinessScorer 可玩性/firstPlay 优先读 trace.playtest, GenMetrics 归因加 playtest/tester_degraded(向后兼容,未编译验证——本机无后端 构建环境,待 mini-desktop 构建窗口) - genconfig/generation.yaml:acceptance.mode 三态(波1 灰度默认 shadow)+ playtest 旋钮(steps/二掷/超时/成本上限);judge 段未动 - 单测 32 项新增(二掷/图像 fail-closed/mode 三态/投影/现象分离/退出码/unsafe 端口) 验收:pytest 512 绿 + node 44/44(主会话亲跑);10 局考卷生产件重跑判对 9/9 (真坏 3/3 零放行、假阴 0/6,heritage-r4/puzzle-r1 二掷翻案、sb2 假超时根因= Chrome unsafe 端口已修);Service E2E(w1v2-e2e1 shadow→v2 一发收敛)与 modify 回归(w1v2-mod1 ROUND_MS 60000→45000,src 与局内截图双证)真浏览器硬证; 总成本 ¥2.3。已知余项:真续修回喂环仅单测覆盖(波2 n=3 冒烟自然覆盖)。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
lili
parent
8edb0ffa39
commit
5bcaf6ac1e
@ -29,6 +29,19 @@ from pathlib import Path
|
||||
|
||||
import cheap_assert
|
||||
import cheap_run
|
||||
import cheap_verify # W-AXIS-V2 波1:modify 两档改调统一验收编排器(create/modify 同标准)
|
||||
|
||||
|
||||
def _read_game_brief(game_id: str) -> str:
|
||||
"""读原游戏 brief(evidence/brief.json;modify 验收给测试员上下文——改后仍应是同一款可玩游戏)。缺/坏 → 空串。"""
|
||||
try:
|
||||
p = cheap_run.game_dir(game_id) / "evidence" / "brief.json"
|
||||
if p.is_file():
|
||||
d = json.loads(p.read_text(encoding="utf-8"))
|
||||
return (d.get("brief") if isinstance(d, dict) else "") or ""
|
||||
except Exception: # noqa: BLE001 brief 读失败不致命,测试员用空 brief 仍可判 broken/hollow
|
||||
pass
|
||||
return ""
|
||||
|
||||
# A11 M4 模块重生成：behavior 在便宜档映射到玩法文件（KTD4）——只重写它，其余文件保持不变。
|
||||
_BEHAVIOR_TARGET = "game-logic.js"
|
||||
@ -326,7 +339,13 @@ def execute_deterministic_modify(game_id: str, source_project, modify_patch,
|
||||
cheap_run.ensure_play_spec(game_id, sm.get("state")) # 已存在不覆盖;缺则据 state 形态补 driver(防裸跑假绿)
|
||||
pr = cheap_run.play(game_id, port=port, cdp_port=cdp_port)
|
||||
verdict = pr.get("verdict")
|
||||
gates_pass = isinstance(verdict, dict) and verdict.get("pass") is True
|
||||
# W-AXIS-V2 波1:create 与 modify 不允许两套验收标准——确定性改也过统一编排器(四门投影 ∧ 测试员真玩)。
|
||||
# modify 同步执行(worker_service 懒绑),用 asyncio.run 跑 async 编排器(与 _default_rewrite 同款);
|
||||
# brief 取原游戏 brief(改数值行为不变,验证「改后仍是同一款可玩游戏」);测试员端口自动派生(与九门 4320 分段)。
|
||||
import asyncio # noqa: PLC0415
|
||||
_acc = asyncio.run(cheap_verify.run_acceptance(
|
||||
{"verdict": verdict}, game_id=game_id, brief=_read_game_brief(game_id), verdict=verdict))
|
||||
gates_pass = bool(_acc.get("accepted"))
|
||||
|
||||
# M5 三结构断言:base→改后 真比对(断言①改动真生效 / 断言②非目标稳)+ 血缘(③ baseVersionId)。
|
||||
# 断言②对 deterministic 是兜 apply bug(理应只改目标文件,真比对抓"是否顺手碰了非目标")。
|
||||
@ -337,11 +356,14 @@ def execute_deterministic_modify(game_id: str, source_project, modify_patch,
|
||||
status = "succeeded" if (gates_pass and a12_pass) else "failed"
|
||||
err = None
|
||||
if status != "succeeded":
|
||||
err = "九门未通过" if not gates_pass else "三断言未通过:" + json.dumps(
|
||||
err = "验收未通过(v2:四门∧测试员)" if not gates_pass else "三断言未通过:" + json.dumps(
|
||||
{k: v for k, v in assertions.items() if isinstance(v, dict) and not v.get("pass")}, ensure_ascii=False)
|
||||
log(f"确定性改重建+九门+三断言完成 game_id={game_id} status={status} gates={gates_pass} a12={a12_pass}")
|
||||
log(f"确定性改重建+v2验收+三断言完成 game_id={game_id} status={status} gates={gates_pass} a12={a12_pass} "
|
||||
f"mode={_acc.get('acceptanceVersion')}")
|
||||
return {"status": status, "manifest": manifest, "verdict": verdict, "assertions": assertions,
|
||||
"stage": "play", "error": err, "smokeOk": sm.get("ok"), "playOk": pr.get("ok")}
|
||||
"stage": "play", "error": err, "smokeOk": sm.get("ok"), "playOk": pr.get("ok"),
|
||||
# v2 验收摘要 additive(供 worker_service 组 run-summary / trace):
|
||||
"acceptance": {k: _acc.get(k) for k in ("acceptanceVersion", "floor", "playtest", "judge", "accepted")}}
|
||||
|
||||
|
||||
# ───────────────────────── ④ execute_regenerate_modify（模块重生成 · 改玩法）─────────────────────────
|
||||
@ -504,7 +526,10 @@ def execute_regenerate_modify(game_id: str, source_project, modify_patch,
|
||||
# （误伤非目标=边界泄漏、目标没变=no-op，即便九门过也判失败）。
|
||||
verdict_full = run_summary.get("verdictFull") if isinstance(run_summary.get("verdictFull"), dict) else None
|
||||
vb = run_summary.get("verdict") if isinstance(run_summary.get("verdict"), dict) else None
|
||||
gates_pass = bool(vb and vb.get("pass") is True)
|
||||
# W-AXIS-V2 波1:_default_rewrite 走 run_studio(run_gates=True),收口已跑统一验收编排器,验收权威 = accepted
|
||||
# (create/modify 同标准);run_summary 缺 accepted(注入桩/老路)→ 回落四门投影,不回落九门 verdict.pass(退役口径)。
|
||||
_acc = run_summary.get("accepted")
|
||||
gates_pass = bool(_acc) if _acc is not None else cheap_verify.project_floor(verdict_full)["pass"]
|
||||
# 三断言 verdict（统一形态，与 deterministic 路一致）：①changed ②非目标稳（含误伤清单）③血缘。
|
||||
nontarget_changed = sorted(k for k in set(nontarget_before) | set(nontarget_after)
|
||||
if nontarget_before.get(k) != nontarget_after.get(k))
|
||||
|
||||
@ -412,16 +412,18 @@ async def drive_cheap_generation(job: dict, *, base_url: str | None = None, user
|
||||
except Exception as e: # noqa: BLE001 —— richness 非阻塞铁律:绝不影响回调
|
||||
summary["richness"] = {"score": None, "degraded": True, "reason": f"richness 接线异常:{type(e).__name__}: {e}"}
|
||||
|
||||
# ⑪ 玩法地板判定 + ok 语义切换(W-AXIS 波2 · 质量模型 SoT 裁定三):机械九门(_build_summary 的 j.passed)
|
||||
# 只作「未见明显死」预筛,过筛者交独立多模态档看真玩证据(首帧/局中连拍/局末截图 + game-log + 取证时间线)
|
||||
# 裁 broken/hollow/off-brief,阻断放行(ok = 预筛 ∧ 判定)。与丰富度(非阻塞观测)是两物:fail-closed,
|
||||
# 评不出/无证据 → 不放行、标 degraded。apply_gameplay_judge 内建 fail-closed 与顶层兜底、绝不抛,
|
||||
# additive 写 summary['accepted']/['judge'] 并更新 ['ok']——result_out 据 accepted 落 status。
|
||||
summary = await cheap_verify.apply_gameplay_judge(summary, game_id=game_id, brief=brief)
|
||||
# ⑪ 统一验收编排器(W-AXIS-V2 波1 · 拆着杀):四门投影(A/B/C/D)作预筛权威,过筛者交测试 agent 视觉引导
|
||||
# 真玩裁 broken/hollow/off-brief,阻断放行(accepted = floor ∧ 测试员;acceptance.mode 三态 v2/shadow/v1)。
|
||||
# verdict 传入做 floor 投影;端口缺省按 game_id 派生(driver 收口可能并发,测试员起服避撞)。
|
||||
# run_acceptance 内建 fail-closed 与顶层兜底、绝不抛,additive 写 floor/playtest/judge/acceptanceVersion,
|
||||
# 更新 ok/accepted——result_out 据 accepted 落 status。
|
||||
summary = await cheap_verify.run_acceptance(summary, game_id=game_id, brief=brief, verdict=verdict)
|
||||
_js = summary.get("judge") or {}
|
||||
print(f"[cheap-driver] game={game_id} 玩法地板判定 ran={_js.get('ran')} blocking={_js.get('blocking')} "
|
||||
f"verdict={_js.get('verdict')} rejectClasses={_js.get('rejectClasses')} degraded={_js.get('degraded')} "
|
||||
f"costRmb={_js.get('costRmb')} → accepted={summary.get('accepted')}", flush=True)
|
||||
_pt = summary.get("playtest") or {}
|
||||
print(f"[cheap-driver] game={game_id} 验收 v2 mode={summary.get('acceptanceVersion')} "
|
||||
f"floor={(summary.get('floor') or {}).get('pass')} playtest.accepted={_pt.get('accepted')} "
|
||||
f"rolls={_pt.get('rollCount')} degraded={_pt.get('degraded')} costRmb={_pt.get('costRmb')} "
|
||||
f"→ accepted={summary.get('accepted')} ok={summary.get('ok')}", flush=True)
|
||||
|
||||
print(f"[cheap-driver] game={game_id} 单 POST 结束: ok={summary['ok']} attempts={summary['attempts']} "
|
||||
f"costRmb={summary['costRmb']} wallSec={summary['wallSec']}", flush=True)
|
||||
|
||||
@ -463,22 +463,19 @@ async def run_studio(game_id, brief, *, max_iters=40, max_resumes=6, max_tokens=
|
||||
"reason": f"richness 接线异常：{type(e).__name__}: {e}"}
|
||||
_rec(f"丰富度评分接线异常（已降级，不影响 run）：{type(e).__name__}: {e}")
|
||||
|
||||
# ── W-AXIS 波2:独立模型玩法地板判定 + ok 语义切换(ok = 预筛 ∧ 判定;质量模型 SoT 裁定三)──
|
||||
# 机械九门只作「未见明显死」预筛,过筛者交独立多模态档看真玩证据裁 broken/hollow/off-brief,阻断放行。
|
||||
# 与丰富度(非阻塞观测)是两物:这里 fail-closed,评不出/无证据 → 不放行、标 degraded。
|
||||
# 只在 run_gates(有真 verdict + 首帧/局中/局末截图证据)时接入;对照路(run_gates=False)零改动、不触发判定。
|
||||
# apply_gameplay_judge 内建 fail-closed 与顶层兜底、绝不抛,additive 写 summary['accepted']/['judge'] 并更新 ['ok']。
|
||||
# ── W-AXIS-V2 波1:统一验收编排器(floor 四门投影 ∧ 测试 agent 真玩;acceptance.mode 三态)──
|
||||
# 拆着杀:契约无关四门(A/B/C/D 投影)= 预筛权威(取代旧 verdict.pass 九门口径,那随契约退役会坍缩),
|
||||
# 过筛者交测试 agent 视觉引导真玩裁 broken/hollow/off-brief;mode=v2 阻断、shadow 灰度对照、v1 旧口径。
|
||||
# run_acceptance 内建 fail-closed 与顶层兜底、绝不抛,additive 写 summary['floor']/['playtest']/['judge']
|
||||
# /['acceptanceVersion'] 并更新 ['ok']/['accepted']。只在 run_gates(有真 verdict)时接入;对照路零改动。
|
||||
if run_gates and finished:
|
||||
# 预筛必须显式喂九门口径(裁定一/三):本路 summary.ok=finished 是「模型自称收敛+check/build 绿」,
|
||||
# 不是机械九门信号——2026-07-09 W3 真跑(w3-verify)暴露:E/G/H 三门挂的局按 finished 当预筛,
|
||||
# 被判定放行成 accepted=True,违 SoT『ok=预筛∧判定』,已修。verdict 缺/坏 → 预筛按未过(fail-closed)。
|
||||
_prefilter = bool((verdict or {}).get("pass"))
|
||||
summary = await cheap_verify.apply_gameplay_judge(summary, game_id=game_id, brief=brief,
|
||||
prefilter=_prefilter)
|
||||
summary = await cheap_verify.run_acceptance(summary, game_id=game_id, brief=brief, verdict=verdict)
|
||||
_js = summary.get("judge") or {}
|
||||
_rec(f"玩法地板判定 ran={_js.get('ran')} blocking={_js.get('blocking')} verdict={_js.get('verdict')} "
|
||||
f"rejectClasses={_js.get('rejectClasses')} degraded={_js.get('degraded')} "
|
||||
f"costRmb={_js.get('costRmb')} → accepted={summary.get('accepted')} ok={summary.get('ok')}")
|
||||
_pt = summary.get("playtest") or {}
|
||||
_rec(f"验收 v2 mode={summary.get('acceptanceVersion')} floor={(summary.get('floor') or {}).get('pass')} "
|
||||
f"playtest.accepted={_pt.get('accepted')} rolls={_pt.get('rollCount')} degraded={_pt.get('degraded')} "
|
||||
f"costRmb={_pt.get('costRmb')} judge.verdict={_js.get('verdict')} "
|
||||
f"→ accepted={summary.get('accepted')} ok={summary.get('ok')}")
|
||||
|
||||
ev_dir = cheap_run.game_dir(game_id) / "evidence"
|
||||
ev_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@ -19,6 +19,7 @@ cheap_verify.py — 便宜档「丰富度」LLM 验证 agent（非阻塞·只报
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
@ -956,6 +957,381 @@ async def apply_gameplay_judge(summary: dict, *, game_id: str, brief: str = "",
|
||||
return summary
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# 验收 v2:四门投影 + 测试 agent 真玩 + 统一编排器(W-AXIS-V2 波1 · 「拆着杀」)
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# W-AXIS 三波把验收权威交给独立模型判定,但证据仍由「一份固定代码玩几十种游戏」的取证契约(play-spec /
|
||||
# tap-targets 驱动器 / _forensicsView)生产——n=5 基线逐局破案坐实这条证据生产线本身就是最大失败源。
|
||||
# 本段兑现创始人 2026-07-09「拆着杀」:契约无关的四门(A/B/C/D)降为地板保留,依赖契约的 E/G/H/I/F 与
|
||||
# 驱动器退役,验收证据的生产与裁决交测试 agent(playtest.cdp.cjs:视觉引导真玩 + 玩法地板判定合一)。
|
||||
# 三路(CLI/Service/modify)共用一个编排器 run_acceptance;acceptance.mode 三态 v2/shadow/v1 控阻断与灰度。
|
||||
|
||||
# 四门 = 契约无关:读装载标志 / 未捕获异常 / 引擎帧号 / canvas 像素,不需游戏配合任何自报接口,
|
||||
# 零成本、确定性、抓真死(基线里 boot 空指针正是 A 门拦的)。F_wiring 的期望前缀集源自 play-spec、
|
||||
# 随契约退役降观测,不入地板(裁定三 2026-07-10 修订)。
|
||||
_FLOOR_GATES = ("A_boot", "B_uncaught", "C_frame", "D_render")
|
||||
|
||||
|
||||
def project_floor(verdict) -> dict:
|
||||
"""四门投影(floor 段的**唯一产地**):从 verdict.guards 显式抽取 A_boot/B_uncaught/C_frame/D_render 求 AND。
|
||||
|
||||
这是消费层函数,**绝不复用 harness 原生 verdict.pass**——那是九门(driven)/七门(undriven)AND、随驱动器语义,
|
||||
拆契约后会静默坍缩;四门投影独立于它,是便宜档预筛的权威取值(裁定一 2026-07-10 修订)。
|
||||
任一门缺失(guards 无该门 dict)或 pass 非 True → floor.pass=False(fail-closed:无四门证据不放行)。
|
||||
返回 {pass: bool, gates: {A,B,C,D: bool}}(短键 A/B/C/D = plan §4 字段协议)。
|
||||
"""
|
||||
guards = (verdict or {}).get("guards") or {}
|
||||
gates = {}
|
||||
all_ok = True
|
||||
for full in _FLOOR_GATES:
|
||||
short = full.split("_")[0] # A_boot → A
|
||||
node = guards.get(full)
|
||||
ok = isinstance(node, dict) and node.get("pass") is True
|
||||
gates[short] = ok
|
||||
all_ok = all_ok and ok
|
||||
return {"pass": all_ok, "gates": gates}
|
||||
|
||||
|
||||
def _acceptance_cfg() -> dict:
|
||||
"""读验收 v2 配置(genconfig acceptance.* + playtest.* + judge.*;worker 不可达回落编译默认,保单测)。"""
|
||||
try:
|
||||
from worker import genconfig # noqa: PLC0415
|
||||
g = genconfig.get
|
||||
except Exception: # noqa: BLE001 无 worker(纯单测)→ 编译默认
|
||||
def g(_area, _key, default):
|
||||
return default
|
||||
return {
|
||||
"mode": str(g("acceptance", "mode", "shadow")), # v2|shadow|v1(波1 灰度窗口默认 shadow)
|
||||
"model": g("judge", "model", _JUDGE_DEFAULT_MODEL), # 测试员 = 判定档同一配置(§2;现 MiniMax-M3)
|
||||
"steps_base": int(g("playtest", "steps_base", 14)), # 基础步数预算(§3 第6件)
|
||||
"steps_max": int(g("playtest", "steps_max", 24)), # 有进展扩展上限
|
||||
"second_roll": bool(g("playtest", "second_roll", True)),# fail 二掷确认(§3 第7件)
|
||||
"timeout_s": float(g("playtest", "timeout_s", 600.0)), # 单掷子进程墙钟超时
|
||||
"cost_cap_rmb": float(g("playtest", "cost_cap_rmb", 1.5)), # 单局验收成本上限(含二掷)
|
||||
"blocking": bool(g("judge", "blocking", True)), # 测试员裁决是否阻断放行
|
||||
}
|
||||
|
||||
|
||||
def _playtest_script() -> Path:
|
||||
"""serve-and-playtest.sh 路径(与 serve-and-play.sh 同套起服编排,末步换调 playtest.cdp.cjs)。"""
|
||||
return cheap_run._WG1_DIR / "_shared" / "serve-and-playtest.sh"
|
||||
|
||||
|
||||
def _playtest_env() -> dict:
|
||||
"""subprocess env:复用 cheap_run 起服 env(CHROME_BIN + NO_PROXY)+ 注入 NEWAPI_KEY/BASE(测试员连 new-api)。
|
||||
|
||||
playtest.cdp.cjs 用 Node http 模块直连 new-api、天然绕系统代理;key/base 由本函数从凭据档解析后注入 env。
|
||||
"""
|
||||
env = dict(cheap_run._shell_env())
|
||||
try:
|
||||
import _bootstrap # noqa: PLC0415 代理旁路时序与 key 注入
|
||||
_bootstrap.ensure_api_key_env()
|
||||
from worker import client # noqa: PLC0415
|
||||
env["NEWAPI_KEY"] = client.get_api_key()
|
||||
env["NEWAPI_BASE_URL"] = client.resolve_base_url()
|
||||
except Exception: # noqa: BLE001 凭据解析失败 → 交 cjs 自身的 env 兜底(仍可能从进程环境拿到)
|
||||
pass
|
||||
return env
|
||||
|
||||
|
||||
# Chrome unsafe ports(net::ERR_UNSAFE_PORT):派生范围 4998..5097 内被 Chrome 封锁的页面加载端口。
|
||||
# 2026-07-10 考卷真跑实锤:hard-sim-business-r2 派生到 5060(SIP),curl 就绪检查 200 而 Chrome navigate
|
||||
# 直接拒(NAT slipstreaming 防护),__genBooted 永不置位 → 假 boot-timeout。派生必须跳过。
|
||||
_CHROME_UNSAFE_PORTS = {5060, 5061}
|
||||
|
||||
|
||||
def _derive_playtest_ports(game_id) -> tuple:
|
||||
"""按 game_id 确定性派生测试员起服端口(避并发多局撞:Service 异步队列 / modify / 批跑可能同时收口)。
|
||||
|
||||
基址 4998/9331(spike 同段,与九门 play 的 4320/9222 分离)+ game_id 稳定 hash % 100(md5,禁 Python
|
||||
随机化 hash)。同一 game_id 恒得同端口(可复现);不同 game_id 大概率不撞(并发 ≤15 远低于 100 槽)。
|
||||
静态服务端口撞 Chrome unsafe 清单(5060/5061=SIP,页面加载被 ERR_UNSAFE_PORT 拒)→ +2 避开
|
||||
(CDP 端口无此限制——unsafe 清单只管页面加载,不管 Chrome 自身监听)。
|
||||
"""
|
||||
import hashlib # noqa: PLC0415
|
||||
h = int(hashlib.md5(str(game_id).encode("utf-8")).hexdigest()[:4], 16) % 100
|
||||
port = 4998 + h
|
||||
if port in _CHROME_UNSAFE_PORTS:
|
||||
port += 2 # 5060→5062 / 5061→5063(5062/5063 不在清单;仍确定性可复现)
|
||||
return port, 9331 + h
|
||||
|
||||
|
||||
def _run_playtest_roll_sync(game_id, brief_file, *, model_name, roll, seed, steps_base, steps_max,
|
||||
evidence_dir, port, cdp_port, timeout) -> dict:
|
||||
"""同步跑一掷(subprocess serve-and-playtest.sh → playtest.cdp.cjs);解析 stdout 末行 JSON 裁决。
|
||||
|
||||
绝不抛——超时/异常/未产裁决一律归 degraded 裁决(fail-closed)。退出码语义(playtest/1 协议):
|
||||
0=正常裁决、2=运行错误/装载失败(degraded)、3=图像通道故障(degraded,归因测试员图像通道非游戏缺陷)。
|
||||
"""
|
||||
argv = ["bash", str(_playtest_script()), str(game_id), str(port), str(cdp_port), "--",
|
||||
f"--model={model_name}", f"--roll={roll}", f"--seed={seed}",
|
||||
f"--steps-base={steps_base}", f"--steps-max={steps_max}",
|
||||
f"--brief-file={brief_file}", f"--evidence-dir={evidence_dir}"]
|
||||
try:
|
||||
r = subprocess.run(argv, cwd=str(cheap_run._GAME_RUNTIME), capture_output=True, text=True,
|
||||
timeout=timeout, env=_playtest_env(), check=False)
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"degraded": True, "reason": f"测试员子进程超时(>{timeout}s),fail-closed", "exitCode": -1, "roll": roll}
|
||||
except Exception as e: # noqa: BLE001 subprocess 层异常一律 fail-closed
|
||||
return {"degraded": True, "reason": f"测试员子进程异常:{type(e).__name__}: {e}", "exitCode": -1, "roll": roll}
|
||||
# 解析 stdout 末行单行 JSON(playtest.cdp.cjs 末行 = 裁决)。
|
||||
out = None
|
||||
for line in reversed((r.stdout or "").splitlines()):
|
||||
line = line.strip()
|
||||
if line.startswith("{") and line.endswith("}"):
|
||||
try:
|
||||
out = json.loads(line)
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if out is None:
|
||||
return {"degraded": True, "reason": "测试员未产出裁决 JSON(见 raw)", "exitCode": r.returncode,
|
||||
"raw": (r.stdout + r.stderr)[-1000:], "roll": roll}
|
||||
out["exitCode"] = r.returncode
|
||||
# 退出码/自报标志 → degraded(fail-closed):运行错误、装载失败、图像通道故障都不放行。
|
||||
if r.returncode == 2 or out.get("runnerError") or out.get("bootDead"):
|
||||
out["degraded"] = True
|
||||
out.setdefault("reason", "测试员运行错误/装载失败,fail-closed")
|
||||
if r.returncode == 3 or out.get("imageBlind"):
|
||||
out["degraded"] = True
|
||||
out["imageBlind"] = True
|
||||
out.setdefault("reason", "测试员图像通道故障(自报看不到截图),fail-closed 归因测试员非游戏")
|
||||
return out
|
||||
|
||||
|
||||
def _roll_brief(roll) -> dict:
|
||||
"""把单掷裁决压成轻量摘要(落 playtest.json 的 rolls 列表 / 批次账,不塞完整转写)。"""
|
||||
return {"roll": roll.get("roll"), "seed": roll.get("seed"), "pass": roll.get("pass"),
|
||||
"degraded": bool(roll.get("degraded")), "canSee": roll.get("canSee"),
|
||||
"steps": roll.get("steps"), "tapPoints": roll.get("tapPoints"),
|
||||
"summary": roll.get("summary"), "reason": roll.get("reason"),
|
||||
"tokens": roll.get("tokens")}
|
||||
|
||||
|
||||
def _playtest_roll_cost(model_name, roll) -> float:
|
||||
"""据 new-api 权威倍率把一掷 token 折成 ¥(与判定成本同口径);取价失败 → 0.0(不伪造成本、不误停)。"""
|
||||
tok = roll.get("tokens") or {}
|
||||
ti, to = int(tok.get("in") or 0), int(tok.get("out") or 0)
|
||||
c = _estimate_judge_cost(model_name, ti, to, 0)
|
||||
return float(c) if isinstance(c, (int, float)) else 0.0
|
||||
|
||||
|
||||
def _persist_playtest_json(evidence_dir, result: dict) -> None:
|
||||
"""测试员验收真相层落盘(best-effort,绝不抛):evidence/playtest/playtest.json = 汇总裁决 + 二掷账。
|
||||
|
||||
与 judge.json 同一动机(W-AXIS 波2):验收结论必须在盘上留痕,§6.13 亲眼验收看的就是它 + roll-N 截图序列。
|
||||
"""
|
||||
try:
|
||||
ev = Path(evidence_dir) if evidence_dir else None
|
||||
if ev is None:
|
||||
return
|
||||
pdir = ev / "playtest"
|
||||
pdir.mkdir(parents=True, exist_ok=True)
|
||||
payload = dict(result)
|
||||
payload["ts"] = int(time.time() * 1000)
|
||||
(pdir / "playtest.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
except Exception: # noqa: BLE001 落盘失败绝不连累验收主链
|
||||
pass
|
||||
|
||||
|
||||
async def run_playtest(game_id, *, brief="", model_name=None, evidence_dir=None,
|
||||
port=None, cdp_port=None, steps_base=None, steps_max=None,
|
||||
second_roll=None, timeout=None, cost_cap_rmb=None, seed=None) -> dict:
|
||||
"""测试 agent 真玩验收(阻断权威;裁定三 v2)。**fail-closed**:超时/图像故障/未裁决 → degraded=拒绝。
|
||||
|
||||
二掷(§3 第7件):roll-1 判 fail 且非 degraded、second_roll 开、成本未撞上限 → roll-2(不同 seed、清
|
||||
localStorage 独立);两掷同 fail 才落 fail,roll-2 pass 即翻案(治 puzzle 类坐标失手假阴)。pass 一掷即过。
|
||||
|
||||
Returns:
|
||||
playtest 段 dict:{accepted, verdict, pass, degraded, reason, problems, feedback, summary,
|
||||
firstPlay, rolls, rollCount, costRmb, model, imageBlind}。落 evidence/playtest/playtest.json。
|
||||
"""
|
||||
cfg = _acceptance_cfg()
|
||||
mn = model_name or cfg["model"]
|
||||
sb = steps_base if steps_base is not None else cfg["steps_base"]
|
||||
smax = steps_max if steps_max is not None else cfg["steps_max"]
|
||||
sr = cfg["second_roll"] if second_roll is None else bool(second_roll)
|
||||
to = timeout if timeout is not None else cfg["timeout_s"]
|
||||
cap = cost_cap_rmb if cost_cap_rmb is not None else cfg["cost_cap_rmb"]
|
||||
ev = Path(evidence_dir) if evidence_dir is not None else _wg1_evidence_dir(game_id)
|
||||
base_seed = seed if seed is not None else int(time.time() * 1000) % 1_000_000
|
||||
if port is None or cdp_port is None: # 缺省 → 按 game_id 派生 per-game 端口(并发避撞)
|
||||
dp, dc = _derive_playtest_ports(game_id)
|
||||
port = dp if port is None else port
|
||||
cdp_port = dc if cdp_port is None else cdp_port
|
||||
_NULL_FIRSTPLAY = {"playableAtMs": None, "firstFeedbackMs": None, "loopClosed": False}
|
||||
|
||||
result = {"model": mn, "rolls": [], "degraded": False}
|
||||
brief_file = None
|
||||
try:
|
||||
import tempfile # noqa: PLC0415 brief 写临时文件(诚实来源,避免命令行转义/长度)
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".brieftxt", delete=False, encoding="utf-8") as bf:
|
||||
bf.write(brief or "")
|
||||
brief_file = bf.name
|
||||
|
||||
async def _one(roll, s):
|
||||
# subprocess 阻塞 → to_thread 让出事件循环(超时由 subprocess timeout 兜)。
|
||||
return await asyncio.to_thread(
|
||||
_run_playtest_roll_sync, game_id, brief_file,
|
||||
model_name=mn, roll=roll, seed=s, steps_base=sb, steps_max=smax,
|
||||
evidence_dir=str(ev), port=port, cdp_port=cdp_port, timeout=to)
|
||||
|
||||
# roll-1
|
||||
r1 = await _one(1, base_seed)
|
||||
result["rolls"].append(_roll_brief(r1))
|
||||
cost = _playtest_roll_cost(mn, r1)
|
||||
final = r1
|
||||
# 二掷:仅当 roll-1 是【确定的 fail】(非 degraded)、开关开、成本未撞线时;degraded 不二掷(图像/运行故障重掷无益)。
|
||||
if (not r1.get("degraded")) and (r1.get("pass") is False) and sr and cost < cap:
|
||||
r2 = await _one(2, base_seed + 7919) # 素数偏移换 seed(禁隐式时钟种子)
|
||||
result["rolls"].append(_roll_brief(r2))
|
||||
cost += _playtest_roll_cost(mn, r2)
|
||||
if r2.get("pass") is True:
|
||||
final = r2 # roll-2 翻案 → pass
|
||||
elif r2.get("degraded"):
|
||||
final = r2 # roll-2 degraded → fail-closed
|
||||
else:
|
||||
final = r1 # 两掷同 fail → 维持 fail
|
||||
except Exception as e: # noqa: BLE001 编排层任何异常 → fail-closed degraded
|
||||
result.update({"accepted": False, "verdict": "reject", "pass": False, "degraded": True,
|
||||
"reason": f"测试员编排异常:{type(e).__name__}: {e}", "problems": [], "feedback": "",
|
||||
"summary": "orchestration-error", "firstPlay": dict(_NULL_FIRSTPLAY),
|
||||
"rollCount": len(result["rolls"]), "costRmb": 0.0, "imageBlind": False})
|
||||
_persist_playtest_json(ev, result)
|
||||
return result
|
||||
finally:
|
||||
if brief_file:
|
||||
try:
|
||||
import os as _os # noqa: PLC0415
|
||||
_os.unlink(brief_file)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
final_pass = final.get("pass") is True
|
||||
final_degraded = bool(final.get("degraded"))
|
||||
result.update({
|
||||
"accepted": final_pass and not final_degraded,
|
||||
"verdict": "accept" if (final_pass and not final_degraded) else "reject",
|
||||
"pass": final_pass,
|
||||
"degraded": final_degraded,
|
||||
"reason": final.get("reason"),
|
||||
"problems": final.get("problems") or [], # 现象+推测(修复反馈拼装只引现象,见 _build_repair_feedback)
|
||||
"feedback": final.get("feedback") or "",
|
||||
"summary": final.get("summary") or "",
|
||||
"firstPlay": final.get("firstPlay") or dict(_NULL_FIRSTPLAY),
|
||||
"rollCount": len(result["rolls"]),
|
||||
"costRmb": round(cost, 5),
|
||||
"imageBlind": bool(final.get("imageBlind")),
|
||||
})
|
||||
_persist_playtest_json(ev, result)
|
||||
return result
|
||||
|
||||
|
||||
def _build_repair_feedback(floor: dict, playtest: dict) -> str:
|
||||
"""修复反馈拼装(现象/推测强制分离,续修**只引现象段**)——给 writer resume 的续修来源。
|
||||
|
||||
治 spike 三次「测试员失手后错误归因」失效模式:测试员 problems 里凡含「推测」的分句剔除,只留带步号+落点的
|
||||
客观现象进反馈;degraded 则不派续修(fail-closed 待人工)。附四门地板逐门结果作接线上下文。
|
||||
"""
|
||||
import re # noqa: PLC0415
|
||||
if playtest.get("degraded"):
|
||||
return f"测试员未能给出有效裁决(degraded:{playtest.get('reason')}),fail-closed 待人工复核,不派续修。"
|
||||
g = (floor or {}).get("gates") or {}
|
||||
gate_str = " ".join(f"{k}={'✓' if g.get(k) else '✗'}" for k in ("A", "B", "C", "D"))
|
||||
lines = []
|
||||
for p in (playtest.get("problems") or []):
|
||||
# 现象/推测分离:按分句剔除含「推测」的段,只留客观现象。
|
||||
phenom = "；".join(seg.strip() for seg in re.split(r"[;；]", str(p)) if seg.strip() and "推测" not in seg)
|
||||
if phenom:
|
||||
lines.append(phenom)
|
||||
head = (f"[测试员真玩现象·续修依据] 四门地板 {gate_str};测试员判不通过。以下仅列客观现象(带步号与落点坐标,"
|
||||
f"推测已剔除),据此定位修复:\n")
|
||||
if not lines:
|
||||
lines = [playtest.get("summary") or "测试员判不通过但未列具体现象"]
|
||||
return head + "\n".join(f"- {ln}" for ln in lines)
|
||||
|
||||
|
||||
async def run_acceptance(summary: dict, *, game_id: str, brief: str = "", verdict=None,
|
||||
evidence_dir=None, mode=None, model=None,
|
||||
port=None, cdp_port=None) -> dict:
|
||||
"""统一验收编排器(CLI / Service / modify 三路共用)。accepted = floor ∧ 测试员(mode=v2)/旧口径(shadow/v1)。
|
||||
|
||||
· floor = project_floor(verdict):四门投影(预筛权威,唯一产地),写 summary['floor']。
|
||||
· mode=v2(目标态):测试员真玩阻断,accepted = floor.pass ∧ playtest.accepted。
|
||||
· mode=shadow(波1 灰度):测试员照跑照落证据(对照),accepted 仍取旧口径(判定器 apply_gameplay_judge);
|
||||
并写 summary['shadowV2Accepted'] 供批次账新旧口径对照表。
|
||||
· mode=v1(回退态):W-AXIS 旧口径(apply_gameplay_judge:driven 九门预筛→四门投影 + 判定器读驱动器证据)。
|
||||
|
||||
additive 写 summary:floor 段、acceptanceVersion、playtest 段、judge 段(§4 双写窗口)、ok/accepted。
|
||||
绝不抛(顶层兜底 fail-closed:异常 → accepted=False)。verdict 缺 → 取 summary.verdict/verdictFull。
|
||||
"""
|
||||
try:
|
||||
cfg = _acceptance_cfg()
|
||||
mode = mode or cfg["mode"]
|
||||
ev = evidence_dir if evidence_dir is not None else _wg1_evidence_dir(game_id)
|
||||
v = verdict if verdict is not None else (summary.get("verdict") or summary.get("verdictFull") or {})
|
||||
floor = project_floor(v)
|
||||
summary["floor"] = floor
|
||||
summary["acceptanceVersion"] = mode
|
||||
|
||||
if mode == "v1":
|
||||
# 回退态:预筛权威统一为四门投影(floor.pass),判定仍走驱动器证据(apply_gameplay_judge→judge_gameplay_floor)。
|
||||
# 时序约束:v1 仅在 play-spec/驱动器仍挂载的波1~波2 前有效(波2 拆除后不可达)。
|
||||
return await apply_gameplay_judge(summary, game_id=game_id, brief=brief,
|
||||
evidence_dir=ev, prefilter=floor["pass"])
|
||||
|
||||
# v2 / shadow:跑测试员——仅预筛(四门)过时跑,省 ¥(预筛没过已被机械门拒,不花测试员钱)。
|
||||
if floor["pass"]:
|
||||
playtest = await run_playtest(game_id, brief=brief, evidence_dir=ev,
|
||||
model_name=model or cfg["model"], port=port, cdp_port=cdp_port)
|
||||
else:
|
||||
playtest = {"accepted": False, "verdict": "reject", "pass": False, "degraded": False,
|
||||
"reason": "四门地板未过,不跑测试员(省 ¥)", "problems": [], "feedback": "",
|
||||
"summary": "floor-fail", "rolls": [], "rollCount": 0, "costRmb": 0.0, "model": cfg["model"],
|
||||
"firstPlay": {"playableAtMs": None, "firstFeedbackMs": None, "loopClosed": False}}
|
||||
summary["playtest"] = playtest
|
||||
v2_accepted = bool(floor["pass"]) and bool(playtest.get("accepted")) and not bool(playtest.get("degraded"))
|
||||
|
||||
if mode == "shadow":
|
||||
# 灰度:accepted 取旧口径(判定器 apply_gameplay_judge 写 judge 段);测试员只观测、并列落对照。
|
||||
summary = await apply_gameplay_judge(summary, game_id=game_id, brief=brief,
|
||||
evidence_dir=ev, prefilter=floor["pass"])
|
||||
summary["shadowV2Accepted"] = v2_accepted # 对照:v2 生效时会是什么(批次账新旧口径对照表)
|
||||
return summary
|
||||
|
||||
# mode == v2:测试员阻断放行。
|
||||
blk = cfg["blocking"]
|
||||
# rejectClasses:v2 测试员不产 broken/hollow/off_brief 细类,归一为 playtest_reject / tester_degraded(供 gate 桶归因)。
|
||||
if playtest.get("degraded"):
|
||||
reject_classes = ["tester_degraded"]
|
||||
elif not v2_accepted:
|
||||
reject_classes = ["playtest_reject"]
|
||||
else:
|
||||
reject_classes = []
|
||||
summary["judge"] = { # §4 双写窗口:v2 下 judge 段镜像 playtest 裁决,喂 trace.gameplayJudge 与旧消费面
|
||||
"ran": True, "blocking": blk, "verdict": playtest.get("verdict"),
|
||||
"accepted": bool(playtest.get("accepted")), "rejectClasses": reject_classes,
|
||||
"degraded": bool(playtest.get("degraded")), "reason": playtest.get("reason"),
|
||||
"notes": playtest.get("summary"), "model": playtest.get("model"),
|
||||
"costRmb": playtest.get("costRmb"), "acceptanceVersion": "v2",
|
||||
}
|
||||
summary["accepted"] = v2_accepted
|
||||
summary["ok"] = v2_accepted if blk else floor["pass"]
|
||||
if not v2_accepted: # 续修回喂来源(只引现象段);accepted 时无失败现象、不拼装(E2E 实测曾误拼自相矛盾文案)
|
||||
summary["repairFeedback"] = _build_repair_feedback(floor, playtest)
|
||||
if blk and not v2_accepted and playtest.get("degraded"):
|
||||
summary.setdefault("failureReason", "tester_degraded") # §4 新增归因层
|
||||
_writeback_verdict_json(ev, v2_accepted, summary["judge"])
|
||||
return summary
|
||||
except Exception as e: # noqa: BLE001 编排接线级异常:fail-closed(v2/shadow 不放行)。
|
||||
summary["acceptanceVersion"] = summary.get("acceptanceVersion") or (mode or "v2")
|
||||
summary["judge"] = {"ran": False, "degraded": True,
|
||||
"reason": f"验收编排异常(fail-closed):{type(e).__name__}: {e}"}
|
||||
summary["accepted"] = False
|
||||
summary["ok"] = False
|
||||
return summary
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 便捷自跑：python cheap-worker/cheap_verify.py <gameId> ["brief"] —— 对已有产物真跑一次评分。
|
||||
import json
|
||||
|
||||
@ -94,18 +94,23 @@ async def _run_one(genre, brief, port, cdp, round_no):
|
||||
write_whitelist=write_whitelist, genre=routed_genre, max_tokens=_MAXTOK)
|
||||
except Exception as e: # noqa: BLE001 harness 级异常也原样计入基线(失败也是基线的一部分)
|
||||
return {"genre": genre, "gid": gid, "round": round_no,
|
||||
"accepted": False, "verdictPass": None, "finished": False, "ok": False,
|
||||
"accepted": False, "floorPass": None, "acceptanceVersion": None,
|
||||
"verdictPass": None, "finished": False, "ok": False,
|
||||
"judge": None, "fail": f"{type(e).__name__}: {e}", "wallSec": round(time.time() - t0, 1)}
|
||||
v = summary.get("verdict") or {}
|
||||
floor = summary.get("floor") or {} # 四门投影(run_acceptance 落;缺则空 → floorPass=None)
|
||||
rich = summary.get("richness") or {}
|
||||
layer = summary.get("failureLayer") or {} # classify_cheap_failure_layer 返回 {layer,reason,failedGates}
|
||||
j = summary.get("judge") or {} # apply_gameplay_judge 写入(仅 run_gates∧finished 时有)
|
||||
j = summary.get("judge") or {} # run_acceptance 写入(仅 run_gates∧finished 时有)
|
||||
return {
|
||||
"genre": genre, "gid": gid, "round": round_no,
|
||||
"template": scaffold_template or "_template(通用)",
|
||||
# ── 三个验收信号分开记(2026-07-09 新语义,别混):
|
||||
"accepted": bool(summary.get("accepted")), # ★最终权威 = 预筛 ∧ 独立模型玩法判定
|
||||
"verdictPass": v.get("pass"), # 预筛(机械九门),只是地板不是验收
|
||||
# ── 验收信号分开记(W-AXIS-V2 波1 语义,别混):
|
||||
"accepted": bool(summary.get("accepted")), # ★最终权威 = 四门投影 ∧ 测试员真玩(v2)/ 旧口径(shadow/v1)
|
||||
"floorPass": floor.get("pass"), # ★预筛权威 = 四门投影(A/B/C/D),取代 verdictPass 作预筛口径
|
||||
"acceptanceVersion": summary.get("acceptanceVersion"), # v2/shadow/v1(shadow 与 v2 行靠它区分,防波3对账串数)
|
||||
"shadowV2Accepted": summary.get("shadowV2Accepted"), # shadow 模式下 v2 若生效的影子裁决(新旧口径对照)
|
||||
"verdictPass": v.get("pass"), # harness 原生九门/七门 AND,窗口内仅对照打印、【不作判定输入】
|
||||
"finished": bool(summary.get("finished")), # 模型自称收敛(check+build 绿),不是验收
|
||||
"ok": bool(summary.get("ok")), # run_studio 收口 ok(blocking 下 = accepted)
|
||||
# ── 判定段(verdict/rejectClasses/degraded/costRmb + 诊断):
|
||||
@ -129,16 +134,16 @@ async def _run_one(genre, brief, port, cdp, round_no):
|
||||
|
||||
|
||||
def _mark(r: dict) -> str:
|
||||
"""一局的达标标记(accepted 口径):
|
||||
"""一局的达标标记(accepted 口径,W-AXIS-V2 波1):
|
||||
✅ accepted=True(最终权威过)
|
||||
⚠️判定拒 预筛过(verdictPass=True)但判定拒(accepted=False)——机械门放行、独立判定逮住的坏游戏
|
||||
finished-only 模型自称收敛(finished=True)但预筛未过(旧 FALSE-PASS 改名,新语义)
|
||||
⚠️测试员拒 四门过(floorPass=True)但测试员拒(accepted=False)——机械门放行、测试员逮住的坏游戏
|
||||
finished-only 模型自称收敛(finished=True)但四门未过
|
||||
❌ 未收敛/harness 挂
|
||||
"""
|
||||
if r.get("accepted"):
|
||||
return "✅"
|
||||
if r.get("verdictPass") is True:
|
||||
return "⚠️判定拒"
|
||||
if r.get("floorPass") is True:
|
||||
return "⚠️测试员拒"
|
||||
if r.get("finished"):
|
||||
return "finished-only"
|
||||
return "❌"
|
||||
@ -166,7 +171,8 @@ async def main():
|
||||
cumulative += _row_total_cost(r)
|
||||
j = r.get("judge") or {}
|
||||
print(f"{_mark(r):>13s} r{round_no} {genre:14s} tpl={r.get('template','?')} "
|
||||
f"accepted={r.get('accepted')} verdictPass={r.get('verdictPass')} finished={r.get('finished')} "
|
||||
f"accepted={r.get('accepted')} floorPass={r.get('floorPass')} ver={r.get('acceptanceVersion')} "
|
||||
f"verdictPass(对照)={r.get('verdictPass')} finished={r.get('finished')} "
|
||||
f"judge={j.get('verdict')}/{j.get('rejectClasses')} degraded={j.get('degraded')} "
|
||||
f"layer={r.get('failureLayer')} attempts={r.get('attempts')} "
|
||||
f"genCost=¥{r.get('costRmb')} judgeCost=¥{j.get('costRmb')} wall={r.get('wallSec')}s "
|
||||
@ -177,17 +183,27 @@ async def main():
|
||||
stopped = True
|
||||
break
|
||||
|
||||
# ── 本次汇总(accepted 口径为主,verdictPass/finished 并列对照)──
|
||||
# ── 本次汇总(accepted 口径为主,floorPass/finished 并列;verdictPass 仅对照、不作判定输入)──
|
||||
n = len(session_rows)
|
||||
acc = sum(1 for r in session_rows if r.get("accepted"))
|
||||
vp = sum(1 for r in session_rows if r.get("verdictPass") is True)
|
||||
fp = sum(1 for r in session_rows if r.get("floorPass") is True) # 预筛权威 = 四门投影
|
||||
vp = sum(1 for r in session_rows if r.get("verdictPass") is True) # harness 原生九门/七门,仅对照
|
||||
fin = sum(1 for r in session_rows if r.get("finished"))
|
||||
deg = sum(1 for r in session_rows if (r.get("judge") or {}).get("degraded"))
|
||||
print("\n========== 本次汇总(新验收语义)==========")
|
||||
deg = sum(1 for r in session_rows if (r.get("judge") or {}).get("degraded")
|
||||
or (r.get("playtest") or {}).get("degraded"))
|
||||
print("\n========== 本次汇总(W-AXIS-V2 验收语义)==========")
|
||||
print(f">>> 真实率(accepted 最终权威):{acc}/{n} = {acc / n:.0%}" if n else ">>> 无完成局")
|
||||
if n:
|
||||
print(f">>> 对照:预筛过(verdictPass) {vp}/{n}={vp / n:.0%} | 收敛(finished) {fin}/{n}={fin / n:.0%} | 判定 degraded {deg}/{n}")
|
||||
print(f">>> 差额 = 预筛过但判定拒 {vp - acc} 局(机械门放行、独立判定逮住);全 finished-only={fin - vp} 局(收敛但预筛未过)")
|
||||
print(f">>> 预筛(floorPass 四门投影):{fp}/{n}={fp / n:.0%} | 收敛(finished){fin}/{n}={fin / n:.0%} | degraded {deg}/{n}")
|
||||
print(f">>> 对照(harness 原生 verdictPass,不作判定输入):{vp}/{n}={vp / n:.0%}")
|
||||
print(f">>> 差额 = 四门过但测试员拒 {fp - acc} 局(机械门放行、测试员逮住)")
|
||||
# shadow 灰度对照表(§4/波1 验收):shadow 行的旧口径 accepted vs v2 影子裁决(逐局对账,防波3串数)。
|
||||
shadow_rows = [r for r in session_rows if r.get("acceptanceVersion") == "shadow"
|
||||
and r.get("shadowV2Accepted") is not None]
|
||||
if shadow_rows:
|
||||
old_acc = sum(1 for r in shadow_rows if r.get("accepted"))
|
||||
v2_acc = sum(1 for r in shadow_rows if r.get("shadowV2Accepted"))
|
||||
print(f">>> shadow 对照表:{len(shadow_rows)} 局灰度,旧口径 accepted={old_acc} vs v2 影子 accepted={v2_acc}")
|
||||
print(f">>> 累计成本(本次+历史)= ¥{cumulative:.2f}")
|
||||
# 本次聚合另存一份(便于取用);逐局真相仍以 JSONL 为准。
|
||||
out = _JSONL.parent / f"wax-baseline-r{start_round}-{start_round + n_rounds - 1}.json"
|
||||
|
||||
@ -48,13 +48,19 @@ async def _run_one(sem, tag, brief, port, cdp):
|
||||
write_whitelist=write_whitelist, genre=routed_genre)
|
||||
except Exception as e: # noqa: BLE001
|
||||
return {"tag": tag, "gid": gid, "template": scaffold_template, "locked": write_whitelist is not None,
|
||||
"ok": False, "fail": f"{type(e).__name__}: {e}", "wallSec": round(time.time() - t0, 1)}
|
||||
"ok": False, "accepted": False, "floorPass": None, "acceptanceVersion": None, "verdictPass": None,
|
||||
"fail": f"{type(e).__name__}: {e}", "wallSec": round(time.time() - t0, 1)}
|
||||
v = summary.get("verdict") or {}
|
||||
floor = summary.get("floor") or {}
|
||||
rich = summary.get("richness") or {}
|
||||
return {
|
||||
"tag": tag, "gid": gid, "template": scaffold_template or "_template(通用)",
|
||||
"locked": write_whitelist is not None,
|
||||
"ok": bool(summary.get("ok")), "verdictPass": v.get("pass"),
|
||||
"ok": bool(summary.get("ok")),
|
||||
"accepted": bool(summary.get("accepted")), # W-AXIS-V2:最终验收(四门 ∧ 测试员)
|
||||
"floorPass": floor.get("pass"), # 预筛权威 = 四门投影(取代 verdictPass)
|
||||
"acceptanceVersion": summary.get("acceptanceVersion"),
|
||||
"verdictPass": v.get("pass"), # harness 原生九门/七门,仅对照、不作判定输入
|
||||
"failedGates": v.get("failedGates"), "attempts": summary.get("attempts"),
|
||||
"costRmb": summary.get("costRmb"), "richness": rich.get("score"),
|
||||
"wallSec": round(time.time() - t0, 1),
|
||||
@ -69,14 +75,16 @@ async def main():
|
||||
print("\n\n========== 轻A 跨主题换皮验证 ==========")
|
||||
passed = 0
|
||||
for r in results:
|
||||
vp = r.get("verdictPass") is True
|
||||
passed += 1 if vp else 0
|
||||
mark = "✅" if vp else ("⚠️FALSE-PASS" if r.get("ok") else "❌")
|
||||
acc = bool(r.get("accepted"))
|
||||
fp = r.get("floorPass") is True
|
||||
passed += 1 if acc else 0
|
||||
mark = "✅" if acc else ("⚠️测试员拒" if fp else ("⚠️FALSE-PASS" if r.get("ok") else "❌"))
|
||||
print(f"{mark} {r['tag']:16s} tpl={r.get('template')} lock={r.get('locked')} "
|
||||
f"verdictPass={r.get('verdictPass')} failedGates={r.get('failedGates')} "
|
||||
f"accepted={r.get('accepted')} floorPass={r.get('floorPass')} ver={r.get('acceptanceVersion')} "
|
||||
f"verdictPass(对照)={r.get('verdictPass')} failedGates={r.get('failedGates')} "
|
||||
f"attempts={r.get('attempts')} rich={r.get('richness')}/12 "
|
||||
f"cost={r.get('costRmb')} wall={r.get('wallSec')}s {('FAIL='+r['fail']) if r.get('fail') else ''}")
|
||||
print(f">>> 跨主题换皮达标(九门 verdict.pass):{passed}/{len(results)} 过门率={passed/len(results):.0%}")
|
||||
print(f">>> 跨主题换皮达标(accepted 最终验收 = 四门 ∧ 测试员):{passed}/{len(results)} 达标率={passed/len(results):.0%}")
|
||||
out = Path(__file__).resolve().parent / "results" / "hard-genre-xtheme.json"
|
||||
out.parent.mkdir(exist_ok=True)
|
||||
out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
@ -17,6 +17,8 @@ import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import cheap_verify # W-AXIS-V2 波1:四门投影(project_floor)= 便宜档预筛权威的唯一产地
|
||||
|
||||
# §6.1 result-out / 契约#6 output.failureReason 枚举(对齐后端 FailureReasonEnum,真验改正:
|
||||
# 原稿用了 content_violation/budget_exceeded/generation_failed 三个不在枚举内的值 → 真 e2e 被后端
|
||||
# DifyCallbackTxService L165 拒「failureReason 不在枚举内」、回调事务回滚。真枚举见 FailureReasonEnum)。
|
||||
@ -215,6 +217,27 @@ def _build_trace(summary: dict) -> dict | None:
|
||||
"blocking": judge.get("blocking"),
|
||||
"ran": judge.get("ran"),
|
||||
}
|
||||
# playtest(W-AXIS-V2 波1):additive 透传测试员真玩裁决 + firstPlay 三字段(§5 首局门语义并入,后端
|
||||
# ReadinessScorer.scoreFirstPlay 改读 trace.playtest.firstPlay)。trace.gameplayJudge 双写窗口保留(旧消费面)。
|
||||
playtest = summary.get("playtest")
|
||||
if isinstance(playtest, dict):
|
||||
fp = playtest.get("firstPlay") if isinstance(playtest.get("firstPlay"), dict) else {}
|
||||
trace["playtest"] = {
|
||||
"accepted": playtest.get("accepted"),
|
||||
"verdict": playtest.get("verdict"),
|
||||
"degraded": playtest.get("degraded"),
|
||||
"rollCount": playtest.get("rollCount"),
|
||||
# firstPlay 键名逐字镜像 ReadinessScorer 读取口径(playableAtMs/firstFeedbackMs/loopClosed)。
|
||||
"firstPlay": {"playableAtMs": fp.get("playableAtMs"), "firstFeedbackMs": fp.get("firstFeedbackMs"),
|
||||
"loopClosed": fp.get("loopClosed")},
|
||||
}
|
||||
# floor 四门投影(预筛权威)与 acceptanceVersion:additive,后端可据它区分预筛与最终验收、对账 shadow/v2。
|
||||
floor = summary.get("floor")
|
||||
if isinstance(floor, dict):
|
||||
trace["floor"] = {"pass": floor.get("pass"), "gates": floor.get("gates")}
|
||||
av = summary.get("acceptanceVersion")
|
||||
if av is not None:
|
||||
trace["acceptanceVersion"] = av
|
||||
return trace
|
||||
|
||||
|
||||
@ -239,10 +262,21 @@ def build_result_out(job: dict, summary: dict, game_dir, *,
|
||||
|
||||
bundle_text = _read_bundle(game_dir)
|
||||
verdict = summary.get("verdict") or {}
|
||||
prefilter_pass = verdict.get("pass") is True
|
||||
# W-AXIS 波2:验收权威 = accepted（= 预筛 ∧ 独立模型玩法判定，见 cheap_verify.apply_gameplay_judge）。
|
||||
# summary 无 accepted 键（旧产物 / 对照路 / 判定未跑）→ 回落预筛通过（向后兼容，不误拒）；
|
||||
# 判定拒 / degraded 时 accepted=False → status=failed，判卷合约与既有 succeeded/failed 二分不变。
|
||||
vf = summary.get("verdictFull") if isinstance(summary.get("verdictFull"), dict) else verdict
|
||||
# W-AXIS-V2 波1:预筛权威 = 四门投影(floor)。三级取值(注意 summary.verdict 是 brief 版、无 guards):
|
||||
# ① summary.floor(run_acceptance 落,生产路)→ 直接用;
|
||||
# ② verdictFull 带 guards(老产物 / W-AXIS 波2 路)→ 现算 project_floor(四门投影,**不复用**九门 verdict.pass);
|
||||
# ③ 完全无 guards(极老产物 / 契约桩只给 verdict.pass)→ 回落 verdict.pass 作预筛(向后兼容、不误拒;无 guards
|
||||
# 的假绿在上游 gate_judge/run_acceptance 已拦,build_result_out 是最终契约组装、非验收门)。
|
||||
if isinstance(summary.get("floor"), dict):
|
||||
floor = summary["floor"]
|
||||
elif vf.get("guards"):
|
||||
floor = cheap_verify.project_floor(vf)
|
||||
else:
|
||||
floor = {"pass": bool(verdict.get("pass")), "gates": {}}
|
||||
prefilter_pass = bool(floor.get("pass"))
|
||||
# W-AXIS 波2:验收权威 = accepted（= 预筛 ∧ 测试 agent 真玩裁决）。summary 无 accepted 键（旧产物 / 对照路 /
|
||||
# 验收未跑）→ 回落四门投影（向后兼容，不误拒）；测试员拒 / degraded 时 accepted=False → status=failed。
|
||||
accepted = summary.get("accepted")
|
||||
gates_pass = (accepted is True) if accepted is not None else prefilter_pass
|
||||
engine_ok = bool(bundle_text) and "__GameBundle" in bundle_text
|
||||
|
||||
@ -249,7 +249,10 @@ def test_worker_deterministic_route_success_carries_bundle_and_source():
|
||||
fake = {"status": "succeeded",
|
||||
"manifest": [{"file": "src/core.js", "kind": "config", "key": "ROUND_MS",
|
||||
"old": "30000", "new": "20000", "found": True}],
|
||||
"verdict": {"pass": True, "guards": {}}, "error": None, "stage": "play"}
|
||||
# 真实 verdict.guards 带完整四门(W-AXIS-V2 预筛权威 = 四门投影;build_result_out 据它算 status)。
|
||||
"verdict": {"pass": True, "guards": {"A_boot": {"pass": True}, "B_uncaught": {"pass": True},
|
||||
"C_frame": {"pass": True}, "D_render": {"pass": True}}},
|
||||
"error": None, "stage": "play"}
|
||||
profile = {"tickModel": "realtime", "inputModel": "discrete-choice", "progressModel": "metric"}
|
||||
state = W.WorkerState(modify_fn=lambda *a, **kw: fake,
|
||||
send_fn=lambda u, p, s: captured.update(payload=p) or (200, ""),
|
||||
@ -292,7 +295,9 @@ def test_create_path_unaffected_by_modify_wiring():
|
||||
(gd / "bundle.iife.js").write_text(_GOOD_BUNDLE, encoding="utf-8")
|
||||
called = {}
|
||||
state = W.WorkerState(
|
||||
run_fn=lambda job: (called.update(ran=True) or {"verdict": {"pass": True}}, gd),
|
||||
run_fn=lambda job: (called.update(ran=True) or {"verdict": {"pass": True, "guards": {
|
||||
"A_boot": {"pass": True}, "B_uncaught": {"pass": True},
|
||||
"C_frame": {"pass": True}, "D_render": {"pass": True}}}}, gd),
|
||||
send_fn=lambda u, p, s: called.update(payload=p) or (200, ""),
|
||||
profile_fn=lambda j, g: None,
|
||||
modify_fn=_boom_modify,
|
||||
|
||||
@ -79,6 +79,9 @@ def _fake_rewrite_ok(game_id, intent, **kw):
|
||||
encoding="utf-8")
|
||||
return {"verdict": {"pass": True, "failedGates": []},
|
||||
"verdictFull": {"pass": True, "guards": {}},
|
||||
# W-AXIS-V2 波1:run_studio 经 run_acceptance 收口带 accepted(v2 验收结论);
|
||||
# execute_regenerate_modify 据它判 gates_pass(create/modify 同标准),不再读九门 verdict.pass。
|
||||
"accepted": True,
|
||||
"attempts": 3, "stage": "play", "costRmb": 0.5,
|
||||
"models": {"code": "MiniMax-M3"}, "gameId": game_id}
|
||||
|
||||
|
||||
334
cheap-worker/tests/test_acceptance_v2.py
Normal file
334
cheap-worker/tests/test_acceptance_v2.py
Normal file
@ -0,0 +1,334 @@
|
||||
"""
|
||||
test_acceptance_v2.py — W-AXIS-V2 波1 验收编排单测(四门投影 / run_playtest 二掷 / run_acceptance mode 三态 /
|
||||
修复反馈现象分离 / fail-closed 各路径)。全部零真网络零起服(fake roll 注入 + tmp 证据目录 + monkeypatch)。
|
||||
|
||||
§3 八件的覆盖分工:第 1-5 件(坐标尺 / 落点回显 / 同点硬提示 / 反早退 / 坐标协议)是 playtest.cdp.cjs 的浏览器
|
||||
DOM 行为,由 10 局考卷真跑覆盖(见 spikes/playtest-agent/README 真相表);本单测覆盖可 Python 单测的部分——
|
||||
第 6 件(firstPlay/预算字段流转)、第 7 件(二掷编排)、第 8 件(图像通道 fail-closed),及编排器 mode 三态、
|
||||
投影函数、fail-closed 各路径(超时 / 运行错误 / 未裁决)。
|
||||
|
||||
跑:cheap-worker/.venv/bin/python -m pytest cheap-worker/tests/test_acceptance_v2.py -q
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # → cheap-worker/
|
||||
import cheap_verify # noqa: E402
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# ① project_floor 四门投影(floor 唯一产地;不复用 harness verdict.pass)
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
def _guards(a=True, b=True, c=True, d=True, extra=None):
|
||||
g = {"A_boot": {"pass": a}, "B_uncaught": {"pass": b}, "C_frame": {"pass": c}, "D_render": {"pass": d}}
|
||||
if extra:
|
||||
g.update(extra)
|
||||
return {"guards": g}
|
||||
|
||||
|
||||
def test_project_floor_all_green():
|
||||
f = cheap_verify.project_floor(_guards())
|
||||
assert f["pass"] is True
|
||||
assert f["gates"] == {"A": True, "B": True, "C": True, "D": True}
|
||||
|
||||
|
||||
def test_project_floor_one_gate_fail():
|
||||
assert cheap_verify.project_floor(_guards(c=False))["pass"] is False
|
||||
assert cheap_verify.project_floor(_guards(a=False))["gates"]["A"] is False
|
||||
|
||||
|
||||
def test_project_floor_missing_gate_fail_closed():
|
||||
# 只有 A_boot,B/C/D 缺 → fail-closed(缺门即 False)
|
||||
f = cheap_verify.project_floor({"guards": {"A_boot": {"pass": True}}})
|
||||
assert f["pass"] is False
|
||||
assert f["gates"]["A"] is True and f["gates"]["B"] is False
|
||||
|
||||
|
||||
def test_project_floor_empty():
|
||||
assert cheap_verify.project_floor(None)["pass"] is False
|
||||
assert cheap_verify.project_floor({})["pass"] is False
|
||||
assert cheap_verify.project_floor({"guards": {}})["pass"] is False
|
||||
|
||||
|
||||
def test_project_floor_not_reuse_verdict_pass():
|
||||
# 即使 harness verdict.pass=True,只要四门不全,floor 就不放行(绝不复用 verdict.pass)。
|
||||
v = {"pass": True, "guards": {"A_boot": {"pass": True}, "E_live": {"pass": True}}}
|
||||
assert cheap_verify.project_floor(v)["pass"] is False
|
||||
|
||||
|
||||
def test_project_floor_ignores_eghif_gates():
|
||||
# E/G/H/I/F 挂但四门全绿 → floor 放行(契约门降观测,不入地板)。
|
||||
v = _guards(extra={"E_live": {"pass": False}, "G_input": {"pass": False}, "H_progress": {"pass": False}})
|
||||
assert cheap_verify.project_floor(v)["pass"] is True
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# ② run_playtest 二掷编排(§3 第7件)+ 图像 fail-closed(§3 第8件)——fake roll 注入
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
def _roll(pass_=None, degraded=False, image_blind=False, reason=None):
|
||||
r = {"gid": "t", "roll": None, "pass": pass_, "canSee": not image_blind, "problems": [],
|
||||
"feedback": "", "summary": "s", "steps": 5, "tapPoints": 3, "tokens": {"in": 1000, "out": 100},
|
||||
"firstPlay": {"playableAtMs": 100, "firstFeedbackMs": 100, "loopClosed": bool(pass_)}}
|
||||
if degraded:
|
||||
r["degraded"] = True
|
||||
r["reason"] = reason or "degraded"
|
||||
if image_blind:
|
||||
r["degraded"] = True
|
||||
r["imageBlind"] = True
|
||||
r["reason"] = "图像故障"
|
||||
return r
|
||||
|
||||
|
||||
def _patch_rolls(monkeypatch, rolls, cost=0.1):
|
||||
"""按 roll 序号返回预设结果(rolls=[roll1, roll2, ...]);成本固定,绕网关取价。"""
|
||||
seq = list(rolls)
|
||||
|
||||
def fake(game_id, brief_file, *, roll, **kw):
|
||||
out = dict(seq[roll - 1] if roll - 1 < len(seq) else seq[-1])
|
||||
out["roll"] = roll
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(cheap_verify, "_run_playtest_roll_sync", fake)
|
||||
monkeypatch.setattr(cheap_verify, "_playtest_roll_cost", lambda mn, r: cost)
|
||||
|
||||
|
||||
def _run_pt(monkeypatch, rolls, cost=0.1, **kw):
|
||||
_patch_rolls(monkeypatch, rolls, cost=cost)
|
||||
ev = Path(tempfile.mkdtemp())
|
||||
return asyncio.run(cheap_verify.run_playtest("t", brief="b", evidence_dir=ev, **kw)), ev
|
||||
|
||||
|
||||
def test_playtest_pass_single_roll(monkeypatch):
|
||||
r, _ = _run_pt(monkeypatch, [_roll(pass_=True)], second_roll=True)
|
||||
assert r["accepted"] is True and r["rollCount"] == 1 # pass 一掷即过,不二掷
|
||||
|
||||
|
||||
def test_playtest_fail_then_pass_second_roll(monkeypatch):
|
||||
r, _ = _run_pt(monkeypatch, [_roll(pass_=False), _roll(pass_=True)], second_roll=True)
|
||||
assert r["accepted"] is True and r["rollCount"] == 2 # 二掷翻案(治坐标失手假阴)
|
||||
|
||||
|
||||
def test_playtest_fail_both_rolls(monkeypatch):
|
||||
r, _ = _run_pt(monkeypatch, [_roll(pass_=False), _roll(pass_=False)], second_roll=True)
|
||||
assert r["accepted"] is False and r["rollCount"] == 2 and r["verdict"] == "reject" # 两掷同 fail
|
||||
|
||||
|
||||
def test_playtest_degraded_no_second_roll(monkeypatch):
|
||||
r, _ = _run_pt(monkeypatch, [_roll(degraded=True), _roll(pass_=True)], second_roll=True)
|
||||
assert r["degraded"] is True and r["accepted"] is False and r["rollCount"] == 1 # degraded 不二掷
|
||||
|
||||
|
||||
def test_playtest_image_blind_fail_closed(monkeypatch):
|
||||
r, _ = _run_pt(monkeypatch, [_roll(image_blind=True)], second_roll=True)
|
||||
assert r["degraded"] is True and r["accepted"] is False and r["imageBlind"] is True # §3 第8件
|
||||
|
||||
|
||||
def test_playtest_second_roll_disabled(monkeypatch):
|
||||
r, _ = _run_pt(monkeypatch, [_roll(pass_=False), _roll(pass_=True)], second_roll=False)
|
||||
assert r["accepted"] is False and r["rollCount"] == 1 # 关二掷 → 单掷 fail
|
||||
|
||||
|
||||
def test_playtest_cost_cap_no_second_roll(monkeypatch):
|
||||
r, _ = _run_pt(monkeypatch, [_roll(pass_=False), _roll(pass_=True)], cost=2.0,
|
||||
second_roll=True, cost_cap_rmb=1.5)
|
||||
assert r["rollCount"] == 1 and r["accepted"] is False # roll-1 成本已撞上限,不二掷
|
||||
|
||||
|
||||
def test_playtest_cost_accumulates(monkeypatch):
|
||||
r, _ = _run_pt(monkeypatch, [_roll(pass_=False), _roll(pass_=False)], cost=0.3, second_roll=True)
|
||||
assert abs(r["costRmb"] - 0.6) < 1e-6 # 两掷成本累加
|
||||
|
||||
|
||||
def test_playtest_firstplay_passthrough(monkeypatch):
|
||||
r, _ = _run_pt(monkeypatch, [_roll(pass_=True)], second_roll=True)
|
||||
assert r["firstPlay"]["loopClosed"] is True and r["firstPlay"]["playableAtMs"] == 100 # §5 三字段透传
|
||||
|
||||
|
||||
def test_playtest_persist_json(monkeypatch):
|
||||
_, ev = _run_pt(monkeypatch, [_roll(pass_=True)], second_roll=True)
|
||||
assert (ev / "playtest" / "playtest.json").is_file() # 真相层落盘(§6.13 亲眼验收看它)
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# ③ run_acceptance mode 三态(v2 / shadow / v1)
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
async def _fake_pt_accept(game_id, **kw):
|
||||
return {"accepted": True, "verdict": "accept", "pass": True, "degraded": False, "problems": [],
|
||||
"feedback": "", "summary": "ok", "rollCount": 1, "costRmb": 0.1, "model": "M",
|
||||
"firstPlay": {"playableAtMs": 100, "firstFeedbackMs": 100, "loopClosed": True}}
|
||||
|
||||
|
||||
async def _fake_pt_reject(game_id, **kw):
|
||||
return {"accepted": False, "verdict": "reject", "pass": False, "degraded": False,
|
||||
"problems": ["第2步点(100,200)后画面未变;推测:事件未绑定"], "feedback": "修", "summary": "no",
|
||||
"rollCount": 2, "costRmb": 0.2, "model": "M",
|
||||
"firstPlay": {"playableAtMs": None, "firstFeedbackMs": None, "loopClosed": False}}
|
||||
|
||||
|
||||
async def _fake_apply_judge(summary, **kw):
|
||||
summary["accepted"] = True
|
||||
summary["ok"] = True
|
||||
summary["judge"] = {"ran": True, "verdict": "accept", "blocking": True}
|
||||
return summary
|
||||
|
||||
|
||||
def _acc(monkeypatch, verdict, mode, pt=_fake_pt_accept, judge=None):
|
||||
monkeypatch.setattr(cheap_verify, "run_playtest", pt)
|
||||
if judge is not None:
|
||||
monkeypatch.setattr(cheap_verify, "apply_gameplay_judge", judge)
|
||||
monkeypatch.setattr(cheap_verify, "_writeback_verdict_json", lambda *a, **k: None)
|
||||
ev = str(Path(tempfile.mkdtemp()))
|
||||
return asyncio.run(cheap_verify.run_acceptance({}, game_id="t", brief="b", verdict=verdict,
|
||||
mode=mode, evidence_dir=ev))
|
||||
|
||||
|
||||
def test_run_acceptance_v2_accept(monkeypatch):
|
||||
s = _acc(monkeypatch, _guards(), "v2", pt=_fake_pt_accept)
|
||||
assert s["accepted"] is True and s["ok"] is True and s["acceptanceVersion"] == "v2"
|
||||
assert s["floor"]["pass"] is True and s["playtest"]["accepted"] is True
|
||||
assert "repairFeedback" not in s # accepted 无失败现象,不拼装续修反馈(E2E 曾误拼自相矛盾文案)
|
||||
|
||||
|
||||
def test_run_acceptance_v2_reject_by_playtest(monkeypatch):
|
||||
s = _acc(monkeypatch, _guards(), "v2", pt=_fake_pt_reject)
|
||||
assert s["accepted"] is False # 四门过但测试员拒
|
||||
assert s["judge"]["rejectClasses"] == ["playtest_reject"]
|
||||
assert "repairFeedback" in s and "第2步点(100,200)后画面未变" in s["repairFeedback"]
|
||||
|
||||
|
||||
def test_run_acceptance_v2_floor_fail_skips_playtest(monkeypatch):
|
||||
called = {"n": 0}
|
||||
|
||||
async def spy(game_id, **kw):
|
||||
called["n"] += 1
|
||||
return {"accepted": True}
|
||||
|
||||
s = _acc(monkeypatch, _guards(c=False), "v2", pt=spy)
|
||||
assert called["n"] == 0 and s["accepted"] is False # 四门不过不跑测试员(省 ¥)
|
||||
assert s["playtest"]["reason"].startswith("四门")
|
||||
|
||||
|
||||
def test_run_acceptance_shadow_takes_old_verdict(monkeypatch):
|
||||
# shadow:accepted 取旧口径(apply_gameplay_judge accept),测试员照跑照落 + shadowV2Accepted 对照。
|
||||
s = _acc(monkeypatch, _guards(), "shadow", pt=_fake_pt_reject, judge=_fake_apply_judge)
|
||||
assert s["accepted"] is True # 旧口径判 accept
|
||||
assert s["shadowV2Accepted"] is False # v2 影子(测试员拒)——新旧口径对照
|
||||
assert s["playtest"]["accepted"] is False # 测试员照跑照落证据
|
||||
|
||||
|
||||
def test_run_acceptance_v1_uses_apply_judge(monkeypatch):
|
||||
called = {"n": 0}
|
||||
|
||||
async def spy(*a, **k):
|
||||
called["n"] += 1
|
||||
return {}
|
||||
|
||||
s = _acc(monkeypatch, _guards(), "v1", pt=spy, judge=_fake_apply_judge)
|
||||
assert s["accepted"] is True and called["n"] == 0 # v1 不跑测试员,走旧判定器
|
||||
assert s["acceptanceVersion"] == "v1" and s["floor"]["pass"] is True
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# ④ 修复反馈现象/推测强制分离(续修只引现象段)
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
def test_repair_feedback_strips_speculation():
|
||||
floor = {"pass": True, "gates": {"A": True, "B": True, "C": True, "D": True}}
|
||||
pt = {"degraded": False, "summary": "no",
|
||||
"problems": ["第2步点(100,200)后画面未变;推测:事件未绑定", "推测:可能是渲染 bug"]}
|
||||
fb = cheap_verify._build_repair_feedback(floor, pt)
|
||||
assert "第2步点(100,200)后画面未变" in fb # 现象保留
|
||||
assert "事件未绑定" not in fb # 分句里的推测剔除
|
||||
assert "渲染 bug" not in fb # 整条推测剔除
|
||||
|
||||
|
||||
def test_repair_feedback_degraded_no_repair():
|
||||
fb = cheap_verify._build_repair_feedback({"gates": {}}, {"degraded": True, "reason": "图像故障"})
|
||||
assert "不派续修" in fb and "图像故障" in fb
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
# ⑤ _run_playtest_roll_sync 退出码语义(fail-closed 各路径)+ 配置/辅助
|
||||
# ══════════════════════════════════════════════════════════════════════════════
|
||||
class _FakeProc:
|
||||
def __init__(self, stdout, returncode):
|
||||
self.stdout, self.stderr, self.returncode = stdout, "", returncode
|
||||
|
||||
|
||||
def _roll_sync(monkeypatch, proc_or_exc):
|
||||
monkeypatch.setattr(cheap_verify, "_playtest_env", lambda: {})
|
||||
if isinstance(proc_or_exc, Exception):
|
||||
def r(*a, **k):
|
||||
raise proc_or_exc
|
||||
else:
|
||||
def r(*a, **k):
|
||||
return proc_or_exc
|
||||
monkeypatch.setattr(cheap_verify.subprocess, "run", r)
|
||||
return cheap_verify._run_playtest_roll_sync(
|
||||
"t", "/tmp/b", model_name="M", roll=1, seed=1, steps_base=14, steps_max=24,
|
||||
evidence_dir="/tmp", port=1, cdp_port=2, timeout=10)
|
||||
|
||||
|
||||
def test_roll_sync_image_blind_exit3(monkeypatch):
|
||||
v = json.dumps({"gid": "t", "pass": False, "canSee": False, "imageBlind": True})
|
||||
r = _roll_sync(monkeypatch, _FakeProc(v, 3))
|
||||
assert r["degraded"] is True and r["imageBlind"] is True # 退出码 3 → 图像故障 fail-closed
|
||||
|
||||
|
||||
def test_roll_sync_runner_error_exit2(monkeypatch):
|
||||
r = _roll_sync(monkeypatch, _FakeProc("", 2))
|
||||
assert r["degraded"] is True # 退出码 2 / 未产裁决 → degraded
|
||||
|
||||
|
||||
def test_roll_sync_normal_pass(monkeypatch):
|
||||
v = json.dumps({"gid": "t", "pass": True, "canSee": True})
|
||||
r = _roll_sync(monkeypatch, _FakeProc(v, 0))
|
||||
assert r["pass"] is True and not r.get("degraded") # 退出码 0 正常裁决
|
||||
|
||||
|
||||
def test_roll_sync_timeout(monkeypatch):
|
||||
r = _roll_sync(monkeypatch, cheap_verify.subprocess.TimeoutExpired("cmd", 10))
|
||||
assert r["degraded"] is True and "超时" in r["reason"] # 子进程超时 → fail-closed
|
||||
|
||||
|
||||
def test_roll_sync_no_json(monkeypatch):
|
||||
r = _roll_sync(monkeypatch, _FakeProc("some non-json noise", 0))
|
||||
assert r["degraded"] is True and "未产出裁决" in r["reason"] # 无裁决 JSON → fail-closed
|
||||
|
||||
|
||||
def test_acceptance_cfg_defaults():
|
||||
cfg = cheap_verify._acceptance_cfg()
|
||||
assert cfg["mode"] in ("v2", "shadow", "v1")
|
||||
assert isinstance(cfg["steps_base"], int) and isinstance(cfg["steps_max"], int)
|
||||
assert cfg["steps_max"] >= cfg["steps_base"] and cfg["cost_cap_rmb"] > 0
|
||||
|
||||
|
||||
def test_derive_playtest_ports_stable():
|
||||
p1 = cheap_verify._derive_playtest_ports("hard-puzzle-r1")
|
||||
p2 = cheap_verify._derive_playtest_ports("hard-puzzle-r1")
|
||||
assert p1 == p2 and 4998 <= p1[0] < 5100 and 9331 <= p1[1] < 9431 # 同 id 恒同端口(可复现)
|
||||
|
||||
|
||||
def test_derive_playtest_ports_skips_chrome_unsafe():
|
||||
# 2026-07-10 考卷实锤:sb2 曾派生到 5060(SIP,Chrome ERR_UNSAFE_PORT 拒加载)→ 假 boot-timeout。
|
||||
# 派生必须跳过 unsafe 清单;全 hash 空间扫一遍保证无一落入。
|
||||
assert cheap_verify._derive_playtest_ports("hard-sim-business-r2")[0] not in cheap_verify._CHROME_UNSAFE_PORTS
|
||||
for h in range(100):
|
||||
port = 4998 + h
|
||||
if port in cheap_verify._CHROME_UNSAFE_PORTS:
|
||||
port += 2
|
||||
assert port not in cheap_verify._CHROME_UNSAFE_PORTS
|
||||
|
||||
|
||||
def test_roll_brief_shape():
|
||||
b = cheap_verify._roll_brief({"roll": 1, "seed": 42, "pass": True, "steps": 5,
|
||||
"summary": "s", "tokens": {"in": 1, "out": 2}})
|
||||
assert b["roll"] == 1 and b["pass"] is True and b["tokens"] == {"in": 1, "out": 2}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pytest
|
||||
sys.exit(pytest.main([__file__, "-q"]))
|
||||
@ -44,7 +44,10 @@ def test_build_summary_shape_feeds_result_out(tmp_path, monkeypatch):
|
||||
gd.mkdir(parents=True)
|
||||
(gd / "bundle.iife.js").write_text("var x=1; window.__GameBundle={};", encoding="utf-8")
|
||||
monkeypatch.setattr(D, "_read_driver_type", lambda gid: "tap-targets")
|
||||
verdict = {"pass": True, "guards": {"A_boot": {"pass": True}, "H_progress": {"pass": True}}}
|
||||
# 真实 verdict.guards 带完整四门(A/B/C/D)——W-AXIS-V2 预筛权威 = 四门投影;H_progress 保留(降观测门,trace 仍带)。
|
||||
verdict = {"pass": True, "guards": {"A_boot": {"pass": True}, "B_uncaught": {"pass": True},
|
||||
"C_frame": {"pass": True}, "D_render": {"pass": True},
|
||||
"H_progress": {"pass": True}}}
|
||||
svc = {"costRmb": 0.42, "rmbGate": "active", "repairs": 2, "budgetSoftTripped": False}
|
||||
turn = {"ended": True, "reason": "REPLY_END"}
|
||||
summary = D._build_summary("g9", "点击得分小游戏", verdict, svc, turn, t0=0.0)
|
||||
@ -102,7 +105,10 @@ def test_build_summary_trace_contract_end_to_end(tmp_path, monkeypatch):
|
||||
gd.mkdir(parents=True)
|
||||
(gd / "bundle.iife.js").write_text("var x=1; window.__GameBundle={};", encoding="utf-8")
|
||||
monkeypatch.setattr(D, "_read_driver_type", lambda gid: "tap-targets")
|
||||
verdict = {"pass": True, "guards": {"A_boot": {"pass": True}, "H_progress": {"pass": True}}}
|
||||
# 真实 verdict.guards 带完整四门(A/B/C/D)——W-AXIS-V2 预筛权威 = 四门投影;H_progress 保留(降观测门,trace 仍带)。
|
||||
verdict = {"pass": True, "guards": {"A_boot": {"pass": True}, "B_uncaught": {"pass": True},
|
||||
"C_frame": {"pass": True}, "D_render": {"pass": True},
|
||||
"H_progress": {"pass": True}}}
|
||||
svc = {"costRmb": 0.42, "rmbGate": "active", "repairs": 2, "budgetSoftTripped": False}
|
||||
summary = D._build_summary("70012", "点击得分小游戏", verdict, svc,
|
||||
{"ended": True, "reason": "REPLY_END"}, t0=0.0)
|
||||
@ -189,16 +195,19 @@ def test_drive_cheap_generation_fake_sse(tmp_path, monkeypatch):
|
||||
# ① 回合真结束(REPLY_END)→ fake turn 写本次 verdict/sidecar → driver 回合后读它组 succeeded 形。
|
||||
async def _ended(*a, **k):
|
||||
vev = tmp_path / "_wg1-gen" / "70012" / "evidence"; vev.mkdir(parents=True, exist_ok=True)
|
||||
(vev / "verdict.json").write_text(json.dumps({"pass": True, "guards": {"A_boot": {"pass": True}}}), encoding="utf-8")
|
||||
(vev / "verdict.json").write_text(json.dumps({"pass": True, "guards": {
|
||||
"A_boot": {"pass": True}, "B_uncaught": {"pass": True},
|
||||
"C_frame": {"pass": True}, "D_render": {"pass": True}}}), encoding="utf-8")
|
||||
sev = tmp_path / "amgen-70012" / "evidence"; sev.mkdir(parents=True, exist_ok=True)
|
||||
(sev / "service-run-summary.json").write_text(
|
||||
json.dumps({"costRmb": 0.5, "rmbGate": "active", "repairs": 1}), encoding="utf-8")
|
||||
return {"ended": True, "reason": "REPLY_END", "endEvent": {}}
|
||||
monkeypatch.setattr(CP, "_wait_for_turn_end", _ended)
|
||||
# W-AXIS 波2:本用例验的是单 POST 驱动器管路(scaffold/session/verdict/sidecar/costRmb → ok 映射)=机械预筛这一半;
|
||||
# 玩法地板判定是新增的独立阻断层(无真截图+无判定模型的桩环境跑不了它),故这里关 blocking,让 ok=预筛,
|
||||
# 隔离出管路本身。判定阻断路由 test_gameplay_judge.py 与真跑 w2a-verify 覆盖。判定仍会 observe-only 跑一遍
|
||||
# (无截图 → fail-closed degraded、无网络无成本),下面顺带断言 judge 段已 additive 落盘、accepted 如实记 False。
|
||||
# W-AXIS-V2 波1:本用例验的是单 POST 驱动器管路(scaffold/session/verdict/sidecar/costRmb → ok 映射)=四门预筛
|
||||
# 这一半;测试 agent 真玩是独立阻断层、桩环境(无 Chrome/无判定模型)起不了服,故设 acceptance.mode=v1 隔离出
|
||||
# 管路本身(v1 走 apply_gameplay_judge 不起服测试员)+ 关 blocking 让 ok=预筛(四门投影)。测试员阻断/mode 三态
|
||||
# 由 test_acceptance_v2.py 隔离覆盖、真玩由考卷真跑覆盖。判定仍 observe-only 跑一遍(无截图 → fail-closed degraded)。
|
||||
monkeypatch.setenv("TIER2_GEN__ACCEPTANCE__MODE", "v1")
|
||||
monkeypatch.setenv("TIER2_GEN__JUDGE__BLOCKING", "false")
|
||||
summary, gdir = asyncio.run(D.drive_cheap_generation({"gameId": 70012, "traceId": "t9", "brief": "点球"}))
|
||||
assert scaffolded["gid"] == "70012", "C2:scaffold 用后端 gameId(str),非 session_id"
|
||||
|
||||
@ -198,16 +198,26 @@ public class GenMetrics {
|
||||
@SuppressWarnings("unchecked")
|
||||
private void recordGateFails(Map<String, Object> trace) {
|
||||
Object verdict = trace.get("sevenGateVerdict");
|
||||
if (!(verdict instanceof Map)) {
|
||||
return;
|
||||
if (verdict instanceof Map) {
|
||||
Object guards = ((Map<String, Object>) verdict).get("guards");
|
||||
if (guards instanceof Map) {
|
||||
for (Map.Entry<String, Object> gate : ((Map<String, Object>) guards).entrySet()) {
|
||||
if (!guardPass(gate.getValue())) {
|
||||
registry.counter(M_GATE_FAIL, "gate", gate.getKey()).increment();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Object guards = ((Map<String, Object>) verdict).get("guards");
|
||||
if (!(guards instanceof Map)) {
|
||||
return;
|
||||
}
|
||||
for (Map.Entry<String, Object> gate : ((Map<String, Object>) guards).entrySet()) {
|
||||
if (!guardPass(gate.getValue())) {
|
||||
registry.counter(M_GATE_FAIL, "gate", gate.getKey()).increment();
|
||||
// W-AXIS-V2 波1（§4）：测试 agent 真玩验收失败也计入 gen_gate_fail 归因——让观测线区分「机械门失败」与
|
||||
// 「玩法验收失败」。gate=playtest（测试员判 broken/hollow/off_brief 拒）/ gate=tester_degraded（评不出、
|
||||
// fail-closed 待人工）。源 = trace.playtest（result_out 落）；playtest 段缺失（v1/老产物）→ 不记，向后兼容。
|
||||
Object playtest = trace.get("playtest");
|
||||
if (playtest instanceof Map) {
|
||||
Map<String, Object> pt = (Map<String, Object>) playtest;
|
||||
Object accepted = pt.get("accepted");
|
||||
if (accepted != null && !asBool(accepted)) {
|
||||
String gate = asBool(pt.get("degraded")) ? "tester_degraded" : "playtest";
|
||||
registry.counter(M_GATE_FAIL, "gate", gate).increment();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -107,12 +107,22 @@ public class ReadinessScorer {
|
||||
}
|
||||
}
|
||||
|
||||
/** 可玩性子项：trace.pass==true → 1.0；明确 false → 0；缺失 → 中性 0.5。 */
|
||||
/** 可玩性子项：优先测试员真玩裁决 trace.playtest.accepted（W-AXIS-V2），回落 trace.pass；缺失 → 中性 0.5。 */
|
||||
private double scorePlayability(Map<String, Object> trace) {
|
||||
return scorePlayability(trace, view());
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private double scorePlayability(Map<String, Object> trace, ReadinessView v) {
|
||||
// W-AXIS-V2 波1（§5）：可玩性优先读测试 agent 真玩裁决（playtest.accepted = 真人玩得通、有决策层、切题）；
|
||||
// 契约退役后 trace.pass 仍作预筛/老产物回落，playtest 段存在时以它为准。
|
||||
Object playtest = trace.get("playtest");
|
||||
if (playtest instanceof Map) {
|
||||
Object accepted = ((Map<String, Object>) playtest).get("accepted");
|
||||
if (accepted != null) {
|
||||
return asBool(accepted) ? 1.0 : 0.0;
|
||||
}
|
||||
}
|
||||
Object pass = trace.get("pass");
|
||||
if (pass == null) {
|
||||
return v.neutralScore(NEUTRAL);
|
||||
@ -136,6 +146,19 @@ public class ReadinessScorer {
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private double scoreFirstPlay(Map<String, Object> trace, ReadinessView v) {
|
||||
// W-AXIS-V2 波1（§5）：首局门三断言随驱动器/H_progress 契约退役，由测试员转写接管——优先读
|
||||
// trace.playtest.firstPlay：测试员真玩到闭环（loopClosed）且首次交互有效（playableAtMs 非空）→ 1.0。
|
||||
Object playtest = trace.get("playtest");
|
||||
if (playtest instanceof Map) {
|
||||
Object firstPlay = ((Map<String, Object>) playtest).get("firstPlay");
|
||||
if (firstPlay instanceof Map) {
|
||||
Map<String, Object> fp = (Map<String, Object>) firstPlay;
|
||||
boolean loopClosed = asBool(fp.get("loopClosed"));
|
||||
boolean playable = fp.get("playableAtMs") != null; // 首次有效交互时刻存在 = 玩得起来
|
||||
return (loopClosed && playable) ? 1.0 : v.neutralScore(NEUTRAL);
|
||||
}
|
||||
}
|
||||
// 回落旧口径（v1 / 老产物：driver + H_progress，契约仍挂载的窗口内有效）——向后兼容。
|
||||
Object gatespec = trace.get("gatespec");
|
||||
boolean hasDriver = gatespec instanceof Map && ((Map<String, Object>) gatespec).get("driver") != null;
|
||||
Object verdict = trace.get("sevenGateVerdict");
|
||||
|
||||
374
game-runtime/games/_wg1-gen/_shared/playtest.cdp.cjs
Normal file
374
game-runtime/games/_wg1-gen/_shared/playtest.cdp.cjs
Normal file
@ -0,0 +1,374 @@
|
||||
/**
|
||||
* playtest.cdp.cjs —— 便宜档验收 v2「测试 agent 真玩取证驱动」(W-AXIS-V2 波1 产出①)。
|
||||
*
|
||||
* 一个多模态测试 agent 按 brief 把一款便宜档小游戏当真人一样玩起来:每步只看人类可见的证据
|
||||
* (截图 + 运行日志尾巴,绝不读 `_forensicsView` 契约状态)→ 决定动作(tap/key/wait)→ CDP 派发 →
|
||||
* 最终给结构化裁决与修复反馈。它替代旧 E_live/G_input/H_progress 三门与 tap-targets 驱动器:验收证据
|
||||
* 不再由「一份固定代码玩几十种游戏」的取证契约生产,而由测试员亲手真玩产生。裁决权威见质量模型 SoT 裁定三。
|
||||
*
|
||||
* ════════════════════════════════════════════════════════════════════════════
|
||||
* 【playtest/1 协议】(实现契约,runner=cheap_verify.run_playtest 与本文件双方遵守)
|
||||
*
|
||||
* · 坐标协议(§3 第1件):自截自用,`Emulation.setDeviceMetricsOverride` 390×844、DPR=1,截图像素与逻辑
|
||||
* 坐标 1:1,模型报的 tap 坐标即 CSS 坐标直接派发;与 play.cdp.cjs 的 DPR2 取证截图互不掺和(两套工具)。
|
||||
*
|
||||
* · 命令行:node playtest.cdp.cjs <gameId>
|
||||
* --brief-file=<path> brief 文本文件(诚实来源;runner 从生成时落盘的 brief 写出,避免命令行转义)
|
||||
* --model=MiniMax-M3 测试员模型(runner 从 genconfig judge.model 透传;不硬编码)
|
||||
* --steps-base=14 基础步数预算(§3 第6件)
|
||||
* --steps-max=24 有进展证据时自动扩到的步数上限(§3 第6件)
|
||||
* --seed=<int> runner 显式生成的种子(记进转写;经 addScriptToEvaluateOnNewDocument 注入
|
||||
* window.__playtestSeed;禁隐式时钟种子——两掷独立性靠它 + 清 localStorage)
|
||||
* --roll=1 第几掷(证据落 evidence/playtest/roll-<roll>/;fail 二掷=roll-2,§3 第7件)
|
||||
* --evidence-dir=<path> 证据根(缺省 games/_wg1-gen/<id>/evidence);实落 <root>/playtest/roll-<roll>/
|
||||
* --base=http://localhost:4998 静态服务(runner 起)
|
||||
* --cdp=http://localhost:9331 Chrome CDP(runner 起)
|
||||
*
|
||||
* · 动作 JSON(模型每步输出恰一个,无 markdown 围栏):
|
||||
* {"act":"tap","x":195,"y":489,"why":"点开始按钮"} 坐标系 390×844,原点左上
|
||||
* {"act":"key","key":"ArrowLeft","why":"向左"}
|
||||
* {"act":"wait","ms":800,"why":"等动画"}
|
||||
* {"act":"verdict","pass":true|false,"canSeeScreenshot":true|false,
|
||||
* "problems":["现象:第N步点(x,y)后画面未变"], 现象必须带步号与落点(现象/推测分离)
|
||||
* "feedback":"给写游戏 agent 的修复指引(先列现象、再标注推测)","summary":"一句话"}
|
||||
*
|
||||
* · 退出码:0=正常给出裁决(pass/fail 都算正常完成) | 2=运行错误(装载失败/未连上/异常) |
|
||||
* 3=图像通道故障 fail-closed(测试员自报看不到截图,§3 第8件——归因测试员图像通道,非游戏缺陷)
|
||||
*
|
||||
* · evidence 目录布局(落 <evidence-dir>/playtest/roll-<roll>/):
|
||||
* transcript.json 完整逐步转写(每步 act/why/落点/日志尾/画面是否变化/相对时间戳)+ 裁决 + firstPlay
|
||||
* step-<NN>.jpg 每步喂给模型的截图(带坐标尺 + 落点回显),即验收硬证据链
|
||||
*
|
||||
* · stdout 末行 = 单行 JSON 裁决(runner 解析);裁决含 firstPlay:{playableAtMs,firstFeedbackMs,loopClosed}
|
||||
* 三字段(§5:从转写机械提取,首局门语义并入)。
|
||||
* ════════════════════════════════════════════════════════════════════════════
|
||||
*/
|
||||
'use strict';
|
||||
const fs = require('node:fs');
|
||||
const http = require('node:http');
|
||||
const path = require('node:path');
|
||||
// 复用 harness 的 CDP 会话类与像素抓取(与 play.cdp.cjs 同一份;cwd=game-runtime 已 npm i ws)。
|
||||
const H = require(path.resolve(__dirname, '../../../test/harness/browser-evidence.cjs'));
|
||||
const { CdpSession, captureImageData, fnv1a32 } = H;
|
||||
const WebSocket = require('ws');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const GID = args.find((a) => !a.startsWith('--'));
|
||||
const opt = (k, d) => { const m = args.find((a) => a.startsWith(`--${k}=`)); return m ? m.split('=').slice(1).join('=') : d; };
|
||||
const MODEL = opt('model', 'MiniMax-M3');
|
||||
const STEPS_BASE = parseInt(opt('steps-base', '14'), 10);
|
||||
const STEPS_MAX = parseInt(opt('steps-max', '24'), 10);
|
||||
const SEED = parseInt(opt('seed', String(Date.now() % 1e9)), 10); // runner 应显式传;缺省仅兜底(会记进转写)
|
||||
const ROLL = parseInt(opt('roll', '1'), 10);
|
||||
const BASE = (opt('base', 'http://localhost:4998') || '').replace(/\/$/, '');
|
||||
const CDP = (opt('cdp', 'http://localhost:9331') || '').replace(/\/$/, '');
|
||||
const BRIEF_FILE = opt('brief-file', '');
|
||||
const EVIDENCE_ROOT = opt('evidence-dir', path.resolve(__dirname, '..', GID || '', 'evidence'));
|
||||
const OUT_DIR = path.join(EVIDENCE_ROOT, 'playtest', `roll-${ROLL}`);
|
||||
|
||||
const NEWAPI_BASE = process.env.NEWAPI_BASE_URL || 'http://100.64.0.8:3000';
|
||||
const KEY = process.env.NEWAPI_KEY;
|
||||
if (!GID) { console.error('缺 <gameId>'); process.exit(2); }
|
||||
if (!KEY) { console.error('缺 NEWAPI_KEY(runner 应从凭据档注入)'); process.exit(2); }
|
||||
|
||||
const delay = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
/** 极简 HTTP(PUT 建 target / GET 列表);Node http 模块直连,天然绕系统代理(内网直连红线)。 */
|
||||
function httpJson(method, urlStr) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const u = new URL(urlStr);
|
||||
const req = http.request({ hostname: u.hostname, port: u.port, path: u.pathname + u.search, method, timeout: 15000 },
|
||||
(res) => { let b = ''; res.on('data', (c) => (b += c)); res.on('end', () => { try { resolve(b ? JSON.parse(b) : {}); } catch (e) { reject(e); } }); });
|
||||
req.on('error', reject); req.on('timeout', () => req.destroy(new Error('timeout'))); req.end();
|
||||
});
|
||||
}
|
||||
|
||||
/** 调 new-api /v1/chat/completions(OpenAI 兼容,image_url data-URI);http 模块直连不走系统代理。 */
|
||||
function chat(messages, maxTokens) {
|
||||
const body = JSON.stringify({ model: MODEL, max_tokens: maxTokens || 3000, messages });
|
||||
const u = new URL(NEWAPI_BASE + '/v1/chat/completions');
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = http.request({ hostname: u.hostname, port: u.port, path: u.pathname, method: 'POST',
|
||||
headers: { 'Authorization': `Bearer ${KEY}`, 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) }, timeout: 120000 },
|
||||
(res) => { let b = ''; res.on('data', (c) => (b += c)); res.on('end', () => {
|
||||
try {
|
||||
const d = JSON.parse(b);
|
||||
if (res.statusCode !== 200) return reject(new Error(`HTTP ${res.statusCode}: ${b.slice(0, 200)}`));
|
||||
const msg = (d.choices || [{}])[0].message || {};
|
||||
resolve({ text: msg.content || '', usage: d.usage || {} });
|
||||
} catch (e) { reject(new Error('响应非 JSON: ' + b.slice(0, 200))); }
|
||||
}); });
|
||||
req.on('error', reject); req.on('timeout', () => req.destroy(new Error('LLM 调用超时'))); req.write(body); req.end();
|
||||
});
|
||||
}
|
||||
|
||||
// 测试员系统提示。现象/推测强制分离(§3 修复反馈契约)= 治 spike 三次「失手后错误归因」失效模式的直接约束:
|
||||
// problems 里凡断言游戏有问题,必须先写【现象】(第 N 步点 (x,y) 后画面/日志有无变化——可复核的客观事实),
|
||||
// 再把「可能是事件没绑定」这类【推测】另起并显式标注;续修只吃现象段。
|
||||
const SYSTEM = `你是小游戏【真人视角测试员】。你只依赖两样人类可见的证据:游戏截图 + 运行日志尾巴。绝不臆测截图里没有的东西。
|
||||
任务:按 brief 把这款游戏当真人一样玩起来(点按钮/点目标/等待),验证三件事:①玩得通(能从菜单进游戏、操作有响应、能推进到有结果)②有决策层(点哪有区别、有输赢/取舍)③与 brief 是同一款游戏。
|
||||
【坐标必读】截图上烧了粉色坐标尺:细格 50px、粗格 100px,左缘 y100..y800、上缘 x100..x300 是刻度标签。
|
||||
报 tap 坐标前,先对着刻度读出目标的位置(如:按钮中心在 y400 与 y500 粗线正中间 → y=450),绝不凭感觉估。
|
||||
【落点回显】截图上的亮黄色圆环+十字 = 你上一步 tap 的真实落点。每一步先看它:落点不在你想点的目标上,
|
||||
就按偏差校正坐标(如圆环在按钮下方 60px → 下一步 y 减 60);落点在目标上而画面没反应,才可能是游戏的问题。
|
||||
【现象与推测分离(硬要求)】你判 fail 时,problems 每一条先写【现象】:第几步点了哪个坐标 (x,y)、之后画面/日志有没有变化——
|
||||
这是可复核的客观事实;若要猜原因(如「可能是点击事件没绑定」),必须另起一句并以「推测:」开头。绝不把推测当现象。
|
||||
每一步你只输出一个 JSON(不要 markdown 围栏):
|
||||
{"act":"tap","x":195,"y":489,"why":"点开始按钮(按钮上沿贴 y500 线上方约半格)"}
|
||||
{"act":"key","key":"ArrowLeft","why":"向左"}
|
||||
{"act":"wait","ms":800,"why":"等动画/观察窗"}
|
||||
{"act":"verdict","pass":true,"canSeeScreenshot":true,"problems":[],"feedback":"通关说明或修复指引","summary":"一句话"}
|
||||
规则:每次 tap 后对照上一步截图判断画面变没变;没变就【必须换坐标】(先沿 y 轴上下各挪 50 试)或 wait,同一坐标最多连点 2 次;
|
||||
若换过 ≥3 处坐标画面仍毫无响应,判 fail 并把「点了哪些坐标都没反应」写进 problems(带步号与坐标);
|
||||
看不到截图时如实报 canSeeScreenshot=false 并判 fail(problems 注明「测试员看不到画面」)。
|
||||
【步数纪律】每步提示会告诉你「第 N 步 / 共 M 步预算」。你【必须】在预算用尽前给出 verdict,绝不要把步数耗光:
|
||||
玩通到有结果就判 pass、确认玩不通(换过 ≥3 处坐标仍无响应)就判 fail;越接近预算上限越要尽快收敛给裁决。`;
|
||||
|
||||
(async () => {
|
||||
fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
// 读 brief(诚实来源):优先 --brief-file(runner 写),回退生成时落盘的 amgen-<id>/evidence/brief.json。
|
||||
let brief = '';
|
||||
try {
|
||||
if (BRIEF_FILE && fs.existsSync(BRIEF_FILE)) {
|
||||
brief = fs.readFileSync(BRIEF_FILE, 'utf-8');
|
||||
} else {
|
||||
const bp = path.resolve(__dirname, '..', '..', `amgen-${GID}`, 'evidence', 'brief.json');
|
||||
if (fs.existsSync(bp)) brief = JSON.parse(fs.readFileSync(bp, 'utf-8')).brief || '';
|
||||
}
|
||||
} catch (_) { brief = ''; }
|
||||
|
||||
// 连页面(DPR=1 省 token,agent 看 390×844 原比例)。清 localStorage/sessionStorage + 注入 seed:
|
||||
// 经 addScriptToEvaluateOnNewDocument 在文档创建时最先执行(早于游戏脚本),保证两掷从干净存档起、
|
||||
// seed 可复现(禁隐式时钟种子)——§3 第7件二掷独立性的地基。
|
||||
const created = await httpJson('PUT', `${CDP}/json/new?${encodeURI('about:blank')}`);
|
||||
const ws = await new Promise((res, rej) => { const s = new WebSocket(created.webSocketDebuggerUrl, { maxPayload: 256 * 1024 * 1024 }); s.on('open', () => res(s)); s.on('error', rej); });
|
||||
const cdp = new CdpSession(ws, created.id);
|
||||
await cdp.send('Page.enable'); await cdp.send('Runtime.enable');
|
||||
await cdp.send('Emulation.setDeviceMetricsOverride', { width: 390, height: 844, deviceScaleFactor: 1, mobile: true });
|
||||
try { await cdp.send('Emulation.setTouchEmulationEnabled', { enabled: true, maxTouchPoints: 1 }); } catch (_) {}
|
||||
await cdp.send('Page.addScriptToEvaluateOnNewDocument', {
|
||||
source: `try{localStorage.clear();sessionStorage.clear();}catch(e){} window.__playtestSeed=${Number.isFinite(SEED) ? SEED : 0};`,
|
||||
});
|
||||
const navStart = Date.now(); // firstPlay 三字段的相对时间基准
|
||||
const nav = await cdp.send('Page.navigate', { url: `${BASE}/games/_wg1-gen/${GID}/index.html` });
|
||||
// navigate 级失败 fail-fast(2026-07-10 考卷实锤:静态服务端口撞 Chrome unsafe 清单时页面被
|
||||
// net::ERR_UNSAFE_PORT 拒——curl 就绪检查照过、__genBooted 永不置位,干等 30s 只会把根因掩成
|
||||
// 假 boot-timeout)。navigate 返回 errorText 即立即带因报错,退出码 2(runner 归 degraded 待人工)。
|
||||
if (nav && nav.errorText) {
|
||||
const out = { gid: GID, model: MODEL, seed: SEED, roll: ROLL, pass: false, canSee: null,
|
||||
problems: [`页面导航失败:${nav.errorText}(runner 环境问题,非游戏缺陷——如端口撞 Chrome unsafe 清单)`],
|
||||
feedback: '', summary: 'nav-error', runnerError: true,
|
||||
steps: 0, tokens: { in: 0, out: 0 }, firstPlay: { playableAtMs: null, firstFeedbackMs: null, loopClosed: false } };
|
||||
fs.writeFileSync(path.join(OUT_DIR, 'transcript.json'), JSON.stringify({ out, transcript: [] }, null, 1));
|
||||
console.log(JSON.stringify(out)); process.exit(2);
|
||||
}
|
||||
const deadline = Date.now() + 30000; await delay(800);
|
||||
for (;;) {
|
||||
const st = await cdp.evaluate('({ b: !!window.__genBooted, e: !!window.__gameHostEngineInitFired, err: (window.__genBootError||window.__gameHostBootError||null) })');
|
||||
if (st && st.err) {
|
||||
// A 门职责,测试员不接:boot 死掉直接如实报,退出码 2(编排器据 floor A 门也会拦)。
|
||||
const out = { gid: GID, model: MODEL, seed: SEED, roll: ROLL, pass: false, canSee: null,
|
||||
problems: [`装载失败:${st.err}`], feedback: '', summary: 'boot-dead', bootDead: true,
|
||||
steps: 0, tokens: { in: 0, out: 0 }, firstPlay: { playableAtMs: null, firstFeedbackMs: null, loopClosed: false } };
|
||||
fs.writeFileSync(path.join(OUT_DIR, 'transcript.json'), JSON.stringify({ out, transcript: [] }, null, 1));
|
||||
console.log(JSON.stringify(out)); process.exit(2);
|
||||
}
|
||||
if (st && st.b && st.e) break;
|
||||
if (Date.now() > deadline) {
|
||||
const out = { gid: GID, model: MODEL, seed: SEED, roll: ROLL, pass: false, canSee: null,
|
||||
problems: ['装载超时(30s 未 booted)'], feedback: '', summary: 'boot-timeout', bootDead: true,
|
||||
steps: 0, tokens: { in: 0, out: 0 }, firstPlay: { playableAtMs: null, firstFeedbackMs: null, loopClosed: false } };
|
||||
fs.writeFileSync(path.join(OUT_DIR, 'transcript.json'), JSON.stringify({ out, transcript: [] }, null, 1));
|
||||
console.log(JSON.stringify(out)); process.exit(2);
|
||||
}
|
||||
await delay(300);
|
||||
}
|
||||
|
||||
// 坐标尺覆盖层(§3 第2件):细网格 50px、粗线+刻度标签 100px;pointer-events:none 不挡输入;只进截图。
|
||||
await cdp.evaluate(`(function(){
|
||||
if (document.getElementById('__testgrid')) return;
|
||||
var d = document.createElement('div');
|
||||
d.id = '__testgrid';
|
||||
d.style.cssText = 'position:fixed;inset:0;pointer-events:none;z-index:99999;' +
|
||||
'background-image:linear-gradient(rgba(255,0,90,.28) 1px,transparent 1px),linear-gradient(90deg,rgba(255,0,90,.28) 1px,transparent 1px),' +
|
||||
'linear-gradient(rgba(255,0,90,.14) 1px,transparent 1px),linear-gradient(90deg,rgba(255,0,90,.14) 1px,transparent 1px);' +
|
||||
'background-size:100px 100px,100px 100px,50px 50px,50px 50px;';
|
||||
for (var y = 100; y < 844; y += 100) {
|
||||
var t = document.createElement('span');
|
||||
t.textContent = 'y' + y;
|
||||
t.style.cssText = 'position:absolute;left:2px;top:' + (y - 14) + 'px;font:12px monospace;color:#ff005a;background:rgba(255,255,255,.75);padding:0 2px;';
|
||||
d.appendChild(t);
|
||||
}
|
||||
for (var x = 100; x < 390; x += 100) {
|
||||
var s = document.createElement('span');
|
||||
s.textContent = 'x' + x;
|
||||
s.style.cssText = 'position:absolute;top:2px;left:' + (x - 14) + 'px;font:12px monospace;color:#ff005a;background:rgba(255,255,255,.75);padding:0 2px;';
|
||||
d.appendChild(s);
|
||||
}
|
||||
document.body.appendChild(d);
|
||||
})()`);
|
||||
|
||||
const shot = async () => {
|
||||
const r = await cdp.send('Page.captureScreenshot', { format: 'jpeg', quality: 62 });
|
||||
return 'data:image/jpeg;base64,' + r.data;
|
||||
};
|
||||
const logTail = async () => cdp.evaluate('(function(){ var lg=window.__gameLog||[]; return lg.slice(-6).map(function(e){ return (e.scope||"")+"/"+(e.tag||"")+":"+String(e.msg||"").slice(0,60); }); })()');
|
||||
const logLen = async () => cdp.evaluate('(function(){ return (window.__gameLog||[]).length; })()');
|
||||
// 画面帧哈希(§5/§3 第6件的确定性信号):抓 canvas 像素算 fnv1a32,判「这一步动作后画面真变了没」。
|
||||
// 与测试员的视觉判断解耦——firstPlay 与预算进展扩展都靠这个机械信号,不靠模型自觉。
|
||||
const frameHash = async () => {
|
||||
// selector 与 play.cdp.cjs D 门同口径:LittleJS 引擎 canvas id=#game-engine(WebGL 主画布经 2d 回读);
|
||||
// 逐个回退保鲁棒——找不到就返 0(firstPlay/预算扩展降精度,绝不崩主流程)。
|
||||
for (const sel of ['#game-engine', '#game', 'canvas']) {
|
||||
try { const img = await captureImageData(cdp, sel); if (img && img.data && img.data.length) return fnv1a32(img.data) || 0; }
|
||||
catch (_) { /* 试下一个 selector */ }
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
|
||||
// 落点回显(§3 第3件):tap 后放亮黄圆环+十字(pointer-events:none),下一帧截图里 agent 能看到自己点在哪。
|
||||
const placeTapMarker = (x, y) => cdp.evaluate(`(function(){
|
||||
var m = document.getElementById('__tapmark');
|
||||
if (!m) { m = document.createElement('div'); m.id='__tapmark'; document.body.appendChild(m); }
|
||||
m.style.cssText = 'position:fixed;left:${x - 14}px;top:${y - 14}px;width:28px;height:28px;pointer-events:none;z-index:100000;' +
|
||||
'border:3px solid #ffe000;border-radius:50%;box-shadow:0 0 0 1px #000;' +
|
||||
'background:radial-gradient(circle,rgba(255,224,0,.9) 2px,transparent 3px);';
|
||||
})()`);
|
||||
|
||||
const transcript = [];
|
||||
const history = [];
|
||||
let usageIn = 0, usageOut = 0, verdict = null;
|
||||
let lastTap = null, sameTapStreak = 0, pushedBack = false;
|
||||
const tapPoints = new Set();
|
||||
let prevLogLen = await logLen(); // 上一步日志长度(逐步 delta:这一步游戏有没有 emit 事件)
|
||||
let prevHash = await frameHash();
|
||||
let firstPlayableStep = null, firstFeedbackStep = null; // firstPlay 机械提取的步索引
|
||||
const stepTimes = []; // 每步相对导航的毫秒(供 firstPlay 时间戳)
|
||||
let stepsCap = STEPS_BASE; // 预算随进展扩展:基础预算,有进展则升到 STEPS_MAX(§3 第6件)
|
||||
let lastSignal = null; // 上一步动作后的机械信号(反馈给测试员,治「失手后错误归因」)
|
||||
|
||||
for (let step = 0; step < stepsCap && !verdict; step++) {
|
||||
const tNow = Date.now() - navStart;
|
||||
stepTimes.push(tNow);
|
||||
const img = await shot();
|
||||
const lg = await logTail();
|
||||
// 防误导硬提示(§3 第4件;runner 侧确定性注入,不靠模型自觉)。
|
||||
let hint = '';
|
||||
if (sameTapStreak >= 2) hint = `\n⚠ 你已连续 ${sameTapStreak} 次点同一坐标(${lastTap});规则:立即换点(先 y±50)或 wait,不许再点原处。`;
|
||||
// 机械信号反馈(runner 侧确定性护栏,治「测试员失手后错误归因」头号风险):把上一步操作后画面/日志的
|
||||
// 客观变化告诉测试员——它常在自己坐标点偏时误判成「游戏没反应/有 bug」,而机械测得的画面像素/日志变化
|
||||
// 是客观事实。给它这个事实,让它把「落点偏了」与「游戏真没响应」分开,不冤枉在响应的游戏。
|
||||
if (lastSignal && lastSignal.isInput) {
|
||||
if (lastSignal.screenChanged || lastSignal.logGrew) {
|
||||
hint += `\n⚙ 客观信号(runner 机械测得,非你的视觉判断):上一步操作后画面像素${lastSignal.screenChanged ? '已变化' : '未变化'}、游戏日志${lastSignal.logGrew ? `新增 ${lastSignal.logDelta} 条` : '无新增'}——游戏【确实响应了】你的操作,别误判成「没反应/判错」;你想点的目标若没动,多半是落点偏了(看黄圆环校正坐标),不是游戏坏了。`;
|
||||
} else {
|
||||
hint += `\n⚙ 客观信号(runner 机械测得):上一步操作后画面像素与游戏日志【均无变化】——可能落点点空了(看黄圆环校正坐标),或此处确实无可交互元素;换个坐标再试。`;
|
||||
}
|
||||
}
|
||||
const remain = stepsCap - step - 1; // 剩余步数(临近上限时确定性催收裁决,不靠模型自觉)
|
||||
if (remain <= 3) hint += `\n⏳ 步数预算将尽:本步之后只剩 ${remain} 步,请尽快收敛给出 verdict(玩通判 pass / 玩不通判 fail)。`;
|
||||
const userParts = [
|
||||
{ type: 'text', text: `brief:${brief}\n已执行历史:${history.length ? history.join(' → ') : '(第一步)'}\n运行日志尾:${JSON.stringify(lg)}${hint}\n当前截图(第 ${step + 1} 步 / 共 ${stepsCap} 步预算;黄圆环=你上一步落点):` },
|
||||
{ type: 'image_url', image_url: { url: img } },
|
||||
{ type: 'text', text: '输出下一个动作 JSON(或 verdict):' },
|
||||
];
|
||||
// 每步截图落盘(验收硬证据链)。
|
||||
try { fs.writeFileSync(path.join(OUT_DIR, `step-${String(step + 1).padStart(2, '0')}.jpg`), Buffer.from(img.split(',')[1], 'base64')); } catch (_) {}
|
||||
|
||||
let resp;
|
||||
try { resp = await chat([{ role: 'system', content: SYSTEM }, { role: 'user', content: userParts }], 3000); }
|
||||
catch (e) { history.push(`step${step}:LLM 调用失败(${String(e.message).slice(0, 40)})`); transcript.push({ step, tMs: tNow, err: String(e.message).slice(0, 120) }); await delay(500); continue; }
|
||||
usageIn += resp.usage.prompt_tokens || 0; usageOut += resp.usage.completion_tokens || 0;
|
||||
let act;
|
||||
try { act = JSON.parse(resp.text.replace(/^```json?\s*|```\s*$/g, '').trim()); }
|
||||
catch (_) { history.push(`step${step}:输出非JSON(截断记录)`); transcript.push({ step, tMs: tNow, raw: resp.text.slice(0, 300) }); continue; }
|
||||
|
||||
if (act.act === 'verdict') {
|
||||
// 反早退(§3 第5件):fail 裁决若探索不足(独立落点 <3)且非图像通道故障,驳回一次逼继续测。
|
||||
if (act.pass === false && act.canSeeScreenshot !== false && tapPoints.size < 3 && !pushedBack) {
|
||||
pushedBack = true;
|
||||
history.push(`裁决被驳回(仅试过 ${tapPoints.size} 处落点,规则要求 ≥3 处;继续测试)`);
|
||||
transcript.push({ step, tMs: tNow, act, pushedBack: true });
|
||||
continue;
|
||||
}
|
||||
transcript.push({ step, tMs: tNow, act });
|
||||
verdict = act; break;
|
||||
}
|
||||
|
||||
if (act.act === 'tap' && typeof act.x === 'number') {
|
||||
await cdp.send('Input.dispatchTouchEvent', { type: 'touchStart', touchPoints: [{ x: act.x, y: act.y }] });
|
||||
await cdp.send('Input.dispatchTouchEvent', { type: 'touchEnd', touchPoints: [] });
|
||||
await placeTapMarker(act.x, act.y);
|
||||
const key = `${Math.round(act.x)},${Math.round(act.y)}`;
|
||||
sameTapStreak = (lastTap === key) ? sameTapStreak + 1 : 1;
|
||||
lastTap = key;
|
||||
tapPoints.add(key);
|
||||
history.push(`tap(${key})[${act.why || ''}]`);
|
||||
await delay(450);
|
||||
} else if (act.act === 'key' && act.key) {
|
||||
await cdp.send('Input.dispatchKeyEvent', { type: 'keyDown', code: act.key, key: act.key });
|
||||
await cdp.send('Input.dispatchKeyEvent', { type: 'keyUp', code: act.key, key: act.key });
|
||||
history.push(`key(${act.key})`);
|
||||
sameTapStreak = 0;
|
||||
await delay(300);
|
||||
} else if (act.act === 'wait') {
|
||||
history.push(`wait(${act.ms || 500})`);
|
||||
sameTapStreak = 0;
|
||||
await delay(Math.min(act.ms || 500, 3000));
|
||||
} else {
|
||||
history.push(`step${step}:未知动作`);
|
||||
}
|
||||
|
||||
// 动作后的机械信号:画面变化 + 逐步日志推进(§5 firstPlay + §3 第6件预算进展 + 机械信号反馈护栏)。
|
||||
const curHash = await frameHash();
|
||||
const curLogLen = await logLen();
|
||||
const screenChanged = curHash !== prevHash;
|
||||
const logDelta = curLogLen - prevLogLen; // 这一步游戏 emit 的事件数(逐步,非累计)
|
||||
const logGrew = logDelta > 0;
|
||||
const isInput = act.act === 'tap' || act.act === 'key';
|
||||
if (isInput && screenChanged && firstPlayableStep === null) firstPlayableStep = step; // 首次有效交互
|
||||
if (logGrew && firstFeedbackStep === null) firstFeedbackStep = step; // 首个可感知反馈
|
||||
// 有进展证据(画面变化或日志推进)→ 预算从基础扩到上限(§3 第6件):难品类节拍长,基础 14 装不下四工序。
|
||||
if ((screenChanged || logGrew) && stepsCap < STEPS_MAX) stepsCap = STEPS_MAX;
|
||||
lastSignal = { isInput, screenChanged, logGrew, logDelta }; // 下一步反馈给测试员(治错误归因)
|
||||
transcript.push({ step, tMs: tNow, act, log: lg, screenChanged, logGrew, logDelta, tapPoints: tapPoints.size });
|
||||
prevHash = curHash;
|
||||
prevLogLen = curLogLen;
|
||||
}
|
||||
|
||||
if (!verdict) verdict = { act: 'verdict', pass: false, canSeeScreenshot: true,
|
||||
problems: [`步数预算内(${stepsCap} 步)未给出裁决`], feedback: '', summary: 'no-verdict', noVerdict: true };
|
||||
|
||||
// firstPlay 三字段机械提取(§5:首局门语义并入测试员转写;时间戳取该步相对导航毫秒):
|
||||
// playableAtMs=首次有效交互(输入后画面真变)、firstFeedbackMs=首个可感知反馈(日志推进)、
|
||||
// loopClosed=测试员判 pass(玩通到有结果)即闭环达成。三值机械可判,不含模型主观。
|
||||
const firstPlay = {
|
||||
playableAtMs: firstPlayableStep !== null ? (stepTimes[firstPlayableStep] ?? null) : null,
|
||||
firstFeedbackMs: firstFeedbackStep !== null ? (stepTimes[firstFeedbackStep] ?? null) : null,
|
||||
loopClosed: verdict.pass === true,
|
||||
};
|
||||
|
||||
// 图像通道自检(§3 第8件):测试员自报看不到截图 → 退出码 3,归因测试员图像通道故障、非游戏缺陷。
|
||||
const canSee = verdict.canSeeScreenshot;
|
||||
const imageBlind = canSee === false;
|
||||
|
||||
const out = { gid: GID, model: MODEL, seed: SEED, roll: ROLL, pass: verdict.pass === true,
|
||||
canSee, imageBlind, problems: verdict.problems || [], feedback: verdict.feedback || '',
|
||||
summary: verdict.summary || '', steps: transcript.length, stepsCap,
|
||||
tapPoints: tapPoints.size, firstPlay, tokens: { in: usageIn, out: usageOut } };
|
||||
fs.writeFileSync(path.join(OUT_DIR, 'transcript.json'), JSON.stringify({ out, transcript }, null, 1));
|
||||
console.log(JSON.stringify(out));
|
||||
try { ws.close(); } catch (_) {}
|
||||
process.exit(imageBlind ? 3 : 0);
|
||||
})().catch((e) => {
|
||||
console.error('PLAYTEST-ERR:', e && e.message);
|
||||
try {
|
||||
fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
const out = { gid: GID, model: MODEL, seed: SEED, roll: ROLL, pass: false, canSee: null,
|
||||
problems: [`测试员运行错误:${e && e.message}`], feedback: '', summary: 'runner-error', runnerError: true,
|
||||
steps: 0, tokens: { in: 0, out: 0 }, firstPlay: { playableAtMs: null, firstFeedbackMs: null, loopClosed: false } };
|
||||
fs.writeFileSync(path.join(OUT_DIR, 'transcript.json'), JSON.stringify({ out, transcript: [] }, null, 1));
|
||||
console.log(JSON.stringify(out));
|
||||
} catch (_) {}
|
||||
process.exit(2);
|
||||
});
|
||||
70
game-runtime/games/_wg1-gen/_shared/serve-and-playtest.sh
Executable file
70
game-runtime/games/_wg1-gen/_shared/serve-and-playtest.sh
Executable file
@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env bash
|
||||
# serve-and-playtest.sh —— 便宜档验收 v2「测试 agent 真玩」一键编排(W-AXIS-V2 波1)。
|
||||
# 与 serve-and-play.sh 同一套起服/Chrome/净场/就绪等待编排(唯一区别:末步换调 playtest.cdp.cjs 而非
|
||||
# play.cdp.cjs),让测试员回路复用久经考验的起服链。cwd 必须 = game-runtime。
|
||||
# 用法:bash games/_wg1-gen/_shared/serve-and-playtest.sh <gameId> [port] [cdpPort] [-- 透传给 playtest.cdp.cjs 的额外参数...]
|
||||
# 例:... hard-puzzle-r1 4998 9331 -- --model=MiniMax-M3 --roll=1 --seed=42 --brief-file=/tmp/b.txt --evidence-dir=/abs/evidence
|
||||
# 依赖:NEWAPI_KEY 由 runner(cheap_verify.run_playtest)从凭据档注入环境;playtest.cdp.cjs 用 http 模块直连不走代理。
|
||||
set -u
|
||||
GAME_ID="${1:?用法: serve-and-playtest.sh <gameId> [port] [cdpPort] [-- 额外参数...]}"
|
||||
PORT="${2:-4998}"
|
||||
CDP_PORT="${3:-9331}"
|
||||
shift $(( $# < 3 ? $# : 3 ))
|
||||
[ "${1:-}" = "--" ] && shift # 吞掉分隔符,其余原样透传给 playtest.cdp.cjs
|
||||
|
||||
# Chrome 路径探测(跨平台;与 serve-and-play.sh 同口径:CHROME_BIN 覆盖 > Linux > macOS)。
|
||||
CHROME=""
|
||||
if [ -n "${CHROME_BIN:-}" ] && [ -x "${CHROME_BIN}" ]; then
|
||||
CHROME="${CHROME_BIN}"
|
||||
else
|
||||
for p in "/usr/bin/google-chrome" "/usr/bin/google-chrome-stable" \
|
||||
"/usr/bin/chromium" "/usr/bin/chromium-browser" "/opt/google/chrome/chrome" \
|
||||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \
|
||||
"/Applications/Chromium.app/Contents/MacOS/Chromium" \
|
||||
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary"; do
|
||||
[ -x "$p" ] && CHROME="$p" && break
|
||||
done
|
||||
[ -z "$CHROME" ] && CHROME="$(command -v google-chrome chromium chromium-browser 2>/dev/null | head -1)"
|
||||
fi
|
||||
[ -z "$CHROME" ] && { echo "❌ 未找到 Chrome(探测 /usr/bin、/Applications、PATH、CHROME_BIN 均无)"; exit 3; }
|
||||
|
||||
USER_DATA_DIR="$(mktemp -d -t wg1-playtest-XXXX)"
|
||||
SERVE_PID=""; CHROME_PID=""
|
||||
cleanup() {
|
||||
[ -n "$CHROME_PID" ] && kill "$CHROME_PID" 2>/dev/null
|
||||
[ -n "$SERVE_PID" ] && kill "$SERVE_PID" 2>/dev/null
|
||||
rm -rf "$USER_DATA_DIR" 2>/dev/null
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# 净场:杀掉占用端口的旧进程(保守,仅本脚本端口)。
|
||||
lsof -ti tcp:"$PORT" 2>/dev/null | xargs kill 2>/dev/null || true
|
||||
lsof -ti tcp:"$CDP_PORT" 2>/dev/null | xargs kill 2>/dev/null || true
|
||||
sleep 0.3
|
||||
|
||||
# 1) 静态服务器:serve game-runtime 根(cwd)。
|
||||
node test/harness/static-serve.cjs . "$PORT" >/tmp/wg1-playtest-serve-$PORT.log 2>&1 &
|
||||
SERVE_PID=$!
|
||||
|
||||
# 2) Chrome headless(CDP 必备旗标:--remote-allow-origins=* + --autoplay-policy=...)。
|
||||
"$CHROME" --headless=new --no-sandbox --no-zygote --disable-gpu \
|
||||
--remote-debugging-port="$CDP_PORT" --remote-allow-origins='*' \
|
||||
--autoplay-policy=no-user-gesture-required \
|
||||
--user-data-dir="$USER_DATA_DIR" about:blank >/tmp/wg1-playtest-chrome-$PORT.log 2>&1 &
|
||||
CHROME_PID=$!
|
||||
|
||||
# 3) 就绪等待。
|
||||
ok_serve=0; ok_cdp=0
|
||||
for i in $(seq 1 20); do
|
||||
curl -sf "http://localhost:$PORT/games/_wg1-gen/$GAME_ID/index.html" -o /dev/null 2>/dev/null && ok_serve=1
|
||||
curl -sf "http://localhost:$CDP_PORT/json/version" -o /dev/null 2>/dev/null && ok_cdp=1
|
||||
[ "$ok_serve" = 1 ] && [ "$ok_cdp" = 1 ] && break
|
||||
sleep 0.5
|
||||
done
|
||||
[ "$ok_serve" != 1 ] && { echo "❌ static-serve 未就绪(见 /tmp/wg1-playtest-serve-$PORT.log)"; cat /tmp/wg1-playtest-serve-$PORT.log; exit 4; }
|
||||
[ "$ok_cdp" != 1 ] && { echo "❌ Chrome CDP 未就绪(见 /tmp/wg1-playtest-chrome-$PORT.log)"; tail -5 /tmp/wg1-playtest-chrome-$PORT.log; exit 5; }
|
||||
|
||||
# 4) 测试员真玩驱动(其余参数由 runner 透传:--model/--roll/--seed/--brief-file/--evidence-dir/--steps-*)。
|
||||
node games/_wg1-gen/_shared/playtest.cdp.cjs "$GAME_ID" --base="http://localhost:$PORT" --cdp="http://localhost:$CDP_PORT" "$@"
|
||||
PLAYTEST_RC=$?
|
||||
exit "$PLAYTEST_RC"
|
||||
@ -107,6 +107,23 @@ judge:
|
||||
max_frames: 6 # 最多喂几帧证据截图(首帧 + 局中连拍 + 局末,控 token/成本;超出保头尾均匀采样)
|
||||
cost_target_rmb: 0.3 # 单局判定成本目标(¥;仅记账对照/告警,不阻断——实测约 ¥0.10/4-6 帧)
|
||||
|
||||
# ── acceptance · 便宜档验收 v2 版本开关(消费方:cheap-worker/cheap_verify.run_acceptance;W-AXIS-V2 波1)──
|
||||
# 三态(§2):v2=测试员真玩阻断放行;shadow=测试员照跑照落证据但 accepted 仍取旧口径(新旧并跑对照,批次账落对照表);
|
||||
# v1=W-AXIS 旧口径(driven 九门预筛 + 判定器读驱动器证据)。波1 灰度窗口默认 shadow——安全侧:测试员未稳时不误拒
|
||||
# 生产;验收真跑/切换时改此值为 v2(或临时 env TIER2_GEN__ACCEPTANCE__MODE=v2)。
|
||||
acceptance:
|
||||
mode: shadow # v2 | shadow | v1
|
||||
|
||||
# ── playtest · 测试 agent 真玩回路旋钮(消费方:cheap-worker/cheap_verify.run_playtest;测试员模型走 judge.model)──
|
||||
# §3 八件里可配的两组:步数预算(base 随进展扩到 max)与二掷/超时/成本上限;其余护栏(坐标尺/落点回显/
|
||||
# 同点硬提示/反早退/图像通道自检)是 runner 侧确定性代码、不留旋钮。
|
||||
playtest:
|
||||
steps_base: 14 # 基础步数预算(§3 第6件)
|
||||
steps_max: 24 # 有进展证据(画面变化/日志推进)时自动扩到的步数上限
|
||||
second_roll: true # fail 二掷确认(§3 第7件):roll-1 判 fail 且四门地板全绿 → 重开一局二掷,两掷同 fail 才落 fail
|
||||
timeout_s: 600 # 单掷子进程墙钟超时(秒;一轮 8~24 步 × M3 每步 10~40s ≈ 2~10 分钟)
|
||||
cost_cap_rmb: 1.5 # 单局验收成本上限(¥;含二掷,单轮设计上限 ¥0.5,对照旧链路契约困死单局 ¥10+ 净赚)
|
||||
|
||||
# ── archetype · business-sim 品类 driver 参数(消费方:worker/archetypes.py)─────────────────
|
||||
# 注:这里只放 driver 真玩规格里的【可调数值】(怎么玩);品类的【结构】(三系统/数据表 schema)
|
||||
# 不在配置层,仍由 fixture 与品类注册表定。改这些 = 调 business-sim driver 怎么把游戏玩到终态。
|
||||
|
||||
@ -459,13 +459,23 @@ def _cheap_verdict_feedback(v: dict, *, game_id=None, staged_dir=None) -> str:
|
||||
return _render_cheap_feedback(obj)
|
||||
|
||||
|
||||
# W-AXIS-V2 波1:便宜档四门投影(契约无关 A_boot/B_uncaught/C_frame/D_render 求 AND)——续修收敛判据的地板,
|
||||
# 与 cheap_verify.project_floor 同口径(此处内联常量,避免 tier2 worker 跨目录 import cheap-worker)。
|
||||
_CHEAP_FLOOR_GATES = ("A_boot", "B_uncaught", "C_frame", "D_render")
|
||||
|
||||
|
||||
def _floor_pass(guards: dict) -> bool:
|
||||
"""四门全绿且逐门真实存在 → True;任一门缺失/挂 → False(fail-closed)。"""
|
||||
return all(isinstance(guards.get(g), dict) and guards[g].get("pass") is True for g in _CHEAP_FLOOR_GATES)
|
||||
|
||||
|
||||
def judge_cheap_verdict(play_result: dict, *, game_id=None, staged_dir=None) -> GateJudgment:
|
||||
"""把便宜档 cheap_run.play 的九门 verdict({pass, guards{门:{pass,...}}})归一成 GateJudgment(与 judge_tier2_verdict 并列)。
|
||||
|
||||
放行判据 = 顶层 pass **且 guards 非空**(便宜档 verdict.pass 已是九门 AND 后的总放行,等价 gen.mjs done 门;判据不放松)。
|
||||
**I3 防假绿**:只看 `bool(v.get("pass"))` 会把「play 没跑出逐门却被强标 pass」当绿——故要求 `guards` 非空
|
||||
(九门真跑必产逐门 map);空 guards + pass=True → passed=False、feedback 落 harness 说明。
|
||||
failed_gates 复用 _verdict_brief 口径(guards 里 pass is False 的门名;H_progress 嵌套已在 g.pass 层判)。
|
||||
放行判据(W-AXIS-V2 波1)= **四门投影**(契约无关 A/B/C/D 求 AND)**且 guards 非空**,不再要求九门 verdict.pass——
|
||||
E/G/H/I/F 依赖取证契约/驱动器,随契约退役会坍缩;续修只需游戏「起得来、没崩、有帧、有渲染」四门地板即算未见明显死。
|
||||
**I3 防假绿**:空 guards 无从证明门真跑过,不放行(四门真跑必产逐门 map)。
|
||||
failed_gates 仍列 guards 里 pass is False 的门名(诊断/反馈用,含降观测门;四门过即放行、不进 feedback 分支)。
|
||||
|
||||
C6 接线(W-S1 单③):feedback 由 C6 结构化对象渲染(build_cheap_feedback_obj → _render_cheap_feedback,
|
||||
单一来源防拼串丢字段);带 phaseNow(latch 两失败支都读)、driverType(判卷驱动器族)、game-log 摘要。
|
||||
@ -475,8 +485,8 @@ def judge_cheap_verdict(play_result: dict, *, game_id=None, staged_dir=None) ->
|
||||
"""
|
||||
v = play_result or {}
|
||||
guards = v.get("guards") or {}
|
||||
# 放行必须「顶层 pass 且有逐门 guards」:空 guards 无从证明九门真跑过,不放行(防空 verdict 假绿)。
|
||||
passed = bool(v.get("pass")) and bool(guards)
|
||||
# 放行 = 四门投影(A/B/C/D)全绿 且 guards 非空(空 guards 无从证明门真跑过,fail-closed 不放行)。
|
||||
passed = _floor_pass(guards) and bool(guards)
|
||||
failed_gates = [k for k, g in guards.items() if isinstance(g, dict) and g.get("pass") is False]
|
||||
if passed:
|
||||
# 全绿:feedback 只作日志/trace 用(GateJudgment 契约:passed=True 时不回喂),给准确描述。
|
||||
|
||||
@ -146,6 +146,23 @@ _BUILTIN_DEFAULTS: dict[str, dict[str, Any]] = {
|
||||
"max_frames": 6, # 最多喂几帧证据截图(首帧+局中连拍+局末)
|
||||
"cost_target_rmb": 0.3, # 单局判定成本目标(¥;仅记账对照,不阻断)
|
||||
},
|
||||
# ── acceptance:便宜档验收 v2 版本开关(cheap-worker/cheap_verify.run_acceptance;W-AXIS-V2 波1)──
|
||||
# 三态开关(§2):v2=测试员真玩阻断放行;shadow=测试员照跑照落证据但 accepted 取旧口径(新旧并跑对照);
|
||||
# v1=W-AXIS 旧口径(driven 九门预筛 + 判定器读驱动器证据)。波1 灰度窗口默认 shadow——安全侧:
|
||||
# 测试员未稳时不误拒生产,验收真跑时显式切 v2(env TIER2_GEN__ACCEPTANCE__MODE=v2 或改 YAML)。
|
||||
"acceptance": {
|
||||
"mode": "shadow", # v2 | shadow | v1
|
||||
},
|
||||
# ── playtest:测试 agent 真玩回路旋钮(cheap-worker/cheap_verify.run_playtest;测试员模型走 judge.model)──
|
||||
# §3 八件里可配的两组:预算(base→max 随进展扩)与二掷/超时/成本上限;其余(坐标尺/落点回显/硬提示/
|
||||
# 反早退/图像自检)是 runner 侧确定性护栏、不留配置。
|
||||
"playtest": {
|
||||
"steps_base": 14, # 基础步数预算(§3 第6件)
|
||||
"steps_max": 24, # 有进展证据(画面变化/日志推进)时自动扩到的上限
|
||||
"second_roll": True, # fail 二掷确认(§3 第7件:roll-1 fail 且四门绿 → 二掷,两掷同 fail 才 fail)
|
||||
"timeout_s": 600.0, # 单掷子进程墙钟超时(一轮 8~24 步 × M3 每步 10~40s ≈ 2~10 分钟)
|
||||
"cost_cap_rmb": 1.5, # 单局验收成本上限(¥;含二掷,单轮设计上限 ¥0.5)
|
||||
},
|
||||
# 注:business-sim driver 参数(steps/stepMs/winThreshold/bankruptSteps/streakLose)不在本配置层——
|
||||
# 曾登记过一个 "archetype" 区声明这五个键,但生成路(archetypes._business_sim_gate_spec /
|
||||
# run.DEFAULT_BUSINESS_SIM_PLAY_SPEC)从不读它、真值一直硬编码在那两处 = 死旋钮(改配置无效、误导)。
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user