feat(cheap-worker): 同游戏双驱动 play 编排 + auto-vs-golden 批跑(M2 U2)
run_pair_dual_sync:同一份 staged src/ 上先自动 spec(ensure_play_spec)、后金标 (inject_golden)各 play 一次,取两组逐门 verdict——隔离生成方差、只暴露驱动器差。 编排序 gen-only→smoke→断言 staged 无残留 spec→ensure→play(auto)→inject→play (golden);auto_gates 在金标注入前捕获。run_batch 端口池+线程前台有界并发(≤15 夹取、复用 bake_off._clamp_conc)、报告 auto-vs-golden-<idx>.json 不覆盖。 mock 编排单测(调用序 + auto 先于 golden + no-spec 守卫)20/20 全绿。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
lili
parent
dd0d1cc224
commit
8038109ce8
@ -12,10 +12,21 @@ tap-targets 不成立(M1 已把自动 spec 断言加厚到与金标逐字段一
|
||||
U1 逐门 delta 判据(本段,纯逻辑可单测);U2 同游戏双驱动 play 真跑;U3 三条齐退役授权判定。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent)) # → cheap-worker/
|
||||
import _bootstrap # noqa: E402,F401 跨包 sys.path + 代理旁路 + key 兜底(须在 import cheap_studio 前)
|
||||
import bake_off # noqa: E402 复用 _clamp_conc 并发夹取口径
|
||||
import cheap_run # noqa: E402 smoke / ensure_play_spec / play / wg1_game_dir / game_dir
|
||||
import cheap_studio # noqa: E402 run_studio(run_gates=False)gen-only
|
||||
import compare_node # noqa: E402 inject_golden / golden_spec_path / gates_from_verdict / src_shape / 端口池
|
||||
|
||||
_RESULTS_DIR = Path(__file__).resolve().parent / "results"
|
||||
_DEFAULT_GENRES = ["click-score", "whack-mole", "shop-serve"]
|
||||
_clamp_conc = bake_off._clamp_conc # 并发夹取 [1,15] 复用 M1 达标门口径(R4 上限纪律)
|
||||
|
||||
# 九门里唯一对驱动器敏感的三门(未驱动时 E_live/H_progress 降 advisory、G_input 无输入 skip):
|
||||
# 「这游戏到底能不能玩」的判据,自动低于金标即驱动更差,容差 0。
|
||||
@ -168,3 +179,123 @@ def retire_authorization(m1_path, compare_path, avg_path, required_genres=None)
|
||||
"note": "Node 退役授权三条齐(M1 达标 ∧ 002 对照等价 ∧ auto-vs-golden 不退化);"
|
||||
"authorized=授权切默认路由的条件齐备,真切路由/D12 扣退在 M3。本机口径,生产复验在 M3。",
|
||||
}
|
||||
|
||||
|
||||
# ───────────────────────── U2 同游戏双驱动 play 编排(真跑)─────────────────────────
|
||||
|
||||
def run_pair_dual_sync(genre: dict, brief: str, gid: str, port: int = 4320, cdp: int = 9222) -> dict:
|
||||
"""一款 auto-vs-golden:同一份 staged src/ 上先自动 spec、后金标各 play 一次,取两组逐门 verdict。
|
||||
|
||||
同步整对(由 run_batch 经 to_thread 并发调度,每对一线程独立端口);隔离生成方差(同款产物 play 两次,
|
||||
只换驱动器)。编排序:gen-only → smoke 取 state → 断言 staged 无残留 spec → ensure_play_spec(自动 spec)
|
||||
→ play 取 auto_gates → inject_golden 强制覆写金标 → play 取 golden_gates。
|
||||
"""
|
||||
# ① gen-only:run_gates=False 跳过内部 ensure_play_spec + play(cheap_studio.py:153)。
|
||||
asyncio.run(cheap_studio.run_studio(gid, brief, run_gates=False, port=port, cdp_port=cdp))
|
||||
# ② smoke → state(供 ensure_play_spec 据 _forensicsView 形态推 driver)。
|
||||
sm = cheap_run.smoke(gid, port=port, cdp_port=cdp)
|
||||
state = (sm or {}).get("state")
|
||||
# ③ 断言 staged 无残留 play-spec.json(ensure_play_spec 是「已存在不覆盖」语义 cheap_run.py:253,
|
||||
# 残留旧 spec 会被静默复用、污染 auto_gates 且无报错;把隐式依赖变显式前置)。
|
||||
spec_path = cheap_run.wg1_game_dir(gid) / "play-spec.json"
|
||||
if spec_path.exists():
|
||||
raise RuntimeError(f"staged 残留 play-spec.json,auto 路会静默复用旧 spec:{spec_path}")
|
||||
cheap_run.ensure_play_spec(gid, state)
|
||||
auto_play = cheap_run.play(gid, port=port, cdp_port=cdp)
|
||||
auto_gates = compare_node.gates_from_verdict((auto_play or {}).get("verdict"))
|
||||
# ④ inject 金标强制覆写 → play(此时 auto_gates 已捕获、不被覆盖)。
|
||||
compare_node.inject_golden(gid, compare_node.golden_spec_path(genre))
|
||||
golden_play = cheap_run.play(gid, port=port, cdp_port=cdp)
|
||||
golden_gates = compare_node.gates_from_verdict((golden_play or {}).get("verdict"))
|
||||
real = compare_node.src_shape(cheap_run.game_dir(gid)).get("realSrcMultifile", False)
|
||||
return {"gid": gid, "autoGates": auto_gates, "goldenGates": golden_gates, "realSrc": real}
|
||||
|
||||
|
||||
def _next_report_path() -> Path:
|
||||
"""报告分批次文件、不覆盖历史。"""
|
||||
_RESULTS_DIR.mkdir(exist_ok=True)
|
||||
idx = compare_node._next_index([p.stem for p in _RESULTS_DIR.glob("auto-vs-golden-*.json")])
|
||||
return _RESULTS_DIR / f"auto-vs-golden-{idx}.json"
|
||||
|
||||
|
||||
async def run_batch(genre_keys: list, n: int, conc: int = 1, offset: int = 0,
|
||||
base_port: int = 4320, base_cdp: int = 9222) -> dict:
|
||||
"""三品类 × n 同游戏双驱动批跑:逐款 run_pair_dual_sync,按品类聚合逐门 delta 判据。
|
||||
|
||||
前台进程内有界并发:端口池(每槽独立 port/cdp,复用 compare_node._port_pool)+ 信号量限并发(≤conc),
|
||||
每款经 to_thread 独立线程跑。禁后台子代理 / monitor / 自我唤醒重试(R4)。
|
||||
"""
|
||||
conc = _clamp_conc(conc)
|
||||
n = max(1, int(n))
|
||||
pool = asyncio.Queue()
|
||||
for pp in compare_node._port_pool(conc, base_port, base_cdp):
|
||||
pool.put_nowait(pp)
|
||||
|
||||
valid, genre_runs = [], {}
|
||||
for key in genre_keys:
|
||||
genre = compare_node._GENRE_BY_KEY.get(key)
|
||||
if genre is None:
|
||||
print(f"[skip] 未知品类 {key}", file=sys.stderr)
|
||||
continue
|
||||
brief = compare_node.load_brief(genre)
|
||||
if not brief:
|
||||
print(f"[skip] {key} 无 brief(base {genre['base']} run-summary 缺失)", file=sys.stderr)
|
||||
continue
|
||||
valid.append((key, genre, brief))
|
||||
genre_runs[key] = []
|
||||
|
||||
async def one(key, genre, brief, k):
|
||||
port, cdp = await pool.get()
|
||||
gid = f"avg-{key}-{offset + k}"
|
||||
try:
|
||||
print(f"[{key} {k + 1}/{n}] port={port}/{cdp} 双驱动(自动 spec vs 金标)…", file=sys.stderr)
|
||||
r = await asyncio.to_thread(run_pair_dual_sync, genre, brief, gid, port, cdp)
|
||||
genre_runs[key].append(r)
|
||||
ag, gg = r["autoGates"], r["goldenGates"]
|
||||
print(f"[{key} {k + 1}/{n}] auto={compare_node._pass_rate(ag)} golden={compare_node._pass_rate(gg)}",
|
||||
file=sys.stderr)
|
||||
except Exception as e: # noqa: BLE001 单款失败不拖垮整批
|
||||
print(f"[{key} {k + 1}/{n}] 异常:{type(e).__name__}: {e}", file=sys.stderr)
|
||||
finally:
|
||||
pool.put_nowait((port, cdp))
|
||||
|
||||
tasks = [one(key, genre, brief, k) for (key, genre, brief) in valid for k in range(n)]
|
||||
print(f"[auto-vs-golden] {len(valid)} 品类 × n={n} = {len(tasks)} 款,conc={conc}", file=sys.stderr)
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
report = aggregate_delta(genre_runs, [k for (k, _, _) in valid])
|
||||
report["runs"] = genre_runs
|
||||
out = _next_report_path()
|
||||
out.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"[report] → {out}", file=sys.stderr)
|
||||
_print_report(report)
|
||||
return report
|
||||
|
||||
|
||||
def _print_report(report: dict) -> None:
|
||||
print("\n=== auto-vs-golden delta 门(同款双驱动 · 关键门容差 0 · 零 LLM)===", file=sys.stderr)
|
||||
for k, j in report["perGenre"].items():
|
||||
extra = ""
|
||||
if j.get("regressedGates"):
|
||||
extra = f" 退化门={j['regressedGates']}"
|
||||
elif j.get("inconclusiveGates"):
|
||||
extra = f" 双低门={j['inconclusiveGates']}"
|
||||
print(f"[{k}] {j['status']} (n={j['total']}){extra}", file=sys.stderr)
|
||||
flag = "不退化 ✅" if report["overallMeets"] else "退化/未达 ❌"
|
||||
print(f">>> 整体:{flag} 缺样本:{report['missingGenres']} 退化:{report['regressedGenres']} "
|
||||
f"双低:{report['inconclusiveGenres']}", file=sys.stderr)
|
||||
print(f">>> 注:{report['note']}\n", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
ap = argparse.ArgumentParser(description="auto-vs-golden delta 门(同款双驱动 · 按品类 · 零 LLM)")
|
||||
ap.add_argument("--genres", default="click-score,whack-mole,shop-serve", help="逗号分隔品类键")
|
||||
ap.add_argument("--n", type=int, default=7, help="每品类款数")
|
||||
ap.add_argument("--conc", type=int, default=1, help="并发款数(端口池+线程,≤15;本机建议 ≤4)")
|
||||
ap.add_argument("--offset", type=int, default=0, help="gameId 起始索引(补跑用)")
|
||||
a = ap.parse_args()
|
||||
keys = [k.strip() for k in a.genres.split(",") if k.strip()]
|
||||
rep = asyncio.run(run_batch(keys, a.n, conc=a.conc, offset=a.offset))
|
||||
sys.exit(0 if rep["overallMeets"] else 1)
|
||||
|
||||
@ -189,6 +189,88 @@ def test_retire_deterministic_zero_llm():
|
||||
assert A.retire_authorization(m1p, cmp, avgp, _REQ) == A.retire_authorization(m1p, cmp, avgp, _REQ)
|
||||
|
||||
|
||||
# ───────────────────────── U2 同游戏双驱动 play 编排(mock · 不真跑模型)─────────────────────────
|
||||
|
||||
def _patch(obj, name, val, saved):
|
||||
saved.append((obj, name, getattr(obj, name)))
|
||||
setattr(obj, name, val)
|
||||
|
||||
|
||||
def _restore(saved):
|
||||
for obj, name, old in saved:
|
||||
setattr(obj, name, old)
|
||||
|
||||
|
||||
def test_dual_orchestration_order_auto_before_golden():
|
||||
"""编排序 = gen-only → smoke → ensure_play_spec → play(auto) → inject_golden → play(golden);
|
||||
auto_gates 在金标注入之前捕获(两次 play 返不同 verdict 可区分)。"""
|
||||
import cheap_studio
|
||||
import cheap_run
|
||||
import compare_node
|
||||
calls = []
|
||||
saved = []
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
td = Path(t)
|
||||
|
||||
async def fake_gen(gid, brief, **kw):
|
||||
calls.append("gen")
|
||||
|
||||
plays = iter([
|
||||
{"verdict": {"guards": {"E_live": {"pass": True}, "H_progress": {"pass": True}}}}, # auto
|
||||
{"verdict": {"guards": {"E_live": {"pass": False}, "H_progress": {"pass": True}}}}, # golden
|
||||
])
|
||||
_patch(cheap_studio, "run_studio", fake_gen, saved)
|
||||
_patch(cheap_run, "smoke", lambda gid, **kw: (calls.append("smoke") or {"ok": True, "state": {"targets": []}}), saved)
|
||||
_patch(cheap_run, "ensure_play_spec", lambda gid, state: (calls.append("ensure") or {"wrote": True}), saved)
|
||||
_patch(cheap_run, "play", lambda gid, **kw: (calls.append("play") or next(plays)), saved)
|
||||
_patch(cheap_run, "wg1_game_dir", lambda gid: td / gid, saved) # staged 无 spec
|
||||
_patch(cheap_run, "game_dir", lambda gid: td / gid, saved)
|
||||
_patch(compare_node, "inject_golden", lambda gid, g: (calls.append("inject") or (td / "g")), saved)
|
||||
_patch(compare_node, "golden_spec_path", lambda genre: td / "golden.json", saved)
|
||||
_patch(compare_node, "src_shape", lambda d: {"realSrcMultifile": True}, saved)
|
||||
try:
|
||||
r = A.run_pair_dual_sync({"key": "click-score", "golden": "x.json"}, "brief", "avg-click-score-0")
|
||||
finally:
|
||||
_restore(saved)
|
||||
|
||||
assert calls == ["gen", "smoke", "ensure", "play", "inject", "play"]
|
||||
# auto_gates 来自第一次 play(E_live=True),golden_gates 来自第二次(E_live=False)。
|
||||
assert r["autoGates"]["E_live"] is True and r["goldenGates"]["E_live"] is False
|
||||
assert r["gid"] == "avg-click-score-0" and r["realSrc"] is True
|
||||
|
||||
|
||||
def test_dual_raises_when_staged_spec_exists():
|
||||
"""staged 残留 play-spec.json → ensure 前置断言触发、不静默用旧 spec 驱动 auto。"""
|
||||
import cheap_studio
|
||||
import cheap_run
|
||||
saved = []
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
td = Path(t)
|
||||
gd = td / "avg-click-score-0"
|
||||
gd.mkdir(parents=True)
|
||||
(gd / "play-spec.json").write_text("{}", encoding="utf-8") # 残留旧 spec
|
||||
|
||||
async def fake_gen(gid, brief, **kw):
|
||||
pass
|
||||
|
||||
_patch(cheap_studio, "run_studio", fake_gen, saved)
|
||||
_patch(cheap_run, "smoke", lambda gid, **kw: {"ok": True, "state": {}}, saved)
|
||||
_patch(cheap_run, "wg1_game_dir", lambda gid: gd, saved)
|
||||
raised = False
|
||||
try:
|
||||
A.run_pair_dual_sync({"key": "click-score", "golden": "x.json"}, "brief", "avg-click-score-0")
|
||||
except RuntimeError:
|
||||
raised = True
|
||||
finally:
|
||||
_restore(saved)
|
||||
assert raised is True
|
||||
|
||||
|
||||
def test_clamp_conc_upper_cap():
|
||||
"""并发上限夹取 ≤15(复用 bake_off 口径)。"""
|
||||
assert A._clamp_conc(100) == 15 and A._clamp_conc(0) == 1 and A._clamp_conc(3) == 3
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_fns = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)]
|
||||
_failed = 0
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user