品类扩展 rubric(质量模型 SoT §4 规范二·五六):cheap_verify.GENRE_CHECKLISTS['trpg'] 5 条
(掷骰过程可见 L2 / 风险回报取舍 L2 / 成长进判定公式 L2 / 难度爬坡登顶 L3 / 冒险战报炫耀 L4),
每条 0/1 标层 + 一对正反例;仍是喂 LLM judge 的评分尺数据、纯 LLM 非阻塞(红线:不写代码校验
/不进九门/不进 verdict/不进脚手架)。同一次评分 additive 追加品类条目,**分母独立小计**落
richness.genre{key,hits,score,max,groups},通用 11 条 groups/score 分母不混(规范二);
接线 = run_studio 据 scaffold_template 查 GENRE_BY_TEMPLATE 透传 genre,未登记/None 行为
与既有完全一致(无品类路 judge prompt 逐字节不变,锚定纪律)。
金标 play-spec:fixtures/golden-specs/trpg.play-spec.json(tap-targets occupied 反应族
+ score increased + expectLatch,formalize 件② p11c-tpl 已验的 occupied spec;不用货币
递减门——SoT §3 裁定二门级判据不跨档);test_golden_specs 期望集同步纳 trpg(schema/
契约依赖/无坐标绑定 20/20 绿)。
金标复验(规范三,真跑 M3):正例 _template-trpg 骨架 品类 5/5、通用 8/11;薄反例(黑箱
点卡+无骰无取舍无成长无终点)品类 0/5、通用 0/11——正反例拉满、评分尺有鉴别力;同一正例
品类路 vs 基线路通用 11 条漂移 = 0(≤±1,加品类段不扰动通用判定)。单测 +5(条目形状/
独立分母/位置兜底偏移/无品类向后兼容/judge prompt additive)= test_cheap_verify 19/19 绿。
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
101 lines
5.2 KiB
Python
101 lines
5.2 KiB
Python
"""
|
|
test_golden_specs.py — U1 三品类金标 play-spec fixtures 校验(静态结构,不依赖真跑)。
|
|
|
|
只验金标 spec 自身正确:schema 合法 / 品类对位(三类同属 occupied 反应族)/ 契约依赖(只引 forensics
|
|
契约字段、不绑定具体游戏坐标)。**不验"现有样本能被驱动"**——那在 U3 实跑、按 R3 判(plan U1 Test expectation)。
|
|
|
|
跑:cheap-worker/.venv/bin/python cheap-worker/tests/test_golden_specs.py
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
_FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "golden-specs"
|
|
|
|
# tap-targets 代表品类:三个基线品类(plan KTD2,各 formalize base1/base2/base4 的 occupied spec)
|
|
# + W-GENRE 品类件④ 新增(trpg=_template-trpg 骨架 formalize;同属 occupied 反应族)。
|
|
_EXPECTED_GENRES = {"click-score", "whack-mole", "shop-serve", "trpg"}
|
|
# driver 只许这些键 —— 全是与具体游戏无关的契约级参数;出现坐标类键(x/y/coords/points)即"绑定具体游戏"违规。
|
|
_ALLOWED_DRIVER_KEYS = {"type", "targetMode", "targetsPath", "steps", "stepMs"}
|
|
_FORBIDDEN_COORD_KEYS = {"x", "y", "coords", "points", "positions", "targets"}
|
|
|
|
|
|
def _load_specs() -> dict:
|
|
"""加载 fixtures/golden-specs/*.play-spec.json → {genre: spec}。"""
|
|
out = {}
|
|
for p in sorted(_FIXTURES.glob("*.play-spec.json")):
|
|
spec = json.loads(p.read_text(encoding="utf-8"))
|
|
out[spec.get("_genre") or p.stem] = spec
|
|
return out
|
|
|
|
|
|
def test_three_genres_present():
|
|
"""tap-targets 代表品类集合恰好对位(无遗漏、无多余;新品类进金标须同步本期望集)。"""
|
|
specs = _load_specs()
|
|
assert set(specs) == _EXPECTED_GENRES, f"品类集不符:{set(specs)} != {_EXPECTED_GENRES}"
|
|
|
|
|
|
def test_schema_tap_targets_occupied():
|
|
"""schema 合法:三份都是 tap-targets + targetMode=occupied + 必填字段在。"""
|
|
for genre, spec in _load_specs().items():
|
|
d = spec.get("driver") or {}
|
|
assert d.get("type") == "tap-targets", f"{genre}: driver.type 应为 tap-targets,得 {d.get('type')}"
|
|
assert d.get("targetMode") == "occupied", f"{genre}: targetMode 应为 occupied,得 {d.get('targetMode')}"
|
|
assert d.get("targetsPath") == "targets", f"{genre}: targetsPath 应为 targets,得 {d.get('targetsPath')}"
|
|
assert isinstance(d.get("steps"), int) and d["steps"] > 0, f"{genre}: steps 缺/非正"
|
|
assert isinstance(d.get("stepMs"), int) and d["stepMs"] > 0, f"{genre}: stepMs 缺/非正"
|
|
|
|
|
|
def test_score_increased_assertion():
|
|
"""品类对位:三份均带 score 递增进度断言(occupied 反应族的可见成长口径)。"""
|
|
for genre, spec in _load_specs().items():
|
|
asserts = spec.get("assertAfterPlay") or []
|
|
score_inc = [a for a in asserts if a.get("path") == "score" and a.get("op") == "increased"]
|
|
assert score_inc, f"{genre}: 缺 score increased 进度断言"
|
|
assert spec.get("expectLatch") is True, f"{genre}: expectLatch 应为 true(限时类有进展降 advisory)"
|
|
|
|
|
|
def test_no_other_driver_families():
|
|
"""有意不产放置族 / key-cycle / safeOnly —— 三类同族,别的家族在五品类无消费者(F5/C2/O2)。"""
|
|
for genre, spec in _load_specs().items():
|
|
d = spec.get("driver") or {}
|
|
assert d.get("type") != "key-cycle", f"{genre}: 不应是 key-cycle(延到输入键契约)"
|
|
assert d.get("targetMode") != "safe" and not d.get("safeOnly"), f"{genre}: 不应是规避 safeOnly 族"
|
|
# 放置族 = occupied 缺省/点空位;本计划三类全 occupied,不允许其它 targetMode。
|
|
assert d.get("targetMode") == "occupied", f"{genre}: 不应是放置族,应 occupied 反应族"
|
|
|
|
|
|
def test_contract_only_no_hardcoded_coords():
|
|
"""契约依赖:driver 只含契约级参数,不含坐标类键(不绑定具体游戏);断言路径只引 forensics 字段。"""
|
|
for genre, spec in _load_specs().items():
|
|
d = spec.get("driver") or {}
|
|
extra = set(d) - _ALLOWED_DRIVER_KEYS
|
|
assert not extra, f"{genre}: driver 含非契约键 {extra}(疑似绑定具体游戏)"
|
|
coord = set(d) & _FORBIDDEN_COORD_KEYS
|
|
assert not coord, f"{genre}: driver 含坐标类键 {coord}(违反'不绑定具体游戏坐标')"
|
|
for a in spec.get("assertAfterPlay") or []:
|
|
assert a.get("path") in ("score", "phase"), f"{genre}: 断言路径 {a.get('path')} 非 forensics 契约字段"
|
|
|
|
|
|
def test_click_score_is_occupied_not_placement():
|
|
"""base1 同族:点击得分按 occupied 反应族写(非放置族)——C2/O2 修正。"""
|
|
specs = _load_specs()
|
|
cs = specs.get("click-score")
|
|
assert cs is not None, "缺 click-score 金标 spec"
|
|
assert cs["driver"]["targetMode"] == "occupied", "点击得分应 occupied(base1 targets 即 occupied:true),非放置族"
|
|
|
|
|
|
if __name__ == "__main__":
|
|
_fns = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)]
|
|
_failed = 0
|
|
for _fn in _fns:
|
|
try:
|
|
_fn()
|
|
print(f" PASS {_fn.__name__}")
|
|
except Exception as e: # noqa: BLE001
|
|
_failed += 1
|
|
print(f" FAIL {_fn.__name__}: {type(e).__name__}: {e}")
|
|
print(f"\n{len(_fns) - _failed}/{len(_fns)} passed")
|
|
sys.exit(1 if _failed else 0)
|