87777975c7
actor/judge prompt eval 闭合: actor 3.0.5 正文强化+baseline(7轮); judge 3.0.14 正文强化+anyTerms同义词补全+baseline(judge-a 13轮/judge-b 33轮); registry/checker 版本对齐+顺序锚点 fail-closed 双保险 W-GOLD-LIVE live_prompt: play-loop 契约开口(request/3+provenance/3+ReferenceAssetRecord/1+Registry/1+迁移清单15条); cheap_verify 切v3+消费对账六闸(只消费active)+生成prompt注入接线(0 active不注入); Node runner playtest-v3.cdp.cjs 升/3+绊线恢复; 真模型回归验向后兼容 full_gate+三批基线: full_gate.py 集成runner串六子门+降级+22测; baseline_gates.py fresh25阈值+historical11预期表(2 needs_human交创始人定标)+shadow20框架+38测 注: registry cheap-system 登记 1.8.0→1.8.1 同步此前在途 cheap-system.md frontmatter 升版(修 pre-commit 版本漂移); cheap_verify/cheap_studio/validate/test_acceptance_v3/test_cheap_service_driver 为在途M含本会话叠加+此前W-AXIS在途(同一线无法hunk分离); 3红测试根因与raw审计未入本提交
273 lines
13 KiB
Python
273 lines
13 KiB
Python
"""full_gate 集成 runner 测试：罐头子门结果验证组合逻辑（确定性，零模型零网络）。
|
||
|
||
覆盖设计档 §3.2 四态降级 + §5.2 失败归因红线：全过→PASS、任一 reject→降级 reject、
|
||
任一 tester_error/缺产物/degraded→tester_error（绝不伪装 accept，也绝不伪装 gameplay reject）、
|
||
优先级 tester_error > reject > inconclusive > accept。
|
||
"""
|
||
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
|
||
import cheap_verify as V # noqa: E402
|
||
import full_gate # noqa: E402
|
||
from full_gate import run_full_gate # noqa: E402
|
||
|
||
|
||
# ── 罐头子门产物（绿态）──
|
||
|
||
def _green_verdict():
|
||
return {"ok": True, "pass": True,
|
||
"guards": {name: {"pass": True} for name in
|
||
("A_boot", "B_uncaught", "C_frame", "D_render", "E_live",
|
||
"F_wiring", "G_input", "H_progress", "I_control")}}
|
||
|
||
|
||
def _green_floor():
|
||
return {"accepted": True, "verdict": "accept", "rejectClasses": [], "degraded": False}
|
||
|
||
|
||
def _green_v3_payload(mode="v3"):
|
||
return {"schemaVersion": "playtest/3", "acceptanceMode": mode, "outcome": "accept",
|
||
"decision": {"accepted": True, "publishFrozen": mode != "v3",
|
||
"parentChainCostRmb": 1.2},
|
||
"finalPostguard": {"pass": True, "checks": {"floorPass": True}},
|
||
"compatibility": {"accepted": mode == "v3", "ok": mode == "v3",
|
||
"publishFrozen": mode != "v3"}}
|
||
|
||
|
||
def _green_prompt_eval():
|
||
return {"promptId": "cheap-actor", "infrastructureComplete": True, "allGreen": True,
|
||
"gate1_schema": True, "gate2_success": True, "gate3_regression": True,
|
||
"gate4_cost_latency": True, "gate5_stability": True}
|
||
|
||
|
||
def _noop_validator(_payload):
|
||
return [] # 罐头 schema 校验：全绿路径不真跑 canonical validator（真校验另有契约测试覆盖）
|
||
|
||
|
||
def _green_reconciler(consumptions, *, consumer_ref=None, registry=None):
|
||
return {"ok": True, "consumed": [{"recordId": c if isinstance(c, str) else c["recordId"],
|
||
"role": "harness_fixture", "artifactHash": "h"}
|
||
for c in consumptions], "errors": []}
|
||
|
||
|
||
def _all_green_kwargs(**overrides):
|
||
kwargs = {
|
||
"verdict": _green_verdict(), "floor_judgment": _green_floor(),
|
||
"v3_payload": _green_v3_payload(), "prompt_eval_record": _green_prompt_eval(),
|
||
"schema_validator": _noop_validator, "gold_reconciler": _green_reconciler,
|
||
}
|
||
kwargs.update(overrides)
|
||
return kwargs
|
||
|
||
|
||
# ── 全过 → PASS ──
|
||
|
||
def test_all_green_passes_and_publishable():
|
||
result = run_full_gate(**_all_green_kwargs())
|
||
assert result["pass"] is True
|
||
assert result["outcome"] == "accept"
|
||
assert result["publishable"] is True
|
||
assert result["reasons"] == []
|
||
assert {name: gate["state"] for name, gate in result["gates"].items()} == {
|
||
"mechanicalNine": "pass", "visualFloor": "pass", "playtestV3": "pass",
|
||
"promptEval": "pass", "schemaSemantics": "pass", "goldReconcile": "pass",
|
||
}
|
||
|
||
|
||
def test_shadow_accept_passes_but_not_publishable():
|
||
"""v3_shadow 的 accept 只供校准：full_gate 可判过，但发布冻结（§3.10）。"""
|
||
result = run_full_gate(**_all_green_kwargs(v3_payload=_green_v3_payload(mode="v3_shadow")))
|
||
assert result["pass"] is True and result["outcome"] == "accept"
|
||
assert result["publishable"] is False
|
||
|
||
|
||
def test_vacuous_gold_reconcile_passes_without_declaration():
|
||
"""未声明参照资产消费 ≡ 旧路径 vacuous 放行（与 _v3_check_declared_reference_assets 语义一致）。"""
|
||
result = run_full_gate(**_all_green_kwargs())
|
||
assert result["gates"]["goldReconcile"]["vacuous"] is True
|
||
assert result["pass"] is True
|
||
|
||
|
||
# ── 任一 reject → 降级 reject ──
|
||
|
||
def test_mechanical_failure_degrades_to_reject():
|
||
verdict = _green_verdict()
|
||
verdict["pass"] = False
|
||
verdict["guards"]["A_boot"] = {"pass": False, "err": "boot 崩溃"}
|
||
result = run_full_gate(**_all_green_kwargs(verdict=verdict))
|
||
assert result["pass"] is False
|
||
assert result["outcome"] == "reject"
|
||
assert result["gates"]["mechanicalNine"]["failedGates"] == ["A_boot"]
|
||
|
||
|
||
def test_check_level_structural_failure_is_reject():
|
||
"""Node check 级失败（src/ 缺失，ok=False）→ 机械 reject。"""
|
||
result = run_full_gate(**_all_green_kwargs(verdict={"ok": False, "errors": ["src/ 目录不存在或不可读"]}))
|
||
assert result["pass"] is False and result["outcome"] == "reject"
|
||
|
||
|
||
def test_visual_floor_reject_degrades_to_reject():
|
||
floor = {"accepted": False, "verdict": "reject", "rejectClasses": ["hollow"], "degraded": False}
|
||
result = run_full_gate(**_all_green_kwargs(floor_judgment=floor))
|
||
assert result["pass"] is False and result["outcome"] == "reject"
|
||
assert result["gates"]["visualFloor"]["rejectClasses"] == ["hollow"]
|
||
|
||
|
||
def test_playtest_v3_reject_degrades_to_reject():
|
||
payload = _green_v3_payload()
|
||
payload.update({"outcome": "reject",
|
||
"decision": {"accepted": False, "failure": {"layer": "gameplay"}}})
|
||
payload["finalPostguard"]["pass"] = False
|
||
result = run_full_gate(**_all_green_kwargs(v3_payload=payload))
|
||
assert result["pass"] is False and result["outcome"] == "reject"
|
||
|
||
|
||
def test_gold_reconcile_errors_are_verified_reject():
|
||
"""声明消费而对账失败 = verified reject（设计语义）。"""
|
||
def _bad_reconciler(consumptions, *, consumer_ref=None, registry=None):
|
||
return {"ok": False, "consumed": [], "errors": ["参照资产未激活不得消费:ref-001(lifecycleStatus=candidate)"]}
|
||
result = run_full_gate(**_all_green_kwargs(
|
||
reference_consumptions=["ref-001"], gold_reconciler=_bad_reconciler))
|
||
assert result["pass"] is False and result["outcome"] == "reject"
|
||
assert "未激活" in result["gates"]["goldReconcile"]["reason"]
|
||
|
||
|
||
# ── tester_error 绝不伪装 accept ──
|
||
|
||
def test_playtest_tester_error_never_disguises_accept():
|
||
payload = _green_v3_payload()
|
||
payload.update({"outcome": "tester_error",
|
||
"decision": {"accepted": False, "failure": {"layer": "tester_error",
|
||
"subtype": "environment_error"}}})
|
||
payload["finalPostguard"]["pass"] = False
|
||
result = run_full_gate(**_all_green_kwargs(v3_payload=payload))
|
||
assert result["pass"] is False
|
||
assert result["outcome"] == "tester_error" # 不是 accept，也不是 reject
|
||
|
||
|
||
def test_self_contradictory_payload_is_tester_error():
|
||
"""decision.accepted=True 但 outcome≠accept：封存产物自相矛盾 = 仪器异常，绝不放行。"""
|
||
payload = _green_v3_payload()
|
||
payload["outcome"] = "reject" # accepted 仍为 True → 矛盾
|
||
result = run_full_gate(**_all_green_kwargs(v3_payload=payload))
|
||
assert result["pass"] is False and result["outcome"] == "tester_error"
|
||
assert "矛盾" in result["gates"]["playtestV3"]["reason"]
|
||
|
||
|
||
def test_degraded_floor_is_tester_error_not_gameplay_reject():
|
||
"""地板 degraded（评不出）按 §5.2 归仪器异常，不伪装 gameplay reject。"""
|
||
floor = {"accepted": False, "verdict": "reject", "rejectClasses": ["degraded"],
|
||
"degraded": True, "reason": "真玩截图证据缺失"}
|
||
result = run_full_gate(**_all_green_kwargs(floor_judgment=floor))
|
||
assert result["pass"] is False and result["outcome"] == "tester_error"
|
||
|
||
|
||
def test_missing_artifacts_fail_closed_to_tester_error():
|
||
"""任何子门产物缺失 → fail-closed tester_error，六个都缺也绝不 accept。"""
|
||
result = run_full_gate(schema_validator=_noop_validator, gold_reconciler=_green_reconciler)
|
||
assert result["pass"] is False and result["outcome"] == "tester_error"
|
||
assert result["publishable"] is False
|
||
missing = [name for name, gate in result["gates"].items() if gate["state"] == "tester_error"]
|
||
assert set(missing) == {"mechanicalNine", "visualFloor", "playtestV3", "promptEval", "schemaSemantics"}
|
||
|
||
|
||
def test_schema_errors_are_tester_error():
|
||
"""canonical validate.py 报错 = schema 异常 → tester_error（§5.2）。"""
|
||
result = run_full_gate(**_all_green_kwargs(
|
||
schema_validator=lambda p: ["#/events/0 payloadHash 与 payloadCanonical 不一致"]))
|
||
assert result["pass"] is False and result["outcome"] == "tester_error"
|
||
assert result["gates"]["schemaSemantics"]["errors"]
|
||
|
||
|
||
def test_prompt_eval_red_gate_is_tester_error():
|
||
"""校准闸红 = Actor/Judge 判读不可信 → tester_error，绝不放行。"""
|
||
record = _green_prompt_eval()
|
||
record.update({"allGreen": False, "gate3_regression": False})
|
||
result = run_full_gate(**_all_green_kwargs(prompt_eval_record=record))
|
||
assert result["pass"] is False and result["outcome"] == "tester_error"
|
||
assert "gate3_regression" in result["gates"]["promptEval"]["failedRecords"][0]["reason"]
|
||
|
||
|
||
def test_prompt_eval_infrastructure_uncertain_is_tester_error():
|
||
record = _green_prompt_eval()
|
||
record["infrastructureComplete"] = False
|
||
result = run_full_gate(**_all_green_kwargs(prompt_eval_record=record))
|
||
assert result["pass"] is False and result["outcome"] == "tester_error"
|
||
|
||
|
||
def test_prompt_eval_multiple_records_all_must_be_green():
|
||
"""Actor + Judge A/B 多条记录：任一不绿整体 tester_error。"""
|
||
judge_b = _green_prompt_eval()
|
||
judge_b.update({"promptId": "cheap-judge-b", "allGreen": False, "gate2_success": False})
|
||
result = run_full_gate(**_all_green_kwargs(
|
||
prompt_eval_record=[_green_prompt_eval(), judge_b]))
|
||
assert result["pass"] is False and result["outcome"] == "tester_error"
|
||
assert result["gates"]["promptEval"]["failedRecords"][0]["promptId"] == "cheap-judge-b"
|
||
|
||
|
||
# ── inconclusive 与优先级 ──
|
||
|
||
def test_playtest_inconclusive_degrades_to_inconclusive():
|
||
payload = _green_v3_payload()
|
||
payload.update({"outcome": "inconclusive",
|
||
"decision": {"accepted": False, "failure": {"layer": "gameplay",
|
||
"subtype": "proof_missing"}}})
|
||
payload["finalPostguard"]["pass"] = False
|
||
result = run_full_gate(**_all_green_kwargs(v3_payload=payload))
|
||
assert result["pass"] is False and result["outcome"] == "inconclusive"
|
||
|
||
|
||
def test_tester_error_outranks_reject():
|
||
"""一子门 reject + 另一子门 tester_error → 整体 tester_error（仪器异常时不能宣称已证缺陷）。"""
|
||
floor = {"accepted": False, "verdict": "reject", "rejectClasses": ["broken"], "degraded": False}
|
||
payload = _green_v3_payload()
|
||
payload.update({"outcome": "reject", "decision": {"accepted": False}})
|
||
payload["finalPostguard"]["pass"] = False
|
||
result = run_full_gate(**_all_green_kwargs(floor_judgment=floor, v3_payload=payload,
|
||
schema_validator=lambda p: ["schema 脏"]))
|
||
assert result["pass"] is False
|
||
assert result["outcome"] == "tester_error" # schema 异常压过 floor reject
|
||
|
||
|
||
def test_reject_outranks_inconclusive():
|
||
payload = _green_v3_payload()
|
||
payload.update({"outcome": "inconclusive", "decision": {"accepted": False}})
|
||
payload["finalPostguard"]["pass"] = False
|
||
floor = {"accepted": False, "verdict": "reject", "rejectClasses": ["off_brief"], "degraded": False}
|
||
result = run_full_gate(**_all_green_kwargs(v3_payload=payload, floor_judgment=floor))
|
||
assert result["pass"] is False and result["outcome"] == "reject"
|
||
|
||
|
||
# ── 默认绑定真子门函数（消费不重写）──
|
||
|
||
def test_default_validators_bind_real_subgate_functions():
|
||
"""不注入时默认绑 cheap_verify 真函数：canonical 校验 + 金标对账，保证集成 runner 消费子门而非影子实现。"""
|
||
import inspect
|
||
src = inspect.getsource(full_gate.run_full_gate)
|
||
assert "V.validate_acceptance_v3_payload" in src
|
||
assert "V.reconcile_v3_reference_asset_consumption" in src
|
||
# 默认 schema_validator 对非法 payload 必须 fail-closed 报错（真跑 canonical 校验器）。
|
||
result = run_full_gate(verdict=_green_verdict(), floor_judgment=_green_floor(),
|
||
v3_payload={"schemaVersion": "playtest/3"},
|
||
prompt_eval_record=_green_prompt_eval())
|
||
assert result["gates"]["schemaSemantics"]["state"] == "tester_error"
|
||
assert result["gates"]["schemaSemantics"]["errors"]
|
||
|
||
|
||
def test_gold_reconcile_default_uses_real_registry_when_declared():
|
||
"""声明消费 + 默认注册表（当前全是 migration_pending、无 active）→ 对账拒绝 reject。
|
||
|
||
这正是 W-GOLD-LIVE「完成前不得新增 live 消费」的机器强制点：声明消费一条在册但未激活的记录即被拒。
|
||
"""
|
||
result = run_full_gate(verdict=_green_verdict(), floor_judgment=_green_floor(),
|
||
v3_payload=_green_v3_payload(), prompt_eval_record=_green_prompt_eval(),
|
||
schema_validator=_noop_validator,
|
||
reference_consumptions=["gold-m3-gem-r3"])
|
||
assert result["gates"]["goldReconcile"]["state"] == "reject"
|
||
assert result["gates"]["goldReconcile"]["vacuous"] is False
|
||
assert "未激活" in result["gates"]["goldReconcile"]["reason"]
|
||
assert result["outcome"] == "reject"
|