- D1: Registry 3 prompt(策划/对抗/fix)+eval 骨架+registry 注册 - D2: agent-loop 2 schema + 4 模板 schema(对齐 gen_spike 真源, additionalProperties:false) - D3: 编排器+裁决引擎(judge D1-D10 决策表/账本幂等重放/预算闸熔断/eval 回流/批报告), 59 单测绿(mini-desktop) - D5: 玩家 CDP 取证 player_cdp.py(真点击/蛇形→驼峰映射/demo 三信号/三角合成判定), 9001 自测 PASS - (D4 后端件已在 9bf3d54 先行入库, 三条 mvn 两轮独立实跑全绿) - spec: 评审版(已拍板)+execution 版(已审定+建设期补充裁决 D4-a~d/D5-a~c/D2-a/D3-a) - W2 试点/种子 9003-9008/B4 四链路烟测 产物与总账/作战清单同步 - eval 种子: C1 spike 52 条转入 config.clicker-designer(inputs 52/labels 13) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
364 lines
20 KiB
Python
364 lines
20 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
裁决引擎单测(§7.8/§10.3):决策表 D1-D10 全分支 + P2-only + fix 后二轮 D6 + 未匹配 raise。
|
||
|
||
全组合覆盖论证(§10.2,以注释+用例固化):输入空间按求值序短路——
|
||
E1 截获全部 schemaOk=false(D1/D2 按 round 二分);E2 截获 dupHit(D3);
|
||
E3 截获 hasP0(D4);E4 截获 hasP1(D5/D6 按 round 二分);
|
||
E5 截获实玩前后全部 infra 与 runnable 失败(D7/D8/D9 按 infraSignal/playAttempt 三分);
|
||
E6 仅剩全真组合(D10)。每用例注释标注对应表行号。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import sys
|
||
import unittest
|
||
|
||
# 保证从任意工作目录可导入被测模块(orchestrator/ 上一级即本文件父目录的父目录)
|
||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
||
import judge # noqa: E402
|
||
|
||
# 仓库根(tests/ → orchestrator/ → 2026-06-09-agent-loop-v1/ → agent-specs/ → docs/ → 根,共上溯 5 级)
|
||
_REPO = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||
"..", "..", "..", "..", ".."))
|
||
|
||
|
||
def good_play_report(**overrides):
|
||
"""
|
||
构造满足五条 AND(§9.4)的 PlayReport 桩;overrides 用于逐条打破。
|
||
含 D5 超集字段 checksumVerifyMethod(§16 D5-a:⑤ 须核验方法可信,none/缺失不得通过)。
|
||
"""
|
||
report = {
|
||
"loaded": True,
|
||
"clicks": 15,
|
||
"gameEnd": {"completed": True, "durationMs": 2345},
|
||
"consoleErrors": [],
|
||
"assetLoadErrors": 0,
|
||
"packageChecksumVerified": True,
|
||
"checksumVerifyMethod": "browser_body", # D5 超集字段:⑤ 可信方法(browser_body/synthesized)
|
||
"emitsCaptured": ["game_loaded", "game_start", "game_end"],
|
||
"demoFallback": False,
|
||
"score": 15,
|
||
}
|
||
report.update(overrides)
|
||
return report
|
||
|
||
|
||
class TestDecisionTable(unittest.TestCase):
|
||
"""§10.2 决策表逐行用例(注释标行号)。"""
|
||
|
||
def test_d1_schema_fail_round0_local_retry(self):
|
||
"""表行 D1:schemaOk=false @round=0 → 本地重出(不回调、消耗唯一回炉额度)。"""
|
||
out = judge.judge(False, False, [], None, 0, False, 0)
|
||
self.assertEqual(out.row, "D1")
|
||
self.assertEqual(out.action, judge.ACTION_SCHEMA_RETRY)
|
||
self.assertIsNone(out.decision) # 非终判行不产 Verdict
|
||
self.assertEqual(out.reasons, ["schema_retry"])
|
||
self.assertIsNone(out.callback) # 不回调
|
||
|
||
def test_d2_schema_fail_round1_kill(self):
|
||
"""表行 D2:schemaOk=false @round=1 → kill;回调 failed+config_invalid。"""
|
||
out = judge.judge(False, False, [], None, 0, False, 1)
|
||
self.assertEqual(out.row, "D2")
|
||
self.assertEqual(out.action, judge.ACTION_KILL)
|
||
self.assertEqual(out.decision, judge.DECISION_KILL)
|
||
self.assertEqual(out.reasons, ["schema_invalid_after_retry"])
|
||
self.assertEqual(out.callback, judge.CALLBACK_FAILED_CONFIG_INVALID)
|
||
self.assertFalse(out.structure_ok) # 结构未过
|
||
|
||
def test_d3_duplicate_kill(self):
|
||
"""表行 D3:dupHit=true → kill(round 无关);归桶 config_invalid(§16-1),真因记 reasons。"""
|
||
for round_ in (0, 1):
|
||
out = judge.judge(True, True, [], None, 0, False, round_)
|
||
self.assertEqual(out.row, "D3")
|
||
self.assertEqual(out.action, judge.ACTION_KILL)
|
||
self.assertEqual(out.reasons, ["duplicate_in_batch"])
|
||
self.assertEqual(out.callback, judge.CALLBACK_FAILED_CONFIG_INVALID)
|
||
|
||
def test_d4_p0_kill(self):
|
||
"""表行 D4:∃P0 → 即杀不给 fix(round 无关);回调 failed+unsafe_prompt;reasons=p0_<类别>。"""
|
||
findings = [{"severity": "P0", "issue": "包含敏感涉政内容", "suggestion": "整体废弃"}]
|
||
for round_ in (0, 1):
|
||
out = judge.judge(True, False, findings, None, 0, False, round_)
|
||
self.assertEqual(out.row, "D4")
|
||
self.assertEqual(out.action, judge.ACTION_KILL)
|
||
self.assertEqual(out.callback, judge.CALLBACK_FAILED_UNSAFE_PROMPT)
|
||
self.assertEqual(out.reasons, ["p0_sensitive"]) # 确定性关键词归桶
|
||
|
||
def test_d5_p1_round0_fix(self):
|
||
"""表行 D5:∃P1 @round=0 → fix(不回调,本轮不落包);decision=fix 可铸中间 Verdict。"""
|
||
findings = [{"severity": "P1", "issue": "题文不符", "suggestion": "改 title"}]
|
||
out = judge.judge(True, False, findings, None, 0, False, 0)
|
||
self.assertEqual(out.row, "D5")
|
||
self.assertEqual(out.action, judge.ACTION_FIX)
|
||
self.assertEqual(out.decision, judge.DECISION_FIX)
|
||
self.assertEqual(out.reasons, ["p1_fix_round"])
|
||
self.assertIsNone(out.callback)
|
||
|
||
def test_d6_p1_round1_kill(self):
|
||
"""表行 D6:∃P1 @round=1 → kill(额度用尽);归桶 config_invalid。"""
|
||
findings = [{"severity": "P1", "issue": "target 与意图矛盾", "suggestion": "调 target"}]
|
||
out = judge.judge(True, False, findings, None, 0, False, 1)
|
||
self.assertEqual(out.row, "D6")
|
||
self.assertEqual(out.action, judge.ACTION_KILL)
|
||
self.assertEqual(out.reasons, ["p1_residual_after_fix"])
|
||
self.assertEqual(out.callback, judge.CALLBACK_FAILED_CONFIG_INVALID)
|
||
|
||
def test_d7_infra_fail(self):
|
||
"""表行 D7:infraSignal=true → infra_fail(不计分母、可重放);reasons=infra_<类别>。"""
|
||
out = judge.judge(True, False, [], None, 0, True, 0, infra_category="demo_fallback")
|
||
self.assertEqual(out.row, "D7")
|
||
self.assertEqual(out.action, judge.ACTION_INFRA_FAIL)
|
||
self.assertIsNone(out.decision) # infra 不产 Verdict(resume 从 submit 重走)
|
||
self.assertEqual(out.reasons, ["infra_demo_fallback"])
|
||
# 类别缺省时兜底 unknown
|
||
out2 = judge.judge(True, False, [], None, 0, True, 1)
|
||
self.assertEqual(out2.reasons, ["infra_unknown"])
|
||
|
||
def test_d8_runnable_fail_attempt1_retry(self):
|
||
"""表行 D8:runnableOk=false ∧ 非 infra ∧ playAttempt=1 → 重试实玩(不消耗回炉额度)。"""
|
||
report = good_play_report(gameEnd={"completed": False, "durationMs": 1200})
|
||
out = judge.judge(True, False, [], report, 1, False, 0)
|
||
self.assertEqual(out.row, "D8")
|
||
self.assertEqual(out.action, judge.ACTION_PLAY_RETRY)
|
||
self.assertEqual(out.reasons, ["runnable_retry"])
|
||
|
||
def test_d9_runnable_fail_attempt2_kill(self):
|
||
"""表行 D9:runnableOk=false ∧ playAttempt=2 → kill-runnable;任务保持 succeeded(不回调)。"""
|
||
report = good_play_report(loaded=False, emitsCaptured=[])
|
||
out = judge.judge(True, False, [], report, 2, False, 0)
|
||
self.assertEqual(out.row, "D9")
|
||
self.assertEqual(out.action, judge.ACTION_KILL)
|
||
self.assertEqual(out.decision, judge.DECISION_KILL)
|
||
self.assertEqual(out.reasons, ["runnable_fail_after_retry"])
|
||
self.assertIsNone(out.callback) # 不再回调(任务已 succeeded)
|
||
|
||
def test_d10_accept(self):
|
||
"""表行 D10:四条 AND 全真 → accept(structureOk ∧ runnableOk 同真)。"""
|
||
out = judge.judge(True, False, [], good_play_report(), 1, False, 0)
|
||
self.assertEqual(out.row, "D10")
|
||
self.assertEqual(out.action, judge.ACTION_ACCEPT)
|
||
self.assertEqual(out.decision, judge.DECISION_ACCEPT)
|
||
self.assertTrue(out.structure_ok)
|
||
self.assertTrue(out.runnable_ok)
|
||
self.assertEqual(out.reasons, ["accept"])
|
||
|
||
def test_p2_only_goes_accept(self):
|
||
"""§10.2 注:P2-only 组合归 D10 路径(P2 不挡 accept,仅留档)。"""
|
||
findings = [{"severity": "P2", "issue": "文案可更生动", "suggestion": "润色"}]
|
||
out = judge.judge(True, False, findings, good_play_report(), 1, False, 0)
|
||
self.assertEqual(out.row, "D10")
|
||
self.assertEqual(out.decision, judge.DECISION_ACCEPT)
|
||
|
||
def test_fix_then_round1_new_p1_goes_d6(self):
|
||
"""§10.2 注:fix 重走后的二轮链路——round=0 P1→D5(fix),round=1 新 P1→D6(kill)。"""
|
||
p1 = [{"severity": "P1", "issue": "文案占位感", "suggestion": "重写"}]
|
||
first = judge.judge(True, False, p1, None, 0, False, 0)
|
||
self.assertEqual(first.row, "D5") # 首轮回炉
|
||
new_p1 = [{"severity": "P1", "issue": "仍题文不符", "suggestion": "再改"}]
|
||
second = judge.judge(True, False, new_p1, None, 0, False, 1)
|
||
self.assertEqual(second.row, "D6") # 二轮 P1 残留即杀
|
||
self.assertEqual(second.reasons, ["p1_residual_after_fix"])
|
||
|
||
def test_unmatched_input_raises(self):
|
||
"""§10.2 末尾防御:文本面全过、无 infra、无实玩报告 → 决策表无此组合,必须 raise。"""
|
||
with self.assertRaises(judge.JudgeError):
|
||
judge.judge(True, False, [], None, 0, False, 0)
|
||
|
||
# ---------------- 防御面补充用例(决策表定义域边界) ----------------
|
||
|
||
def test_invalid_round_raises(self):
|
||
"""防御:round 越出 0/1(回炉额度合计 1 轮)立即暴露。"""
|
||
with self.assertRaises(judge.JudgeError):
|
||
judge.judge(True, False, [], good_play_report(), 1, False, 2)
|
||
|
||
def test_play_attempt_3_raises(self):
|
||
"""防御:playAttempt=3 越出决策表(实玩预算闸应已拦截)→ raise。"""
|
||
report = good_play_report(consoleErrors=["TypeError: boom"])
|
||
with self.assertRaises(judge.JudgeError):
|
||
judge.judge(True, False, [], report, 3, False, 0)
|
||
|
||
def test_demo_fallback_without_infra_signal_raises(self):
|
||
"""防御:demoFallback=true 必须以 infraSignal 上报(§9.5),漏报即编排器缺陷 → raise。"""
|
||
with self.assertRaises(judge.JudgeError):
|
||
judge.judge(True, False, [], good_play_report(demoFallback=True), 1, False, 0)
|
||
|
||
def test_invalid_severity_raises(self):
|
||
"""防御:findings.severity 越出 P0/P1/P2 受控枚举 → raise(不静默放行 LLM 噪声)。"""
|
||
with self.assertRaises(judge.JudgeError):
|
||
judge.judge(True, False, [{"severity": "P3", "issue": "x"}], None, 0, False, 0)
|
||
|
||
|
||
class TestRunnableFiveAnd(unittest.TestCase):
|
||
"""§9.4 五条 AND 逐条打破用例(D8 路径的归因条款)。"""
|
||
|
||
def test_clause3_duration_zero_not_runnable(self):
|
||
"""③ durationMs=0(防 startMs 缺省假象)→ runnable=false。"""
|
||
report = good_play_report(gameEnd={"completed": True, "durationMs": 0})
|
||
ok, failed = judge.compute_runnable_ok(report)
|
||
self.assertFalse(ok)
|
||
self.assertTrue(any("③" in c for c in failed))
|
||
|
||
def test_clause4_game_error_emit_not_runnable(self):
|
||
"""④ 捕获 game_error 信封 → runnable=false。"""
|
||
report = good_play_report(emitsCaptured=["game_loaded", "game_error", "game_end"])
|
||
ok, failed = judge.compute_runnable_ok(report)
|
||
self.assertFalse(ok)
|
||
self.assertTrue(any("game_error" in c for c in failed))
|
||
|
||
def test_clause5_checksum_unverified_not_runnable(self):
|
||
"""⑤ packageChecksumVerified=false → runnable=false(防 demo 假阳性的契约位)。"""
|
||
ok, failed = judge.compute_runnable_ok(good_play_report(packageChecksumVerified=False))
|
||
self.assertFalse(ok)
|
||
self.assertTrue(any("⑤" in c for c in failed))
|
||
|
||
def test_clause5_verify_method_none_not_runnable(self):
|
||
"""⑤ §16 D5-a:packageChecksumVerified=true 但 checksumVerifyMethod=none(或缺失)不得通过。"""
|
||
# method=none:D5 报告自相矛盾(true 却无可信核验法)→ ⑤ 判不过
|
||
ok, failed = judge.compute_runnable_ok(good_play_report(checksumVerifyMethod="none"))
|
||
self.assertFalse(ok)
|
||
self.assertTrue(any("D5-a" in c for c in failed))
|
||
# method 字段缺失(旧形状/异常报告):安全侧同判不过
|
||
report = good_play_report()
|
||
del report["checksumVerifyMethod"]
|
||
ok2, failed2 = judge.compute_runnable_ok(report)
|
||
self.assertFalse(ok2)
|
||
self.assertTrue(any("⑤" in c for c in failed2))
|
||
|
||
def test_clause5_synthesized_method_passes(self):
|
||
"""⑤ §16 D5-a:三角合成判定(checksumVerifyMethod=synthesized)为可信方法,应通过。"""
|
||
ok, failed = judge.compute_runnable_ok(good_play_report(checksumVerifyMethod="synthesized"))
|
||
self.assertTrue(ok, failed)
|
||
|
||
def test_asset_load_errors_do_not_veto(self):
|
||
"""④ 注:assetLoadErrors>0 仅记录不否决(v1 assets=[])。"""
|
||
ok, failed = judge.compute_runnable_ok(good_play_report(assetLoadErrors=2))
|
||
self.assertTrue(ok)
|
||
self.assertEqual(failed, [])
|
||
|
||
|
||
class TestSchemaValidator(unittest.TestCase):
|
||
"""
|
||
模板 schema 深度校验(结构门权威;镜像 §6.4 验收口径)。
|
||
夹具=直接加载 D2 真契约文件 contracts/templates/clicker.schema.json
|
||
(D2 核验意见落实:内联夹具曾缺 pattern \\S,改为以生产契约为唯一被测口径——
|
||
title/theme/scoreLabel 的「非纯空白」由真 schema 的 pattern \\S 与校验器 strip 语义双重保障)。
|
||
"""
|
||
|
||
@classmethod
|
||
def setUpClass(cls):
|
||
schema_path = os.path.join(_REPO, "contracts", "templates", "clicker.schema.json")
|
||
with open(schema_path, "r", encoding="utf-8") as fh:
|
||
cls.CLICKER_SCHEMA = json.load(fh)
|
||
|
||
def good_config(self, **overrides):
|
||
config = {"templateId": "clicker", "title": "深夜便利店", "theme": "夜宵",
|
||
"target": 12, "scoreLabel": "已服务顾客"}
|
||
config.update(overrides)
|
||
return config
|
||
|
||
def test_real_schema_has_nonblank_pattern(self):
|
||
"""前置自检:D2 真 schema 的三个字符串字段必须带 pattern \\S(非纯空白契约位)。"""
|
||
props = self.CLICKER_SCHEMA["properties"]
|
||
for key in ("title", "theme", "scoreLabel"):
|
||
self.assertEqual(props[key].get("pattern"), "\\S",
|
||
"clicker.schema.json 的 %s 缺 pattern \\S(D2 契约口径)" % key)
|
||
|
||
def test_valid_config_passes(self):
|
||
"""合法 clicker config(取自 spike 实测产物形态)应通过。"""
|
||
ok, errors = judge.validate_config_against_schema(self.good_config(), self.CLICKER_SCHEMA)
|
||
self.assertTrue(ok, errors)
|
||
|
||
def test_target_31_rejected(self):
|
||
"""越界样本(§6.4):target=31 拒绝。"""
|
||
ok, errors = judge.validate_config_against_schema(self.good_config(target=31), self.CLICKER_SCHEMA)
|
||
self.assertFalse(ok)
|
||
|
||
def test_empty_title_rejected(self):
|
||
"""越界样本(§6.4):title="" 与纯空白 title 均拒绝(真 schema pattern \\S + strip 双口径)。"""
|
||
for bad in ("", " ", " "): # 含全角空格(NBSP 类空白防绕过)
|
||
ok, _errors = judge.validate_config_against_schema(self.good_config(title=bad), self.CLICKER_SCHEMA)
|
||
self.assertFalse(ok, "title=%r 应被拒绝" % bad)
|
||
|
||
def test_bool_target_rejected(self):
|
||
"""bool 是 int 子类:target=True 必须拒绝(真源 _int_in 同口径)。"""
|
||
ok, _errors = judge.validate_config_against_schema(self.good_config(target=True), self.CLICKER_SCHEMA)
|
||
self.assertFalse(ok)
|
||
|
||
def test_extra_field_rejected(self):
|
||
"""additionalProperties:false:契约外多余字段拒绝。"""
|
||
ok, _errors = judge.validate_config_against_schema(self.good_config(extra="x"), self.CLICKER_SCHEMA)
|
||
self.assertFalse(ok)
|
||
|
||
def test_wrong_template_id_rejected(self):
|
||
"""templateId const "clicker":跨模板串台(dodge)拒绝(真源 _enum 精确等值口径)。"""
|
||
ok, _errors = judge.validate_config_against_schema(
|
||
self.good_config(templateId="dodge"), self.CLICKER_SCHEMA)
|
||
self.assertFalse(ok)
|
||
|
||
|
||
class TestVerdictSeal(unittest.TestCase):
|
||
"""Verdict 铸章与发布闸验章(§7.1 代码层强制)。"""
|
||
|
||
def test_sealed_accept_passes_gate(self):
|
||
"""judge 铸章的 accept Verdict 应通过验章。"""
|
||
out = judge.judge(True, False, [], good_play_report(), 1, False, 0)
|
||
verdict = judge.build_verdict(out, "d" * 64, 1001, 2001, good_play_report(),
|
||
[], 0, "aigc-trace-x", {"config.clicker-designer": "1.0.0"})
|
||
self.assertTrue(judge.is_judge_accept(verdict))
|
||
# 内部字段不入 Verdict 契约(§9.6 / §16 D5-a:checksumVerifyMethod 仅作裁决输入与账本披露)
|
||
self.assertNotIn("demoFallback", verdict["playReport"])
|
||
self.assertNotIn("score", verdict["playReport"])
|
||
self.assertNotIn("checksumVerifyMethod", verdict["playReport"])
|
||
|
||
def test_play_report_projection_seven_keys_closed_set(self):
|
||
"""playReport 投影=verdict.schema 七字段闭集(additionalProperties:false 的代码层落实)。"""
|
||
out = judge.judge(True, False, [], good_play_report(), 1, False, 0)
|
||
verdict = judge.build_verdict(out, "d" * 64, 1001, 2001, good_play_report(fiveAnd={"x": 1}),
|
||
[], 0, "t", {})
|
||
self.assertEqual(set(verdict["playReport"].keys()),
|
||
{"loaded", "clicks", "gameEnd", "consoleErrors",
|
||
"assetLoadErrors", "packageChecksumVerified", "emitsCaptured"})
|
||
# gameEnd 子投影=两键闭集(completed/durationMs)
|
||
self.assertEqual(set(verdict["playReport"]["gameEnd"].keys()), {"completed", "durationMs"})
|
||
|
||
def test_game_end_duration_null_normalized_to_zero(self):
|
||
"""gameEnd.durationMs 契约为 integer 不可 null:D5 线缆缺失(None)时投影防御归一化为 0。"""
|
||
# 构造 D9 场景(runnable 不过仍携报告):durationMs=None(线缆 duration_ms 缺失的 D5 形态)
|
||
report = good_play_report(gameEnd={"completed": True, "durationMs": None, "extraKey": 1})
|
||
out = judge.judge(True, False, [], report, 2, False, 0) # D9 kill-runnable(③不过)
|
||
verdict = judge.build_verdict(out, "d" * 64, 1001, 2001, report, [], 0, "t", {})
|
||
game_end = verdict["playReport"]["gameEnd"]
|
||
self.assertEqual(game_end["durationMs"], 0) # None → 0(integer 契约)
|
||
self.assertIsInstance(game_end["durationMs"], int)
|
||
self.assertNotIn("extraKey", game_end) # gameEnd 两键闭集(多余键剥除)
|
||
# float 取整防御
|
||
report2 = good_play_report(gameEnd={"completed": True, "durationMs": 1234.7})
|
||
out2 = judge.judge(True, False, [], report2, 1, False, 0) # D10(float>0 仍 runnable)
|
||
verdict2 = judge.build_verdict(out2, "d" * 64, 1001, 2001, report2, [], 0, "t", {})
|
||
self.assertEqual(verdict2["playReport"]["gameEnd"]["durationMs"], 1234)
|
||
self.assertIsInstance(verdict2["playReport"]["gameEnd"]["durationMs"], int)
|
||
|
||
def test_hand_built_dict_rejected(self):
|
||
"""手工拼的 dict(伪造 accept)必须被发布闸拒绝。"""
|
||
fake = {"decision": "accept", "designId": "x"}
|
||
self.assertFalse(judge.is_judge_accept(fake))
|
||
|
||
def test_sealed_kill_rejected_by_gate(self):
|
||
"""铸章但 decision=kill 的 Verdict 不得通过发布闸。"""
|
||
out = judge.judge(False, False, [], None, 0, False, 1) # D2 kill
|
||
verdict = judge.build_verdict(out, "d" * 64, 1001, None, None, [], 1, "t", {})
|
||
self.assertFalse(judge.is_judge_accept(verdict))
|
||
|
||
def test_non_terminal_outcome_cannot_build_verdict(self):
|
||
"""非终判行(D1 schema_retry)禁止铸 Verdict。"""
|
||
out = judge.judge(False, False, [], None, 0, False, 0) # D1
|
||
with self.assertRaises(judge.JudgeError):
|
||
judge.build_verdict(out, "d" * 64, 1001, None, None, [], 0, "t", {})
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|