zizi 208a36ee6c feat(agent-loop): v1 全闭环建设波收口 — D1-D5 五件交付 + 双spec + 种子/烟测产物
- D1: Registry 3 prompt(策划/对抗/fix)+eval 骨架+registry 注册
- D2: agent-loop 2 schema + 4 模板 schema(对齐 gen_spike 真源, additionalProperties:false)
- D3: 编排器+裁决引擎(judge D1-D10 决策表/账本幂等重放/预算闸熔断/eval 回流/批报告), 59 单测绿(mini-desktop)
- D5: 玩家 CDP 取证 player_cdp.py(真点击/蛇形→驼峰映射/demo 三信号/三角合成判定), 9001 自测 PASS
- (D4 后端件已在 9bf3d54 先行入库, 三条 mvn 两轮独立实跑全绿)
- spec: 评审版(已拍板)+execution 版(已审定+建设期补充裁决 D4-a~d/D5-a~c/D2-a/D3-a)
- W2 试点/种子 9003-9008/B4 四链路烟测 产物与总账/作战清单同步
- eval 种子: C1 spike 52 条转入 config.clicker-designer(inputs 52/labels 13)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-10 06:24:43 +08:00

364 lines
20 KiB
Python
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
裁决引擎单测§7.8/§10.3):决策表 D1-D10 全分支 + P2-only + fix 后二轮 D6 + 未匹配 raise。
全组合覆盖论证§10.2,以注释+用例固化):输入空间按求值序短路——
E1 截获全部 schemaOk=falseD1/D2 按 round 二分E2 截获 dupHitD3
E3 截获 hasP0D4E4 截获 hasP1D5/D6 按 round 二分);
E5 截获实玩前后全部 infra 与 runnable 失败D7/D8/D9 按 infraSignal/playAttempt 三分);
E6 仅剩全真组合D10。每用例注释标注对应表行号。
"""
import json
import os
import sys
import unittest
# 保证从任意工作目录可导入被测模块orchestrator/ 上一级即本文件父目录的父目录)
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import judge # noqa: E402
# 仓库根tests/ → orchestrator/ → 2026-06-09-agent-loop-v1/ → agent-specs/ → docs/ → 根,共上溯 5 级)
_REPO = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)),
"..", "..", "..", "..", ".."))
def good_play_report(**overrides):
"""
构造满足五条 AND§9.4)的 PlayReport 桩overrides 用于逐条打破。
含 D5 超集字段 checksumVerifyMethod§16 D5-a⑤ 须核验方法可信none/缺失不得通过)。
"""
report = {
"loaded": True,
"clicks": 15,
"gameEnd": {"completed": True, "durationMs": 2345},
"consoleErrors": [],
"assetLoadErrors": 0,
"packageChecksumVerified": True,
"checksumVerifyMethod": "browser_body", # D5 超集字段:⑤ 可信方法browser_body/synthesized
"emitsCaptured": ["game_loaded", "game_start", "game_end"],
"demoFallback": False,
"score": 15,
}
report.update(overrides)
return report
class TestDecisionTable(unittest.TestCase):
"""§10.2 决策表逐行用例(注释标行号)。"""
def test_d1_schema_fail_round0_local_retry(self):
"""表行 D1schemaOk=false @round=0 → 本地重出(不回调、消耗唯一回炉额度)。"""
out = judge.judge(False, False, [], None, 0, False, 0)
self.assertEqual(out.row, "D1")
self.assertEqual(out.action, judge.ACTION_SCHEMA_RETRY)
self.assertIsNone(out.decision) # 非终判行不产 Verdict
self.assertEqual(out.reasons, ["schema_retry"])
self.assertIsNone(out.callback) # 不回调
def test_d2_schema_fail_round1_kill(self):
"""表行 D2schemaOk=false @round=1 → kill回调 failed+config_invalid。"""
out = judge.judge(False, False, [], None, 0, False, 1)
self.assertEqual(out.row, "D2")
self.assertEqual(out.action, judge.ACTION_KILL)
self.assertEqual(out.decision, judge.DECISION_KILL)
self.assertEqual(out.reasons, ["schema_invalid_after_retry"])
self.assertEqual(out.callback, judge.CALLBACK_FAILED_CONFIG_INVALID)
self.assertFalse(out.structure_ok) # 结构未过
def test_d3_duplicate_kill(self):
"""表行 D3dupHit=true → killround 无关);归桶 config_invalid§16-1真因记 reasons。"""
for round_ in (0, 1):
out = judge.judge(True, True, [], None, 0, False, round_)
self.assertEqual(out.row, "D3")
self.assertEqual(out.action, judge.ACTION_KILL)
self.assertEqual(out.reasons, ["duplicate_in_batch"])
self.assertEqual(out.callback, judge.CALLBACK_FAILED_CONFIG_INVALID)
def test_d4_p0_kill(self):
"""表行 D4∃P0 → 即杀不给 fixround 无关);回调 failed+unsafe_promptreasons=p0_<类别>。"""
findings = [{"severity": "P0", "issue": "包含敏感涉政内容", "suggestion": "整体废弃"}]
for round_ in (0, 1):
out = judge.judge(True, False, findings, None, 0, False, round_)
self.assertEqual(out.row, "D4")
self.assertEqual(out.action, judge.ACTION_KILL)
self.assertEqual(out.callback, judge.CALLBACK_FAILED_UNSAFE_PROMPT)
self.assertEqual(out.reasons, ["p0_sensitive"]) # 确定性关键词归桶
def test_d5_p1_round0_fix(self):
"""表行 D5∃P1 @round=0 → fix不回调本轮不落包decision=fix 可铸中间 Verdict。"""
findings = [{"severity": "P1", "issue": "题文不符", "suggestion": "改 title"}]
out = judge.judge(True, False, findings, None, 0, False, 0)
self.assertEqual(out.row, "D5")
self.assertEqual(out.action, judge.ACTION_FIX)
self.assertEqual(out.decision, judge.DECISION_FIX)
self.assertEqual(out.reasons, ["p1_fix_round"])
self.assertIsNone(out.callback)
def test_d6_p1_round1_kill(self):
"""表行 D6∃P1 @round=1 → kill额度用尽归桶 config_invalid。"""
findings = [{"severity": "P1", "issue": "target 与意图矛盾", "suggestion": "调 target"}]
out = judge.judge(True, False, findings, None, 0, False, 1)
self.assertEqual(out.row, "D6")
self.assertEqual(out.action, judge.ACTION_KILL)
self.assertEqual(out.reasons, ["p1_residual_after_fix"])
self.assertEqual(out.callback, judge.CALLBACK_FAILED_CONFIG_INVALID)
def test_d7_infra_fail(self):
"""表行 D7infraSignal=true → infra_fail不计分母、可重放reasons=infra_<类别>。"""
out = judge.judge(True, False, [], None, 0, True, 0, infra_category="demo_fallback")
self.assertEqual(out.row, "D7")
self.assertEqual(out.action, judge.ACTION_INFRA_FAIL)
self.assertIsNone(out.decision) # infra 不产 Verdictresume 从 submit 重走)
self.assertEqual(out.reasons, ["infra_demo_fallback"])
# 类别缺省时兜底 unknown
out2 = judge.judge(True, False, [], None, 0, True, 1)
self.assertEqual(out2.reasons, ["infra_unknown"])
def test_d8_runnable_fail_attempt1_retry(self):
"""表行 D8runnableOk=false ∧ 非 infra ∧ playAttempt=1 → 重试实玩(不消耗回炉额度)。"""
report = good_play_report(gameEnd={"completed": False, "durationMs": 1200})
out = judge.judge(True, False, [], report, 1, False, 0)
self.assertEqual(out.row, "D8")
self.assertEqual(out.action, judge.ACTION_PLAY_RETRY)
self.assertEqual(out.reasons, ["runnable_retry"])
def test_d9_runnable_fail_attempt2_kill(self):
"""表行 D9runnableOk=false ∧ playAttempt=2 → kill-runnable任务保持 succeeded不回调"""
report = good_play_report(loaded=False, emitsCaptured=[])
out = judge.judge(True, False, [], report, 2, False, 0)
self.assertEqual(out.row, "D9")
self.assertEqual(out.action, judge.ACTION_KILL)
self.assertEqual(out.decision, judge.DECISION_KILL)
self.assertEqual(out.reasons, ["runnable_fail_after_retry"])
self.assertIsNone(out.callback) # 不再回调(任务已 succeeded
def test_d10_accept(self):
"""表行 D10四条 AND 全真 → acceptstructureOk ∧ runnableOk 同真)。"""
out = judge.judge(True, False, [], good_play_report(), 1, False, 0)
self.assertEqual(out.row, "D10")
self.assertEqual(out.action, judge.ACTION_ACCEPT)
self.assertEqual(out.decision, judge.DECISION_ACCEPT)
self.assertTrue(out.structure_ok)
self.assertTrue(out.runnable_ok)
self.assertEqual(out.reasons, ["accept"])
def test_p2_only_goes_accept(self):
"""§10.2 注P2-only 组合归 D10 路径P2 不挡 accept仅留档"""
findings = [{"severity": "P2", "issue": "文案可更生动", "suggestion": "润色"}]
out = judge.judge(True, False, findings, good_play_report(), 1, False, 0)
self.assertEqual(out.row, "D10")
self.assertEqual(out.decision, judge.DECISION_ACCEPT)
def test_fix_then_round1_new_p1_goes_d6(self):
"""§10.2 注fix 重走后的二轮链路——round=0 P1→D5(fix)round=1 新 P1→D6(kill)。"""
p1 = [{"severity": "P1", "issue": "文案占位感", "suggestion": "重写"}]
first = judge.judge(True, False, p1, None, 0, False, 0)
self.assertEqual(first.row, "D5") # 首轮回炉
new_p1 = [{"severity": "P1", "issue": "仍题文不符", "suggestion": "再改"}]
second = judge.judge(True, False, new_p1, None, 0, False, 1)
self.assertEqual(second.row, "D6") # 二轮 P1 残留即杀
self.assertEqual(second.reasons, ["p1_residual_after_fix"])
def test_unmatched_input_raises(self):
"""§10.2 末尾防御:文本面全过、无 infra、无实玩报告 → 决策表无此组合,必须 raise。"""
with self.assertRaises(judge.JudgeError):
judge.judge(True, False, [], None, 0, False, 0)
# ---------------- 防御面补充用例(决策表定义域边界) ----------------
def test_invalid_round_raises(self):
"""防御round 越出 0/1回炉额度合计 1 轮)立即暴露。"""
with self.assertRaises(judge.JudgeError):
judge.judge(True, False, [], good_play_report(), 1, False, 2)
def test_play_attempt_3_raises(self):
"""防御playAttempt=3 越出决策表(实玩预算闸应已拦截)→ raise。"""
report = good_play_report(consoleErrors=["TypeError: boom"])
with self.assertRaises(judge.JudgeError):
judge.judge(True, False, [], report, 3, False, 0)
def test_demo_fallback_without_infra_signal_raises(self):
"""防御demoFallback=true 必须以 infraSignal 上报§9.5),漏报即编排器缺陷 → raise。"""
with self.assertRaises(judge.JudgeError):
judge.judge(True, False, [], good_play_report(demoFallback=True), 1, False, 0)
def test_invalid_severity_raises(self):
"""防御findings.severity 越出 P0/P1/P2 受控枚举 → raise不静默放行 LLM 噪声)。"""
with self.assertRaises(judge.JudgeError):
judge.judge(True, False, [{"severity": "P3", "issue": "x"}], None, 0, False, 0)
class TestRunnableFiveAnd(unittest.TestCase):
"""§9.4 五条 AND 逐条打破用例D8 路径的归因条款)。"""
def test_clause3_duration_zero_not_runnable(self):
"""③ durationMs=0防 startMs 缺省假象)→ runnable=false。"""
report = good_play_report(gameEnd={"completed": True, "durationMs": 0})
ok, failed = judge.compute_runnable_ok(report)
self.assertFalse(ok)
self.assertTrue(any("" in c for c in failed))
def test_clause4_game_error_emit_not_runnable(self):
"""④ 捕获 game_error 信封 → runnable=false。"""
report = good_play_report(emitsCaptured=["game_loaded", "game_error", "game_end"])
ok, failed = judge.compute_runnable_ok(report)
self.assertFalse(ok)
self.assertTrue(any("game_error" in c for c in failed))
def test_clause5_checksum_unverified_not_runnable(self):
"""⑤ packageChecksumVerified=false → runnable=false防 demo 假阳性的契约位)。"""
ok, failed = judge.compute_runnable_ok(good_play_report(packageChecksumVerified=False))
self.assertFalse(ok)
self.assertTrue(any("" in c for c in failed))
def test_clause5_verify_method_none_not_runnable(self):
"""⑤ §16 D5-apackageChecksumVerified=true 但 checksumVerifyMethod=none或缺失不得通过。"""
# method=noneD5 报告自相矛盾true 却无可信核验法)→ ⑤ 判不过
ok, failed = judge.compute_runnable_ok(good_play_report(checksumVerifyMethod="none"))
self.assertFalse(ok)
self.assertTrue(any("D5-a" in c for c in failed))
# method 字段缺失(旧形状/异常报告):安全侧同判不过
report = good_play_report()
del report["checksumVerifyMethod"]
ok2, failed2 = judge.compute_runnable_ok(report)
self.assertFalse(ok2)
self.assertTrue(any("" in c for c in failed2))
def test_clause5_synthesized_method_passes(self):
"""⑤ §16 D5-a三角合成判定checksumVerifyMethod=synthesized为可信方法应通过。"""
ok, failed = judge.compute_runnable_ok(good_play_report(checksumVerifyMethod="synthesized"))
self.assertTrue(ok, failed)
def test_asset_load_errors_do_not_veto(self):
"""④ 注assetLoadErrors>0 仅记录不否决v1 assets=[])。"""
ok, failed = judge.compute_runnable_ok(good_play_report(assetLoadErrors=2))
self.assertTrue(ok)
self.assertEqual(failed, [])
class TestSchemaValidator(unittest.TestCase):
"""
模板 schema 深度校验(结构门权威;镜像 §6.4 验收口径)。
夹具=直接加载 D2 真契约文件 contracts/templates/clicker.schema.json
D2 核验意见落实:内联夹具曾缺 pattern \\S改为以生产契约为唯一被测口径——
title/theme/scoreLabel 的「非纯空白」由真 schema 的 pattern \\S 与校验器 strip 语义双重保障)。
"""
@classmethod
def setUpClass(cls):
schema_path = os.path.join(_REPO, "contracts", "templates", "clicker.schema.json")
with open(schema_path, "r", encoding="utf-8") as fh:
cls.CLICKER_SCHEMA = json.load(fh)
def good_config(self, **overrides):
config = {"templateId": "clicker", "title": "深夜便利店", "theme": "夜宵",
"target": 12, "scoreLabel": "已服务顾客"}
config.update(overrides)
return config
def test_real_schema_has_nonblank_pattern(self):
"""前置自检D2 真 schema 的三个字符串字段必须带 pattern \\S非纯空白契约位"""
props = self.CLICKER_SCHEMA["properties"]
for key in ("title", "theme", "scoreLabel"):
self.assertEqual(props[key].get("pattern"), "\\S",
"clicker.schema.json 的 %s 缺 pattern \\SD2 契约口径)" % key)
def test_valid_config_passes(self):
"""合法 clicker config取自 spike 实测产物形态)应通过。"""
ok, errors = judge.validate_config_against_schema(self.good_config(), self.CLICKER_SCHEMA)
self.assertTrue(ok, errors)
def test_target_31_rejected(self):
"""越界样本§6.4target=31 拒绝。"""
ok, errors = judge.validate_config_against_schema(self.good_config(target=31), self.CLICKER_SCHEMA)
self.assertFalse(ok)
def test_empty_title_rejected(self):
"""越界样本§6.4title="" 与纯空白 title 均拒绝(真 schema pattern \\S + strip 双口径)。"""
for bad in ("", " ", "  "): # 含全角空格NBSP 类空白防绕过)
ok, _errors = judge.validate_config_against_schema(self.good_config(title=bad), self.CLICKER_SCHEMA)
self.assertFalse(ok, "title=%r 应被拒绝" % bad)
def test_bool_target_rejected(self):
"""bool 是 int 子类target=True 必须拒绝(真源 _int_in 同口径)。"""
ok, _errors = judge.validate_config_against_schema(self.good_config(target=True), self.CLICKER_SCHEMA)
self.assertFalse(ok)
def test_extra_field_rejected(self):
"""additionalProperties:false契约外多余字段拒绝。"""
ok, _errors = judge.validate_config_against_schema(self.good_config(extra="x"), self.CLICKER_SCHEMA)
self.assertFalse(ok)
def test_wrong_template_id_rejected(self):
"""templateId const "clicker"跨模板串台dodge拒绝真源 _enum 精确等值口径)。"""
ok, _errors = judge.validate_config_against_schema(
self.good_config(templateId="dodge"), self.CLICKER_SCHEMA)
self.assertFalse(ok)
class TestVerdictSeal(unittest.TestCase):
"""Verdict 铸章与发布闸验章§7.1 代码层强制)。"""
def test_sealed_accept_passes_gate(self):
"""judge 铸章的 accept Verdict 应通过验章。"""
out = judge.judge(True, False, [], good_play_report(), 1, False, 0)
verdict = judge.build_verdict(out, "d" * 64, 1001, 2001, good_play_report(),
[], 0, "aigc-trace-x", {"config.clicker-designer": "1.0.0"})
self.assertTrue(judge.is_judge_accept(verdict))
# 内部字段不入 Verdict 契约§9.6 / §16 D5-achecksumVerifyMethod 仅作裁决输入与账本披露)
self.assertNotIn("demoFallback", verdict["playReport"])
self.assertNotIn("score", verdict["playReport"])
self.assertNotIn("checksumVerifyMethod", verdict["playReport"])
def test_play_report_projection_seven_keys_closed_set(self):
"""playReport 投影=verdict.schema 七字段闭集additionalProperties:false 的代码层落实)。"""
out = judge.judge(True, False, [], good_play_report(), 1, False, 0)
verdict = judge.build_verdict(out, "d" * 64, 1001, 2001, good_play_report(fiveAnd={"x": 1}),
[], 0, "t", {})
self.assertEqual(set(verdict["playReport"].keys()),
{"loaded", "clicks", "gameEnd", "consoleErrors",
"assetLoadErrors", "packageChecksumVerified", "emitsCaptured"})
# gameEnd 子投影=两键闭集completed/durationMs
self.assertEqual(set(verdict["playReport"]["gameEnd"].keys()), {"completed", "durationMs"})
def test_game_end_duration_null_normalized_to_zero(self):
"""gameEnd.durationMs 契约为 integer 不可 nullD5 线缆缺失None时投影防御归一化为 0。"""
# 构造 D9 场景runnable 不过仍携报告durationMs=None线缆 duration_ms 缺失的 D5 形态)
report = good_play_report(gameEnd={"completed": True, "durationMs": None, "extraKey": 1})
out = judge.judge(True, False, [], report, 2, False, 0) # D9 kill-runnable③不过
verdict = judge.build_verdict(out, "d" * 64, 1001, 2001, report, [], 0, "t", {})
game_end = verdict["playReport"]["gameEnd"]
self.assertEqual(game_end["durationMs"], 0) # None → 0integer 契约)
self.assertIsInstance(game_end["durationMs"], int)
self.assertNotIn("extraKey", game_end) # gameEnd 两键闭集(多余键剥除)
# float 取整防御
report2 = good_play_report(gameEnd={"completed": True, "durationMs": 1234.7})
out2 = judge.judge(True, False, [], report2, 1, False, 0) # D10float>0 仍 runnable
verdict2 = judge.build_verdict(out2, "d" * 64, 1001, 2001, report2, [], 0, "t", {})
self.assertEqual(verdict2["playReport"]["gameEnd"]["durationMs"], 1234)
self.assertIsInstance(verdict2["playReport"]["gameEnd"]["durationMs"], int)
def test_hand_built_dict_rejected(self):
"""手工拼的 dict伪造 accept必须被发布闸拒绝。"""
fake = {"decision": "accept", "designId": "x"}
self.assertFalse(judge.is_judge_accept(fake))
def test_sealed_kill_rejected_by_gate(self):
"""铸章但 decision=kill 的 Verdict 不得通过发布闸。"""
out = judge.judge(False, False, [], None, 0, False, 1) # D2 kill
verdict = judge.build_verdict(out, "d" * 64, 1001, None, None, [], 1, "t", {})
self.assertFalse(judge.is_judge_accept(verdict))
def test_non_terminal_outcome_cannot_build_verdict(self):
"""非终判行D1 schema_retry禁止铸 Verdict。"""
out = judge.judge(False, False, [], None, 0, False, 0) # D1
with self.assertRaises(judge.JudgeError):
judge.build_verdict(out, "d" * 64, 1001, None, None, [], 0, "t", {})
if __name__ == "__main__":
unittest.main()