131 lines
4.5 KiB
Python
131 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""细纲 detector 机器合同的离线测试。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pathlib
|
|
import sys
|
|
import unittest
|
|
|
|
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
SCRIPT_DIR = PROJECT_ROOT / "muse" / "lifecycle" / "quality" / "skills" / "replay" / "evaluate-frozen-replay" / "scripts"
|
|
if str(SCRIPT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPT_DIR))
|
|
|
|
from fine_outline_detector import validate_detector_report # noqa: E402
|
|
|
|
|
|
class FineOutlineDetectorTest(unittest.TestCase):
|
|
def test_report_must_not_reveal_arm_or_judge_target_role_coverage(self):
|
|
arm_leak = {
|
|
"protocol": "fine_outline_detector_v0",
|
|
"candidateId": "blind-1",
|
|
"arm": "outline_only",
|
|
"findings": [],
|
|
"coverageFindings": [],
|
|
}
|
|
self.assertFalse(validate_detector_report(arm_leak, "blind-1")["ok"])
|
|
|
|
target_role_judgment = {
|
|
"protocol": "fine_outline_detector_v0",
|
|
"candidateId": "blind-1",
|
|
"findings": [],
|
|
"coverageFindings": [
|
|
{"category": "missing_target_role_card", "summary": "越权判断"}
|
|
],
|
|
}
|
|
self.assertFalse(validate_detector_report(target_role_judgment, "blind-1")["ok"])
|
|
|
|
def test_semantic_alias_and_unknown_categories_fail_closed(self):
|
|
for section, category in (
|
|
("findings", "target_new_character_card_missing"),
|
|
("coverageFindings", "new_role_without_card"),
|
|
("findings", "invented_detector_category"),
|
|
("coverageFindings", ["frozen_context_gap"]),
|
|
):
|
|
report = {
|
|
"protocol": "fine_outline_detector_v0",
|
|
"candidateId": "blind-1",
|
|
"findings": [],
|
|
"coverageFindings": [],
|
|
}
|
|
finding = {
|
|
"category": category,
|
|
"severity": "low",
|
|
"location": "candidate",
|
|
"evidenceSummary": "测试",
|
|
}
|
|
report[section] = [finding]
|
|
self.assertFalse(validate_detector_report(report, "blind-1")["ok"])
|
|
|
|
def test_registered_categories_are_accepted_and_detect_skill_assigns_target_coverage_to_eval(self):
|
|
report = {
|
|
"protocol": "fine_outline_detector_v0",
|
|
"candidateId": "blind-1",
|
|
"findings": [
|
|
{
|
|
"category": "entity_state",
|
|
"severity": "medium",
|
|
"location": "entities[0]",
|
|
"evidenceSummary": "与冻结状态不一致",
|
|
}
|
|
],
|
|
"coverageFindings": [
|
|
{
|
|
"category": "frozen_context_gap",
|
|
"summary": "冻结资料缺少已知实体字段",
|
|
}
|
|
],
|
|
}
|
|
self.assertTrue(validate_detector_report(report, "blind-1")["ok"])
|
|
|
|
skill_path = (
|
|
PROJECT_ROOT
|
|
/ "muse"
|
|
/ "lifecycle"
|
|
/ "quality"
|
|
/ "skills"
|
|
/ "semantic"
|
|
/ "check-content-consistency"
|
|
/ "SKILL.md"
|
|
)
|
|
skill = skill_path.read_text(encoding="utf-8")
|
|
self.assertNotIn("要单列为资料覆盖发现", skill)
|
|
self.assertIn("归 judge/eval", skill)
|
|
|
|
def test_nested_findings_reject_arm_and_card_manifest_leakage(self):
|
|
base = {
|
|
"protocol": "fine_outline_detector_v0",
|
|
"candidateId": "blind-1",
|
|
"findings": [
|
|
{
|
|
"category": "entity_state",
|
|
"severity": "low",
|
|
"location": "entities[0]",
|
|
"evidenceSummary": "冻结状态核对",
|
|
}
|
|
],
|
|
"coverageFindings": [
|
|
{
|
|
"category": "frozen_context_gap",
|
|
"summary": "公共冻结事实缺字段",
|
|
}
|
|
],
|
|
}
|
|
leaking_finding = {
|
|
**base,
|
|
"findings": [{**base["findings"][0], "arm": "outline_plus_cards"}],
|
|
}
|
|
leaking_coverage = {
|
|
**base,
|
|
"coverageFindings": [
|
|
{**base["coverageFindings"][0], "cardManifest": {"count": 1}}
|
|
],
|
|
}
|
|
self.assertFalse(validate_detector_report(leaking_finding, "blind-1")["ok"])
|
|
self.assertFalse(validate_detector_report(leaking_coverage, "blind-1")["ok"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|