091b66a9bb
将角色与 Skill 从 .claude 迁入 .agent，移除 Claude CLI 运行时并接入固定 Opus 角色 profile、完整 schema、预算 deadline、raw 与回执证据链。 同步拆分 Skill 职责、复利 lesson、Gate 回放、Dashboard 人审入口、数据库登记和机械门禁；候选设计正文不包含在本提交中。
110 lines
5.1 KiB
Python
110 lines
5.1 KiB
Python
#!/usr/bin/env python3
|
||
"""Skill 行为评测引擎离线自测：裁决逻辑、失败关闭和报告合同。
|
||
|
||
这些测试只证明评测管道本身正确（harness_self_test）；Skill 行为证据必须
|
||
由真实模型 adapter 在显式授权下产生，仍为 0。
|
||
"""
|
||
import copy
|
||
import json
|
||
import pathlib
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
|
||
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||
sys.path.insert(0, str(PROJECT_ROOT / "harness" / "evals"))
|
||
sys.path.insert(0, str(PROJECT_ROOT / "harness" / "evals" / "skills" / "diagnose-ai-flavor"))
|
||
|
||
import skill_eval as se # noqa: E402
|
||
import run_eval as diagnose_eval # noqa: E402
|
||
|
||
SKILL_DIR = PROJECT_ROOT / ".agent" / "skills" / "diagnose-ai-flavor"
|
||
SCENARIO_FILE = diagnose_eval.SCENARIO_FILE
|
||
|
||
|
||
class EvalEngineTest(unittest.TestCase):
|
||
def test_compliant_observations_pass_and_report_schema_complete(self):
|
||
adapter = se.FakeAdapter(diagnose_eval._FAKE_OBSERVATIONS)
|
||
report = se.run_eval(SKILL_DIR, SCENARIO_FILE, adapter, adapter_name="fake")
|
||
self.assertEqual(report["schema_version"], se.REPORT_SCHEMA)
|
||
self.assertEqual(report["skill"], "diagnose-ai-flavor")
|
||
self.assertEqual(report["scenario_count"], 4)
|
||
self.assertEqual(report["failed"], 0)
|
||
text = (SKILL_DIR / "SKILL.md").read_text(encoding="utf-8")
|
||
import hashlib
|
||
self.assertEqual(
|
||
report["skill_md_sha256"],
|
||
"sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
||
)
|
||
categories = {v["category"] for v in report["verdicts"]}
|
||
self.assertIn("positive_trigger", categories)
|
||
self.assertIn("forbidden_action", categories)
|
||
|
||
def test_violating_observation_fails_with_evidence(self):
|
||
scripted = copy.deepcopy(diagnose_eval._FAKE_OBSERVATIONS)
|
||
# 缺 work_ref 场景若 Agent 返回成功退出，必须裁决失败并给出证据
|
||
scripted["missing-work-ref"]["exit_code"] = 0
|
||
scripted["missing-work-ref"]["output"] = "ok"
|
||
report = se.run_eval(SKILL_DIR, SCENARIO_FILE, se.FakeAdapter(scripted), adapter_name="fake")
|
||
self.assertEqual(report["failed"], 1)
|
||
verdict = next(v for v in report["verdicts"] if v["scenario_id"] == "missing-work-ref")
|
||
self.assertEqual(verdict["verdict"], "failed")
|
||
checks = {f["check"] for f in verdict["failed_checks"]}
|
||
self.assertIn("expected_exit_codes", checks)
|
||
self.assertIn("output_must_contain", checks)
|
||
|
||
def test_forbidden_invocation_is_detected(self):
|
||
scripted = copy.deepcopy(diagnose_eval._FAKE_OBSERVATIONS)
|
||
scripted["forbidden-modify-text"]["invoked_commands"].append(
|
||
".venv/bin/python .agent/skills/revise-ai-flavor/scripts/revise_ai_flavor.py --x"
|
||
)
|
||
report = se.run_eval(SKILL_DIR, SCENARIO_FILE, se.FakeAdapter(scripted), adapter_name="fake")
|
||
verdict = next(v for v in report["verdicts"] if v["scenario_id"] == "forbidden-modify-text")
|
||
self.assertEqual(verdict["verdict"], "failed")
|
||
self.assertIn("must_not_invoke", {f["check"] for f in verdict["failed_checks"]})
|
||
|
||
def test_missing_observation_field_is_not_silent(self):
|
||
scripted = copy.deepcopy(diagnose_eval._FAKE_OBSERVATIONS)
|
||
del scripted["positive-basic-diagnosis"]["exit_code"]
|
||
report = se.run_eval(SKILL_DIR, SCENARIO_FILE, se.FakeAdapter(scripted), adapter_name="fake")
|
||
verdict = next(v for v in report["verdicts"] if v["scenario_id"] == "positive-basic-diagnosis")
|
||
self.assertEqual(verdict["verdict"], "failed")
|
||
self.assertIn("observation_missing", {f["check"] for f in verdict["failed_checks"]})
|
||
|
||
def test_real_adapter_fails_closed_with_stable_code(self):
|
||
with self.assertRaises(se.EvalAdapterUnavailable) as ctx:
|
||
se.RoleAgentAdapter().run("skill", {"id": "x"})
|
||
self.assertEqual(ctx.exception.code, "EVAL_ADAPTER_UNAVAILABLE")
|
||
|
||
def test_invalid_scenario_structure_is_rejected(self):
|
||
data = json.loads(SCENARIO_FILE.read_text(encoding="utf-8"))
|
||
data["scenarios"][0]["category"] = "not-a-category"
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
bad = pathlib.Path(tmp) / "scenarios.json"
|
||
bad.write_text(json.dumps(data), encoding="utf-8")
|
||
with self.assertRaisesRegex(se.EvalContractError, "category"):
|
||
se.load_scenarios(bad)
|
||
|
||
def test_cli_default_adapter_is_blocked_and_fake_runs(self):
|
||
import contextlib
|
||
import io
|
||
|
||
buffer = io.StringIO()
|
||
with contextlib.redirect_stdout(buffer):
|
||
code_blocked = diagnose_eval.main([])
|
||
self.assertEqual(code_blocked, 2)
|
||
blocked = json.loads(buffer.getvalue())
|
||
self.assertEqual(blocked["status"], "blocked")
|
||
self.assertEqual(blocked["code"], "EVAL_ADAPTER_UNAVAILABLE")
|
||
|
||
buffer = io.StringIO()
|
||
with contextlib.redirect_stdout(buffer):
|
||
code_fake = diagnose_eval.main(["--adapter", "fake"])
|
||
self.assertEqual(code_fake, 0)
|
||
report = json.loads(buffer.getvalue())
|
||
self.assertEqual(report["failed"], 0)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|