d80df3ed7c
范围（不含 design-story-foundation、docs/design、docs/write-chapter、
craft/、humanization/README.md 等进行中改动）:
1. humanization 规则/样例运行时数据库权威
- db/ddl/111：example_ai_flavor_rule / example_ai_flavor_sample /
example_ai_flavor_rule_event（append-only 生命周期留痕），已应用到 muse-example
- deai/load_db.py：数据库装载器，激活门/重复检测/指纹与文件装载器同源；
数据库失败关闭，不静默回退 Git 文件资产
- humanization/tools/seed_rules_db.py：YAML 种子单事务同步，幂等、
变化留痕、--strict 对 db-only 行失败关闭；真实库已种入 26 规则/107 样例
- prevent/diagnose/revise 生产路径切到数据库读取（--offline 显式读文件），
落库前新鲜度检查与合同声明来源一致；四个 SKILL.md 数据库合同同步
- 真实库验证：规则库指纹与文件种子一致（v-609bc40e21d0b5db），
三个生产脚本端到端从库装载通过
2. PostgreSQL 集成：显式授权后 9/9 通过
- 此前被依赖门阻断的 6 个 _db/smoke 测试全部通过
- extract rollback 冒烟改为回滚事务内自给夹具（pending 窗/草稿缺失时自建），
不再依赖瞬时生产状态；夹具残留核验为 0
3. Skill 行为评测脚手架（真实执行数量仍为 0）
- harness/evals/skill_eval.py：场景合同、六类评测范畴、适配器和结构化裁决报告
- diagnose-ai-flavor 参考场景 4 条 + 管道自测 7 项通过
- 真实模型适配器未授权时以稳定码 EVAL_ADAPTER_UNAVAILABLE 失败关闭；
清单登记 skill_behavior_eval 条目，默认被依赖门阻断
4. evaluate-frozen-replay raw 存储边界冲突
- docs/2026-08-19 备忘录：平台 DB-first 合同（创始人批准）与回放链
仓外 vault 强制的冲突事实、两个选项和裁决前约束；运行时合同未单方面改写
5. harness 自身修复
- runner 对账语义：行为评测入口不参与测试资产双向等值，但登记文件必须存在；
manifest 保留 skill_behavior_eval 布尔字段并校验类型
- 新增 2 条对账回归用例
验证证据: harness 三组自测 15+15+7 通过；静态审计 32 Skill / 0 问题；
76 个非数据库条目通过；9 个 PostgreSQL 集成条目显式授权后通过；
行为评测条目默认阻断；py_compile 与 git diff --check 通过。
未调用真实模型、embedding 或额度；真实行为评测执行数量仍为 0。
110 lines
5.1 KiB
Python
110 lines
5.1 KiB
Python
#!/usr/bin/env python3
|
||
"""Skill 行为评测引擎离线自测：裁决逻辑、失败关闭和报告合同。
|
||
|
||
这些测试只证明评测管道本身正确（harness_self_test）；Skill 行为证据必须
|
||
由真实模型 adapter 在显式授权下产生，仍为 0。
|
||
"""
|
||
import copy
|
||
import json
|
||
import pathlib
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
|
||
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||
sys.path.insert(0, str(PROJECT_ROOT / "harness" / "evals"))
|
||
sys.path.insert(0, str(PROJECT_ROOT / "harness" / "evals" / "skills" / "diagnose-ai-flavor"))
|
||
|
||
import skill_eval as se # noqa: E402
|
||
import run_eval as diagnose_eval # noqa: E402
|
||
|
||
SKILL_DIR = PROJECT_ROOT / ".claude" / "skills" / "diagnose-ai-flavor"
|
||
SCENARIO_FILE = diagnose_eval.SCENARIO_FILE
|
||
|
||
|
||
class EvalEngineTest(unittest.TestCase):
|
||
def test_compliant_observations_pass_and_report_schema_complete(self):
|
||
adapter = se.FakeAdapter(diagnose_eval._FAKE_OBSERVATIONS)
|
||
report = se.run_eval(SKILL_DIR, SCENARIO_FILE, adapter, adapter_name="fake")
|
||
self.assertEqual(report["schema_version"], se.REPORT_SCHEMA)
|
||
self.assertEqual(report["skill"], "diagnose-ai-flavor")
|
||
self.assertEqual(report["scenario_count"], 4)
|
||
self.assertEqual(report["failed"], 0)
|
||
text = (SKILL_DIR / "SKILL.md").read_text(encoding="utf-8")
|
||
import hashlib
|
||
self.assertEqual(
|
||
report["skill_md_sha256"],
|
||
"sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
||
)
|
||
categories = {v["category"] for v in report["verdicts"]}
|
||
self.assertIn("positive_trigger", categories)
|
||
self.assertIn("forbidden_action", categories)
|
||
|
||
def test_violating_observation_fails_with_evidence(self):
|
||
scripted = copy.deepcopy(diagnose_eval._FAKE_OBSERVATIONS)
|
||
# 缺 work_ref 场景若 Agent 返回成功退出，必须裁决失败并给出证据
|
||
scripted["missing-work-ref"]["exit_code"] = 0
|
||
scripted["missing-work-ref"]["output"] = "ok"
|
||
report = se.run_eval(SKILL_DIR, SCENARIO_FILE, se.FakeAdapter(scripted), adapter_name="fake")
|
||
self.assertEqual(report["failed"], 1)
|
||
verdict = next(v for v in report["verdicts"] if v["scenario_id"] == "missing-work-ref")
|
||
self.assertEqual(verdict["verdict"], "failed")
|
||
checks = {f["check"] for f in verdict["failed_checks"]}
|
||
self.assertIn("expected_exit_codes", checks)
|
||
self.assertIn("output_must_contain", checks)
|
||
|
||
def test_forbidden_invocation_is_detected(self):
|
||
scripted = copy.deepcopy(diagnose_eval._FAKE_OBSERVATIONS)
|
||
scripted["forbidden-modify-text"]["invoked_commands"].append(
|
||
".venv/bin/python .claude/skills/revise-ai-flavor/scripts/revise_ai_flavor.py --x"
|
||
)
|
||
report = se.run_eval(SKILL_DIR, SCENARIO_FILE, se.FakeAdapter(scripted), adapter_name="fake")
|
||
verdict = next(v for v in report["verdicts"] if v["scenario_id"] == "forbidden-modify-text")
|
||
self.assertEqual(verdict["verdict"], "failed")
|
||
self.assertIn("must_not_invoke", {f["check"] for f in verdict["failed_checks"]})
|
||
|
||
def test_missing_observation_field_is_not_silent(self):
|
||
scripted = copy.deepcopy(diagnose_eval._FAKE_OBSERVATIONS)
|
||
del scripted["positive-basic-diagnosis"]["exit_code"]
|
||
report = se.run_eval(SKILL_DIR, SCENARIO_FILE, se.FakeAdapter(scripted), adapter_name="fake")
|
||
verdict = next(v for v in report["verdicts"] if v["scenario_id"] == "positive-basic-diagnosis")
|
||
self.assertEqual(verdict["verdict"], "failed")
|
||
self.assertIn("observation_missing", {f["check"] for f in verdict["failed_checks"]})
|
||
|
||
def test_real_adapter_fails_closed_with_stable_code(self):
|
||
with self.assertRaises(se.EvalAdapterUnavailable) as ctx:
|
||
se.ClaudeCliAdapter().run("skill", {"id": "x"})
|
||
self.assertEqual(ctx.exception.code, "EVAL_ADAPTER_UNAVAILABLE")
|
||
|
||
def test_invalid_scenario_structure_is_rejected(self):
|
||
data = json.loads(SCENARIO_FILE.read_text(encoding="utf-8"))
|
||
data["scenarios"][0]["category"] = "not-a-category"
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
bad = pathlib.Path(tmp) / "scenarios.json"
|
||
bad.write_text(json.dumps(data), encoding="utf-8")
|
||
with self.assertRaisesRegex(se.EvalContractError, "category"):
|
||
se.load_scenarios(bad)
|
||
|
||
def test_cli_default_adapter_is_blocked_and_fake_runs(self):
|
||
import contextlib
|
||
import io
|
||
|
||
buffer = io.StringIO()
|
||
with contextlib.redirect_stdout(buffer):
|
||
code_blocked = diagnose_eval.main([])
|
||
self.assertEqual(code_blocked, 2)
|
||
blocked = json.loads(buffer.getvalue())
|
||
self.assertEqual(blocked["status"], "blocked")
|
||
self.assertEqual(blocked["code"], "EVAL_ADAPTER_UNAVAILABLE")
|
||
|
||
buffer = io.StringIO()
|
||
with contextlib.redirect_stdout(buffer):
|
||
code_fake = diagnose_eval.main(["--adapter", "fake"])
|
||
self.assertEqual(code_fake, 0)
|
||
report = json.loads(buffer.getvalue())
|
||
self.assertEqual(report["failed"], 0)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|