zizi c9f69d9d6d 治理: Skill 测试治理第一阶段——harness 控制平面 + 实现测试迁出运行时目录
范围(不含 design-story-foundation、docs/、humanization/README.md 等进行中改动):

1. 新增 harness/ 控制平面
   - skill_harness.py 静态审计:32 个运行时 Skill 的 frontmatter/manifest/文档污染,当前 0 问题
   - run_selected.py 选择性执行器:manifest 与磁盘一一对账、依赖阻断、
     空跑与 skip-only 失败关闭、AST 测试形状门
   - manifests/skills.json:32 个 Skill 的合同责任方与协作领域登记
   - manifests/test-inventory.json:81 个测试资产登记
   - specs/skill-testing.md 与 README.md:测试分层、证据边界与 harness 职责

2. 实现测试从 .claude/skills/*/scripts/ 迁至 tests/skills/<skill>/
   - 71 个测试文件迁移并修复项目根与临时目录运行导入
   - 数据库触发器测试宽泛异常收窄为 psycopg.errors.RaiseException
   - 抽取离线大测试拆出真实 PG smoke(默认阻断,不计入离线通过)
   - 抽取 presence 去重边界拆出独立测试:493 + 78 = 571 项检查不变

3. 运行时文档清理
   - 13 个 SKILL.md 移除自测/离线验证段落、测试命令与测试文件事实源表述,
     只保留运行时合同;业务运行合同、额度、授权与离线模式均保留

4. SoT 同步
   - AGENTS.md:新增 Skill 领域索引(7 个合同责任方分组,覆盖 32 个运行时 Skill)
   - 领域 07:测试入口改由 harness/manifests/ 登记,SKILL.md 不承载测试命令
   - humanization 覆盖矩阵:活动测试路径同步迁移

验证证据: harness 自测 15 项 + runner 自测 13 项通过;静态审计 32 Skill / 0 问题;
73 个非数据库测试通过;8 个集成条目中 6 个 PostgreSQL 项被依赖门明确阻断;
py_compile 与 git diff --check 通过。未连接 PostgreSQL、网络、真实模型或额度。

已知边界: 真正 skill_behavior_eval 仍为 0,尚未验证任何 Skill 自然语言行为;
evaluate-frozen-replay 的 raw 存储边界冲突留待单独治理。
2026-08-19 01:50:20 +08:00

159 lines
5.6 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""内容级快照泄露审计的纯函数测试。"""
from __future__ import annotations
import json
import pathlib
import sys
import unittest
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
SCRIPT_DIR = PROJECT_ROOT / ".claude" / "skills" / "freeze-context" / "scripts"
sys.path.insert(0, str(SCRIPT_DIR))
from audit_leakage import audit_snapshot # noqa: E402
def target_facts():
"""提供只供审计侧使用的目标章事实,不进入规划器上下文。"""
return {
"targetChapter": 489,
"forbiddenFacts": [
{
"id": "betrayal-489",
"firstChapter": 489,
"text": "阿铎反水击毙加特朗",
}
],
}
class LeakageAuditTest(unittest.TestCase):
def test_safe_snapshot_is_ready(self):
result = audit_snapshot(
{"events": [{"chapter": 488, "fact": "环星防御仍在"}]},
target_facts(),
as_of=488,
target=489,
)
self.assertTrue(result["ok"])
self.assertEqual(result["status"], "ready")
self.assertEqual(result["findingCount"], 0)
def test_future_nested_record_blocks_snapshot(self):
result = audit_snapshot(
{"events": [{"chapter": 489, "fact": "未来事件"}]},
target_facts(),
as_of=488,
target=489,
)
self.assertFalse(result["ok"])
self.assertEqual(result["status"], "invalid_snapshot")
self.assertEqual(result["findings"][0]["reason"], "future_or_crosses_as_of")
def test_mislabelled_target_fact_blocks_snapshot_without_echoing_text(self):
result = audit_snapshot(
{"events": [{"chapter": 488, "fact": "阿铎反水击毙加特朗"}]},
target_facts(),
as_of=488,
target=489,
)
encoded = json.dumps(result, ensure_ascii=False)
self.assertFalse(result["ok"])
self.assertEqual(result["status"], "invalid_snapshot")
self.assertNotIn("阿铎反水击毙加特朗", encoded)
self.assertEqual(result["findings"][0]["reason"], "target_fact_match")
self.assertIn("factHash", result["findings"][0])
self.assertIn("valueHash", result["findings"][0])
def test_paraphrased_real_target_event_is_blocked(self):
"""真实错标卡不会逐字复制目标细纲,仍须识别同一事件的多个强锚点。"""
facts = {
"targetChapter": 489,
"forbiddenFacts": [
{
"id": "key-events-489",
"firstChapter": 489,
"text": "关键事件:苏铭投两枚核雷撕开缺口、安若雪同步率100%强攻加特朗、铎叔反水贯杀加特朗",
}
],
}
result = audit_snapshot(
{
"cards": [
{
"chapter": 488,
"fact": "与加特朗激战中深陷压力,转交主攻位置后用生命进发让同步率飙至100%,击碎加特朗能量屏障",
}
]
},
facts,
as_of=488,
target=489,
)
encoded = json.dumps(result, ensure_ascii=False)
self.assertFalse(result["ok"])
self.assertEqual(result["findings"][0]["reason"], "target_fact_partial_match")
self.assertNotIn("加特朗", encoded)
self.assertNotIn("同步率", encoded)
def test_related_but_distinct_history_does_not_false_positive(self):
"""只共享旧角色和体系术语、但缺少目标事件关键锚点时不得误报。"""
facts = {
"targetChapter": 489,
"forbiddenFacts": [
{
"id": "key-events-489",
"firstChapter": 489,
"text": "安若雪同步率100%强攻加特朗",
}
],
}
result = audit_snapshot(
{"events": [{"chapter": 387, "fact": "安若雪开启拘束解除,同步率突破95%重创死亡收割者"}]},
facts,
as_of=488,
target=489,
)
self.assertTrue(result["ok"])
self.assertEqual(result["findingCount"], 0)
def test_entities_scattered_across_long_history_do_not_form_one_event(self):
"""多个实体分散在旧细纲的不同事件里,不能仅凭名字凑成目标事件。"""
facts = {
"targetChapter": 489,
"forbiddenFacts": [
{
"id": "network-attack-489",
"firstChapter": 489,
"text": "赛莉丝入虚拟城外圈夺控防御闸门并坑杀凯多萨康里弗两方",
}
],
}
historical_outline = (
"佛罗科密授康里弗追求安若雪以图谋王位;"
"赛莉丝入侵要塞惊见囚禁的高等级灵械后退出;"
"另一场景中凯多萨与康里弗争执,搅乱苏铭和安若雪的会面。"
)
result = audit_snapshot(
{"chapters": [{"chapter": 484, "outline": historical_outline}]},
facts,
as_of=488,
target=489,
)
self.assertTrue(result["ok"])
self.assertEqual(result["findingCount"], 0)
def test_target_and_as_of_must_form_next_chapter_pair(self):
result = audit_snapshot({}, target_facts(), as_of=487, target=489)
self.assertFalse(result["ok"])
self.assertEqual(result["status"], "invalid_audit_input")
if __name__ == "__main__":
unittest.main()