c9f69d9d6d
范围（不含 design-story-foundation、docs/、humanization/README.md 等进行中改动）:
1. 新增 harness/ 控制平面
- skill_harness.py 静态审计：32 个运行时 Skill 的 frontmatter/manifest/文档污染，当前 0 问题
- run_selected.py 选择性执行器：manifest 与磁盘一一对账、依赖阻断、
空跑与 skip-only 失败关闭、AST 测试形状门
- manifests/skills.json：32 个 Skill 的合同责任方与协作领域登记
- manifests/test-inventory.json：81 个测试资产登记
- specs/skill-testing.md 与 README.md：测试分层、证据边界与 harness 职责
2. 实现测试从 .claude/skills/*/scripts/ 迁至 tests/skills/<skill>/
- 71 个测试文件迁移并修复项目根与临时目录运行导入
- 数据库触发器测试宽泛异常收窄为 psycopg.errors.RaiseException
- 抽取离线大测试拆出真实 PG smoke（默认阻断，不计入离线通过）
- 抽取 presence 去重边界拆出独立测试：493 + 78 = 571 项检查不变
3. 运行时文档清理
- 13 个 SKILL.md 移除自测/离线验证段落、测试命令与测试文件事实源表述，
只保留运行时合同；业务运行合同、额度、授权与离线模式均保留
4. SoT 同步
- AGENTS.md：新增 Skill 领域索引（7 个合同责任方分组，覆盖 32 个运行时 Skill）
- 领域 07：测试入口改由 harness/manifests/ 登记，SKILL.md 不承载测试命令
- humanization 覆盖矩阵：活动测试路径同步迁移
验证证据: harness 自测 15 项 + runner 自测 13 项通过；静态审计 32 Skill / 0 问题；
73 个非数据库测试通过；8 个集成条目中 6 个 PostgreSQL 项被依赖门明确阻断；
py_compile 与 git diff --check 通过。未连接 PostgreSQL、网络、真实模型或额度。
已知边界: 真正 skill_behavior_eval 仍为 0，尚未验证任何 Skill 自然语言行为；
evaluate-frozen-replay 的 raw 存储边界冲突留待单独治理。
383 lines
15 KiB
Python
383 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""check_snapshot.py 的无网络离线测试。"""
|
|
|
|
import pathlib
|
|
import sys
|
|
import unittest
|
|
|
|
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
SCRIPT_DIR = PROJECT_ROOT / ".claude" / "skills" / "freeze-context" / "scripts"
|
|
sys.path.insert(0, str(SCRIPT_DIR))
|
|
from check_snapshot import ( # noqa: E402
|
|
STATUS_BLOCKED_AUTHORIZATION,
|
|
STATUS_INVALID_ARM_DIFF,
|
|
STATUS_READY,
|
|
STATUS_SCHEMA_INVALID,
|
|
STATUS_TARGET_SOURCE_FORBIDDEN,
|
|
check_arm_manifests,
|
|
check_authorization,
|
|
check_candidate_output,
|
|
check_replay,
|
|
check_target_sources,
|
|
)
|
|
|
|
|
|
AUTH = {
|
|
"sourceStatus": "active",
|
|
"copyrightStatus": "research_only",
|
|
"sourceHash": "sha256:02cf1f8c1ca03c26e0b839d88fe536e83c0af20fd8972235b7aedca6a33becf4",
|
|
"sourceVersion": "raw-file-v1:sha256:02cf1f8c1ca03c26e0b839d88fe536e83c0af20fd8972235b7aedca6a33becf4",
|
|
"allowedPurpose": ["offline_evaluation"],
|
|
"forbiddenPurpose": ["external_distribution"],
|
|
"authorizationSnapshot": {
|
|
"id": "auth-1",
|
|
"version": "v1",
|
|
"immutable": True,
|
|
"sourceHash": "sha256:02cf1f8c1ca03c26e0b839d88fe536e83c0af20fd8972235b7aedca6a33becf4",
|
|
"sourceVersion": "raw-file-v1:sha256:02cf1f8c1ca03c26e0b839d88fe536e83c0af20fd8972235b7aedca6a33becf4",
|
|
"sourceStatus": "active",
|
|
"copyrightStatus": "research_only",
|
|
"authorizationBasis": "user_authorization",
|
|
"allowedPurpose": ["offline_evaluation"],
|
|
"forbiddenPurpose": ["external_distribution"],
|
|
"checkedAt": "2026-07-19T00:00:00Z",
|
|
"revalidationAt": "2099-07-20T00:00:00Z",
|
|
},
|
|
}
|
|
|
|
|
|
def arm(common, cards):
|
|
return {**common, "cardInjection": cards}
|
|
|
|
|
|
class CheckSnapshotTest(unittest.TestCase):
|
|
def test_authorization_is_fail_closed(self):
|
|
self.assertEqual(check_authorization(None)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
self.assertTrue(check_authorization(AUTH)["ok"])
|
|
self.assertEqual(check_authorization({"sourceStatus": "active"})["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
denied = {**AUTH, "allowedPurpose": ["read"]}
|
|
self.assertEqual(check_authorization(denied)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
unknown_status = {**AUTH, "sourceStatus": "temporary"}
|
|
self.assertEqual(check_authorization(unknown_status)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
unauthorized = {**AUTH, "copyrightStatus": "unauthorized"}
|
|
self.assertEqual(check_authorization(unauthorized)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
|
|
def test_research_only_and_public_domain_allow_offline_evaluation(self):
|
|
self.assertTrue(check_authorization(AUTH)["ok"])
|
|
public_domain = {
|
|
**AUTH,
|
|
"copyrightStatus": "public_domain",
|
|
"authorizationSnapshot": {
|
|
**AUTH["authorizationSnapshot"],
|
|
"copyrightStatus": "public_domain",
|
|
"authorizationBasis": "public_domain_record",
|
|
},
|
|
}
|
|
self.assertTrue(check_authorization(public_domain)["ok"])
|
|
|
|
def test_sanitized_contract_fixture_uses_distinct_hashed_version(self):
|
|
source_hash = "sha256:" + "b" * 64
|
|
fixture = {
|
|
**AUTH,
|
|
"sourceHash": source_hash,
|
|
"sourceVersion": f"sanitized-fixture-v1:{source_hash}",
|
|
"authorizationSnapshot": {
|
|
**AUTH["authorizationSnapshot"],
|
|
"sourceHash": source_hash,
|
|
"sourceVersion": f"sanitized-fixture-v1:{source_hash}",
|
|
"authorizationBasis": "sanitized_contract_fixture",
|
|
},
|
|
}
|
|
self.assertTrue(check_authorization(fixture)["ok"])
|
|
|
|
forged_raw = {**fixture, "sourceVersion": f"raw-file-v1:{source_hash}"}
|
|
self.assertEqual(
|
|
check_authorization(forged_raw)["status"], STATUS_BLOCKED_AUTHORIZATION
|
|
)
|
|
|
|
def test_user_authorization_cannot_be_forged_as_licensed(self):
|
|
forged = {
|
|
**AUTH,
|
|
"copyrightStatus": "licensed",
|
|
"authorizationSnapshot": {
|
|
**AUTH["authorizationSnapshot"],
|
|
"copyrightStatus": "licensed",
|
|
},
|
|
}
|
|
self.assertEqual(check_authorization(forged)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
|
|
def test_source_hash_and_version_must_match_snapshot(self):
|
|
bad_hash = {**AUTH, "sourceHash": "sha256:" + "0" * 64}
|
|
self.assertEqual(check_authorization(bad_hash)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
bad_version = {**AUTH, "sourceVersion": "raw-file-v1:sha256:" + "0" * 64}
|
|
self.assertEqual(check_authorization(bad_version)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
|
|
def test_forbidden_purpose_must_be_arrays_and_match_snapshot(self):
|
|
outer_not_array = {**AUTH, "forbiddenPurpose": "external_distribution"}
|
|
self.assertEqual(check_authorization(outer_not_array)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
|
|
snapshot_not_array = {
|
|
**AUTH,
|
|
"authorizationSnapshot": {
|
|
**AUTH["authorizationSnapshot"],
|
|
"forbiddenPurpose": "external_distribution",
|
|
},
|
|
}
|
|
self.assertEqual(check_authorization(snapshot_not_array)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
|
|
mismatch = {
|
|
**AUTH,
|
|
"authorizationSnapshot": {
|
|
**AUTH["authorizationSnapshot"],
|
|
"forbiddenPurpose": ["training"],
|
|
},
|
|
}
|
|
self.assertEqual(check_authorization(mismatch)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
|
|
def test_forbidden_purpose_blocks_overlap_and_offline_evaluation(self):
|
|
overlapping = {
|
|
**AUTH,
|
|
"forbiddenPurpose": ["offline_evaluation"],
|
|
"authorizationSnapshot": {
|
|
**AUTH["authorizationSnapshot"],
|
|
"forbiddenPurpose": ["offline_evaluation"],
|
|
},
|
|
}
|
|
self.assertEqual(check_authorization(overlapping)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
|
|
public_domain_overlap = {
|
|
**AUTH,
|
|
"copyrightStatus": "public_domain",
|
|
"allowedPurpose": ["offline_evaluation", "research"],
|
|
"forbiddenPurpose": ["research"],
|
|
"authorizationSnapshot": {
|
|
**AUTH["authorizationSnapshot"],
|
|
"copyrightStatus": "public_domain",
|
|
"authorizationBasis": "public_domain_record",
|
|
"allowedPurpose": ["offline_evaluation", "research"],
|
|
"forbiddenPurpose": ["research"],
|
|
},
|
|
}
|
|
self.assertEqual(check_authorization(public_domain_overlap)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
|
|
def test_expired_or_due_revalidation_is_blocked(self):
|
|
expired = {
|
|
**AUTH,
|
|
"authorizationSnapshot": {
|
|
**AUTH["authorizationSnapshot"],
|
|
"expiresAt": "2020-01-01T00:00:00Z",
|
|
"revalidationAt": None,
|
|
},
|
|
}
|
|
self.assertEqual(check_authorization(expired)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
due = {
|
|
**AUTH,
|
|
"authorizationSnapshot": {
|
|
**AUTH["authorizationSnapshot"],
|
|
"revalidationAt": "2020-01-01T00:00:00Z",
|
|
},
|
|
}
|
|
self.assertEqual(check_authorization(due)["status"], STATUS_BLOCKED_AUTHORIZATION)
|
|
|
|
def test_target_source_is_blocked(self):
|
|
allowed = check_target_sources(
|
|
489,
|
|
[
|
|
{"sourceId": "chapter-488", "sourceVersion": "v1", "chapter": 488},
|
|
{"sourceId": "outline-450-482", "sourceVersion": "v1", "from_order": 450, "to_order": 482},
|
|
],
|
|
)
|
|
self.assertEqual(allowed["status"], STATUS_READY)
|
|
blocked = check_target_sources(489, [{"sourceId": "chapter-489", "sourceVersion": "v1", "chapter": 489}])
|
|
self.assertEqual(blocked["status"], STATUS_TARGET_SOURCE_FORBIDDEN)
|
|
|
|
def test_arm_common_input_must_match(self):
|
|
common = {"snapshotVersion": "v0", "asOfChapter": 488, "l0": {"target": 489}}
|
|
manifests = {
|
|
"outline_only": arm(common, []),
|
|
"outline_plus_cards": arm(common, [{"id": 1}]),
|
|
"outline_plus_placebo_cards": arm(common, [{"id": 2}]),
|
|
}
|
|
self.assertTrue(check_arm_manifests(manifests)["ok"])
|
|
extra = {**manifests, "unregistered_arm": arm(common, [])}
|
|
self.assertEqual(check_arm_manifests(extra)["status"], STATUS_INVALID_ARM_DIFF)
|
|
changed = dict(manifests)
|
|
changed["outline_plus_cards"] = arm({**common, "l0": {"target": 490}}, [{"id": 1}])
|
|
self.assertEqual(check_arm_manifests(changed)["status"], STATUS_INVALID_ARM_DIFF)
|
|
|
|
def test_two_arm_smoke_can_be_explicit(self):
|
|
common = {"snapshotVersion": "v0", "asOfChapter": 488}
|
|
manifests = {"outline_only": arm(common, []), "outline_plus_cards": arm(common, [{"id": 1}])}
|
|
self.assertTrue(check_arm_manifests(manifests, ["outline_only", "outline_plus_cards"], smoke=True)["ok"])
|
|
|
|
def test_candidate_contract_is_structural(self):
|
|
candidate = {
|
|
"targetChapter": 489,
|
|
"chapterGoal": "突破",
|
|
"keyEvents": [
|
|
{
|
|
"id": "event-1",
|
|
"order": 1,
|
|
"event": "侦察敌情",
|
|
"participants": ["苏铭"],
|
|
"trigger": "收到异常信号",
|
|
"resultDirection": "确认威胁存在",
|
|
}
|
|
],
|
|
"entities": [],
|
|
"foreshadowing": [],
|
|
"stateChanges": [],
|
|
"hook": "悬念",
|
|
"unknowns": [],
|
|
"assumptions": [],
|
|
}
|
|
self.assertTrue(check_candidate_output(candidate, 489)["ok"])
|
|
bad = {**candidate, "正文全文": "原文"}
|
|
self.assertEqual(check_candidate_output(bad, 489)["status"], STATUS_SCHEMA_INVALID)
|
|
bad_type = {**candidate, "keyEvents": "事件"}
|
|
self.assertEqual(check_candidate_output(bad_type, 489)["status"], STATUS_SCHEMA_INVALID)
|
|
duplicate = {**candidate, "keyEvents": [{"id": "event-1"}, {"id": "event-1"}]}
|
|
self.assertEqual(check_candidate_output(duplicate, 489)["status"], STATUS_SCHEMA_INVALID)
|
|
missing_id = {**candidate, "keyEvents": [{"event": "没有 id"}]}
|
|
self.assertEqual(check_candidate_output(missing_id, 489)["status"], STATUS_SCHEMA_INVALID)
|
|
unknown_field = {**candidate, "futureSources": ["target-scaffold"]}
|
|
self.assertEqual(check_candidate_output(unknown_field, 489)["status"], STATUS_SCHEMA_INVALID)
|
|
future_ref = {**candidate, "sourceRefs": ["target-scaffold"]}
|
|
self.assertEqual(
|
|
check_candidate_output(
|
|
future_ref,
|
|
489,
|
|
[{"sourceId": "target-scaffold", "chapter": 489}],
|
|
)["status"],
|
|
STATUS_TARGET_SOURCE_FORBIDDEN,
|
|
)
|
|
|
|
def test_candidate_requires_non_empty_and_contiguous_key_events(self):
|
|
candidate = {
|
|
"targetChapter": 489,
|
|
"chapterGoal": "突破",
|
|
"keyEvents": [
|
|
{
|
|
"id": "event-1",
|
|
"order": 1,
|
|
"event": "侦察敌情",
|
|
"participants": ["苏铭"],
|
|
"trigger": "收到异常信号",
|
|
"resultDirection": "确认威胁存在",
|
|
},
|
|
{
|
|
"id": "event-2",
|
|
"order": 2,
|
|
"event": "布置伏击",
|
|
"participants": ["苏铭", "队友"],
|
|
"trigger": "确认威胁存在",
|
|
"resultDirection": "完成前置布防",
|
|
},
|
|
],
|
|
"entities": [],
|
|
"foreshadowing": [],
|
|
"stateChanges": [],
|
|
"hook": "悬念",
|
|
"unknowns": [],
|
|
"assumptions": [],
|
|
}
|
|
self.assertTrue(check_candidate_output(candidate, 489)["ok"])
|
|
|
|
empty_events = {**candidate, "keyEvents": []}
|
|
self.assertEqual(check_candidate_output(empty_events, 489)["status"], STATUS_SCHEMA_INVALID)
|
|
|
|
duplicate_order = {
|
|
**candidate,
|
|
"keyEvents": [
|
|
{**candidate["keyEvents"][0], "order": 1},
|
|
{**candidate["keyEvents"][1], "order": 1},
|
|
],
|
|
}
|
|
self.assertEqual(check_candidate_output(duplicate_order, 489)["status"], STATUS_SCHEMA_INVALID)
|
|
|
|
gap_order = {
|
|
**candidate,
|
|
"keyEvents": [
|
|
{**candidate["keyEvents"][0], "order": 1},
|
|
{**candidate["keyEvents"][1], "order": 3},
|
|
],
|
|
}
|
|
self.assertEqual(check_candidate_output(gap_order, 489)["status"], STATUS_SCHEMA_INVALID)
|
|
|
|
bad_start = {
|
|
**candidate,
|
|
"keyEvents": [
|
|
{**candidate["keyEvents"][0], "order": 2},
|
|
{**candidate["keyEvents"][1], "order": 3},
|
|
],
|
|
}
|
|
self.assertEqual(check_candidate_output(bad_start, 489)["status"], STATUS_SCHEMA_INVALID)
|
|
|
|
def test_replay_fails_closed_before_model(self):
|
|
common = {"snapshotVersion": "v0", "asOfChapter": 488}
|
|
manifests = {
|
|
"outline_only": arm(common, []),
|
|
"outline_plus_cards": arm(common, [{"id": 1}]),
|
|
"outline_plus_placebo_cards": arm(common, [{"id": 2}]),
|
|
}
|
|
result = check_replay(
|
|
authorization=AUTH,
|
|
as_of_chapter=488,
|
|
target_chapter=489,
|
|
planner_sources=[{"chapter": 489}],
|
|
arm_manifests=manifests,
|
|
)
|
|
self.assertFalse(result["ok"])
|
|
self.assertEqual(result["status"], STATUS_TARGET_SOURCE_FORBIDDEN)
|
|
|
|
def test_production_arm_semantics_and_chapter_binding(self):
|
|
common = {"snapshotVersion": "v0", "asOfChapter": 488, "targetChapter": 489}
|
|
manifests = {
|
|
"outline_only": {
|
|
**common,
|
|
"cardInjectionCount": 0,
|
|
"cardSourceIds": [],
|
|
"cardStrategy": "none",
|
|
},
|
|
"outline_plus_cards": {
|
|
**common,
|
|
"cardInjectionCount": 1,
|
|
"cardSourceIds": ["correct-1"],
|
|
"cardStrategy": "correct",
|
|
},
|
|
"outline_plus_placebo_cards": {
|
|
**common,
|
|
"cardInjectionCount": 1,
|
|
"cardSourceIds": ["placebo-1"],
|
|
"cardStrategy": "placebo",
|
|
},
|
|
}
|
|
self.assertTrue(
|
|
check_replay(
|
|
authorization=AUTH,
|
|
as_of_chapter=488,
|
|
target_chapter=489,
|
|
planner_sources=[{"sourceId": "history-488", "sourceVersion": "v1", "chapter": 488}],
|
|
arm_manifests=manifests,
|
|
)["ok"]
|
|
)
|
|
wrong = {**manifests, "outline_only": {**manifests["outline_only"], "cardInjectionCount": 1}}
|
|
self.assertEqual(
|
|
check_replay(
|
|
authorization=AUTH,
|
|
as_of_chapter=488,
|
|
target_chapter=489,
|
|
planner_sources=[{"sourceId": "history-488", "sourceVersion": "v1", "chapter": 488}],
|
|
arm_manifests=wrong,
|
|
)["status"],
|
|
STATUS_INVALID_ARM_DIFF,
|
|
)
|
|
|
|
def test_missing_source_chapter_is_blocked(self):
|
|
result = check_target_sources(489, [{"sourceId": "unknown", "sourceVersion": "v1", "payload": "future"}])
|
|
self.assertEqual(result["status"], STATUS_TARGET_SOURCE_FORBIDDEN)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|