c9f69d9d6d
范围（不含 design-story-foundation、docs/、humanization/README.md 等进行中改动）:
1. 新增 harness/ 控制平面
- skill_harness.py 静态审计：32 个运行时 Skill 的 frontmatter/manifest/文档污染，当前 0 问题
- run_selected.py 选择性执行器：manifest 与磁盘一一对账、依赖阻断、
空跑与 skip-only 失败关闭、AST 测试形状门
- manifests/skills.json：32 个 Skill 的合同责任方与协作领域登记
- manifests/test-inventory.json：81 个测试资产登记
- specs/skill-testing.md 与 README.md：测试分层、证据边界与 harness 职责
2. 实现测试从 .claude/skills/*/scripts/ 迁至 tests/skills/<skill>/
- 71 个测试文件迁移并修复项目根与临时目录运行导入
- 数据库触发器测试宽泛异常收窄为 psycopg.errors.RaiseException
- 抽取离线大测试拆出真实 PG smoke（默认阻断，不计入离线通过）
- 抽取 presence 去重边界拆出独立测试：493 + 78 = 571 项检查不变
3. 运行时文档清理
- 13 个 SKILL.md 移除自测/离线验证段落、测试命令与测试文件事实源表述，
只保留运行时合同；业务运行合同、额度、授权与离线模式均保留
4. SoT 同步
- AGENTS.md：新增 Skill 领域索引（7 个合同责任方分组，覆盖 32 个运行时 Skill）
- 领域 07：测试入口改由 harness/manifests/ 登记，SKILL.md 不承载测试命令
- humanization 覆盖矩阵：活动测试路径同步迁移
验证证据: harness 自测 15 项 + runner 自测 13 项通过；静态审计 32 Skill / 0 问题；
73 个非数据库测试通过；8 个集成条目中 6 个 PostgreSQL 项被依赖门明确阻断；
py_compile 与 git diff --check 通过。未连接 PostgreSQL、网络、真实模型或额度。
已知边界: 真正 skill_behavior_eval 仍为 0，尚未验证任何 Skill 自然语言行为；
evaluate-frozen-replay 的 raw 存储边界冲突留待单独治理。
316 lines
14 KiB
Python
316 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""raw vault 的 lease 顺序、权限、路径边界与恢复测试。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import pathlib
|
||
import stat
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from datetime import datetime, timedelta, timezone
|
||
from unittest import mock
|
||
|
||
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
||
SCRIPT_DIR = PROJECT_ROOT / ".claude" / "skills" / "record-run-evidence" / "scripts"
|
||
sys.path.insert(0, str(SCRIPT_DIR))
|
||
|
||
import raw_vault # noqa: E402
|
||
from raw_vault import RawVaultError, RawVaultManager # noqa: E402
|
||
|
||
|
||
def future_time(hours: int = 1) -> str:
|
||
"""生成不超过 24 小时的 UTC 保留截止时间。"""
|
||
|
||
return (datetime.now(timezone.utc) + timedelta(hours=hours)).isoformat()
|
||
|
||
|
||
class RawVaultTest(unittest.TestCase):
|
||
"""验证敏感字节只在有效 lease 覆盖下存在。"""
|
||
|
||
def setUp(self) -> None:
|
||
"""为每个测试创建独立 vault 根和非敏感 journal。"""
|
||
|
||
self.temporary = tempfile.TemporaryDirectory()
|
||
root = pathlib.Path(self.temporary.name)
|
||
self.vault_root = root / "private-tmp"
|
||
self.journal_root = root / "journal"
|
||
self.vault_root.mkdir(mode=0o700)
|
||
self.manager = RawVaultManager(self.journal_root, vault_root=self.vault_root)
|
||
|
||
def tearDown(self) -> None:
|
||
"""删除测试产生的独立目录。"""
|
||
|
||
self.temporary.cleanup()
|
||
|
||
def _create(self):
|
||
"""创建字段完整、审批角色合法的单次 lease。"""
|
||
|
||
return self.manager.create_vault(
|
||
authorization_id="auth-1",
|
||
approved_by="创始人",
|
||
run_id="run-1",
|
||
source_version="source-v1",
|
||
content_hashes=("sha256:" + "a" * 64,),
|
||
purpose="offline_evaluation",
|
||
retain_until=future_time(),
|
||
)
|
||
|
||
def test_lease_precedes_vault_and_permissions_are_fixed(self):
|
||
"""journal 必须先存在，随后 vault=0700、raw 文件=0600。"""
|
||
|
||
lease = self._create()
|
||
lease_file = self.journal_root / "leases" / f"{lease.vault_id}.json"
|
||
self.assertTrue(lease_file.exists())
|
||
self.assertEqual(stat.S_IMODE(lease_file.stat().st_mode), 0o600)
|
||
lease_json = json.loads(lease_file.read_text(encoding="utf-8"))
|
||
self.assertEqual(lease_json["status"], "open")
|
||
self.assertNotIn("vaultPath", lease_json)
|
||
|
||
vaults = list(self.vault_root.iterdir())
|
||
self.assertEqual(len(vaults), 1)
|
||
self.assertEqual(stat.S_IMODE(vaults[0].stat().st_mode), 0o700)
|
||
self.manager.write_bytes(lease, "responses/model.json", b"raw-secret")
|
||
raw_file = vaults[0] / "responses" / "model.json"
|
||
self.assertEqual(raw_file.read_bytes(), b"raw-secret")
|
||
self.assertEqual(stat.S_IMODE(raw_file.stat().st_mode), 0o600)
|
||
|
||
def test_path_escape_absolute_path_and_symlink_are_rejected(self):
|
||
"""调用方不能用绝对路径、.. 或软链接把 raw 写到 vault 外。"""
|
||
|
||
lease = self._create()
|
||
for candidate in ("../escape", "/tmp/escape", "nested/../../escape"):
|
||
with self.subTest(candidate=candidate), self.assertRaises(RawVaultError) as raised:
|
||
self.manager.write_bytes(lease, candidate, b"secret")
|
||
self.assertEqual(raised.exception.code, "RAW_PATH_INVALID")
|
||
|
||
vault = next(self.vault_root.iterdir())
|
||
outside = pathlib.Path(self.temporary.name) / "outside"
|
||
outside.write_bytes(b"unchanged")
|
||
os.symlink(outside, vault / "link")
|
||
with self.assertRaises(RawVaultError) as raised:
|
||
self.manager.write_bytes(lease, "link", b"changed")
|
||
self.assertEqual(raised.exception.code, "RAW_PATH_INVALID")
|
||
self.assertEqual(outside.read_bytes(), b"unchanged")
|
||
|
||
def test_cleanup_closes_lease_without_disclosing_raw_path(self):
|
||
"""清理成功后 vault 消失、lease 关闭，回执不含本机 raw 路径。"""
|
||
|
||
lease = self._create()
|
||
self.manager.write_bytes(lease, "input.json", b"secret")
|
||
receipt = self.manager.cleanup(lease)
|
||
|
||
self.assertEqual(list(self.vault_root.iterdir()), [])
|
||
self.assertEqual(receipt["status"], "closed")
|
||
self.assertNotIn(str(self.vault_root), json.dumps(receipt, ensure_ascii=False))
|
||
lease_json = json.loads(
|
||
(self.journal_root / "leases" / f"{lease.vault_id}.json").read_text(encoding="utf-8")
|
||
)
|
||
self.assertEqual(lease_json["status"], "closed")
|
||
|
||
def test_migrate_preserves_raw_and_writes_path_free_receipt(self):
|
||
"""正式收口必须迁移完整 raw，源 vault 消失但受控归档内容和校验回执仍在。"""
|
||
|
||
lease = self._create()
|
||
self.manager.write_bytes(lease, "inputs/prompt.json", b"prompt-secret")
|
||
self.manager.write_bytes(lease, "outputs/response.json", b"response-secret")
|
||
archive_root = pathlib.Path(self.temporary.name) / "archive"
|
||
|
||
receipt = self.manager.migrate(lease, archive_root=archive_root)
|
||
|
||
self.assertEqual(receipt["status"], "migrated")
|
||
self.assertEqual(receipt["fileCount"], 2)
|
||
self.assertEqual(receipt["retentionPolicy"], "explicit_cleanup_required")
|
||
self.assertNotIn(str(archive_root), json.dumps(receipt, ensure_ascii=False))
|
||
self.assertEqual(list(self.vault_root.iterdir()), [])
|
||
archived = next(archive_root.glob(f"muse-raw-archive-{lease.vault_id}"))
|
||
self.assertEqual(
|
||
(archived / "inputs" / "prompt.json").read_bytes(), b"prompt-secret"
|
||
)
|
||
self.assertEqual(
|
||
(archived / "outputs" / "response.json").read_bytes(), b"response-secret"
|
||
)
|
||
self.assertTrue((archived / ".migration-receipt.json").is_file())
|
||
self.assertEqual(stat.S_IMODE(archive_root.stat().st_mode), 0o700)
|
||
self.assertEqual(stat.S_IMODE(archived.stat().st_mode), 0o700)
|
||
lease_json = json.loads(
|
||
(self.journal_root / "leases" / f"{lease.vault_id}.json").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
)
|
||
self.assertEqual(lease_json["status"], "migrated")
|
||
self.assertEqual(lease_json["archiveId"], lease.vault_id)
|
||
self.assertNotIn("archivePath", lease_json)
|
||
|
||
def test_migrate_rejects_relative_archive_before_moving_raw(self):
|
||
"""归档根不是绝对路径时必须失败关闭，原始 vault 与 open lease 保持可重试。"""
|
||
|
||
lease = self._create()
|
||
self.manager.write_bytes(lease, "response.json", b"secret")
|
||
with self.assertRaises(RawVaultError) as raised:
|
||
self.manager.migrate(lease, archive_root="relative/archive")
|
||
self.assertEqual(raised.exception.code, "RAW_ARCHIVE_INVALID")
|
||
self.assertEqual(len(list(self.vault_root.iterdir())), 1)
|
||
lease_json = json.loads(
|
||
(self.journal_root / "leases" / f"{lease.vault_id}.json").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
)
|
||
self.assertEqual(lease_json["status"], "open")
|
||
|
||
def test_recover_never_deletes_migrating_vault(self):
|
||
"""迁移中断必须保留 raw 供人工恢复，recover 不得退化为删除。"""
|
||
|
||
lease = self._create()
|
||
self.manager.write_bytes(lease, "response.json", b"secret")
|
||
lease_path = self.journal_root / "leases" / f"{lease.vault_id}.json"
|
||
lease_json = json.loads(lease_path.read_text(encoding="utf-8"))
|
||
lease_json["status"] = "migrating"
|
||
raw_vault._atomic_write_json(lease_path, lease_json)
|
||
|
||
recovery = self.manager.recover()
|
||
|
||
self.assertEqual(recovery["cleanedVaultIds"], [])
|
||
self.assertEqual(len(list(self.vault_root.iterdir())), 1)
|
||
|
||
def test_recovery_cleans_open_vault_and_closes_missing_lease(self):
|
||
"""恢复扫描必须清理未关闭 vault，并补记已经缺失的 lease。"""
|
||
|
||
existing = self._create()
|
||
self.manager.write_bytes(existing, "response.json", b"secret")
|
||
|
||
missing = self.manager.create_vault(
|
||
authorization_id="auth-2",
|
||
approved_by="用户",
|
||
run_id="run-2",
|
||
source_version="source-v2",
|
||
content_hashes=("sha256:" + "b" * 64,),
|
||
purpose="offline_evaluation",
|
||
retain_until=future_time(),
|
||
)
|
||
missing_dir = [path for path in self.vault_root.iterdir() if missing.vault_id in path.name][0]
|
||
missing_dir.rmdir()
|
||
|
||
recovery = self.manager.recover()
|
||
|
||
self.assertIn(existing.vault_id, recovery["cleanedVaultIds"])
|
||
self.assertIn(missing.vault_id, recovery["closedMissingVaultIds"])
|
||
self.assertEqual(list(self.vault_root.iterdir()), [])
|
||
|
||
def test_recovery_removes_orphan_in_manager_namespace(self):
|
||
"""同一管理器命名空间内没有 lease 的 raw 目录不能遗留。"""
|
||
|
||
orphan = self.vault_root / f"{self.manager.vault_prefix}orphan"
|
||
orphan.mkdir(mode=0o700)
|
||
(orphan / "raw").write_bytes(b"secret")
|
||
os.chmod(orphan / "raw", 0o600)
|
||
|
||
recovery = self.manager.recover()
|
||
|
||
self.assertIn("orphan", recovery["cleanedOrphanIds"])
|
||
self.assertFalse(orphan.exists())
|
||
|
||
def test_recovery_removes_raw_directory_left_after_closed_lease(self):
|
||
"""lease 已关闭后若同名 raw 目录再次出现，恢复扫描仍必须删除。"""
|
||
|
||
lease = self._create()
|
||
vault_name = next(self.vault_root.iterdir()).name
|
||
self.manager.cleanup(lease)
|
||
stale = self.vault_root / vault_name
|
||
stale.mkdir(mode=0o700)
|
||
(stale / "raw").write_bytes(b"secret")
|
||
os.chmod(stale / "raw", 0o600)
|
||
|
||
recovery = self.manager.recover()
|
||
|
||
self.assertIn(lease.vault_id, recovery["cleanedVaultIds"])
|
||
self.assertFalse(stale.exists())
|
||
|
||
def test_create_rejects_insufficient_min_remaining_without_lease_or_vault(self):
|
||
"""声明的最低剩余租期不足时必须在落 lease / 建 vault 前稳定失败关闭。"""
|
||
|
||
short_retention = (datetime.now(timezone.utc) + timedelta(seconds=30)).isoformat()
|
||
with self.assertRaises(RawVaultError) as raised:
|
||
self.manager.create_vault(
|
||
authorization_id="auth-min",
|
||
approved_by="创始人",
|
||
run_id="run-min",
|
||
source_version="source-v1",
|
||
content_hashes=("sha256:" + "d" * 64,),
|
||
purpose="offline_evaluation",
|
||
retain_until=short_retention,
|
||
min_remaining=timedelta(seconds=90),
|
||
)
|
||
self.assertEqual(raised.exception.code, "RAW_LEASE_INSUFFICIENT_RETENTION")
|
||
# 失败关闭不能留下任何 lease journal 或 raw vault 目录。
|
||
self.assertEqual(list((self.journal_root / "leases").iterdir()), [])
|
||
self.assertEqual(list(self.vault_root.iterdir()), [])
|
||
|
||
def test_create_accepts_sufficient_min_remaining(self):
|
||
"""剩余租期覆盖最低要求时正常创建 lease 与 vault。"""
|
||
|
||
lease = self.manager.create_vault(
|
||
authorization_id="auth-ok",
|
||
approved_by="创始人",
|
||
run_id="run-ok",
|
||
source_version="source-v1",
|
||
content_hashes=("sha256:" + "e" * 64,),
|
||
purpose="offline_evaluation",
|
||
retain_until=future_time(hours=1),
|
||
min_remaining=timedelta(seconds=90),
|
||
)
|
||
self.assertTrue((self.journal_root / "leases" / f"{lease.vault_id}.json").exists())
|
||
self.assertEqual(len(list(self.vault_root.iterdir())), 1)
|
||
|
||
def test_write_rejected_after_expiry_but_cleanup_still_works(self):
|
||
"""租约到期后 write 立即失败关闭，但 cleanup 仍能收敛过期 lease。"""
|
||
|
||
base = datetime(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||
clock = {"now": base}
|
||
with mock.patch.object(raw_vault, "_utc_now", side_effect=lambda: clock["now"]):
|
||
lease = self.manager.create_vault(
|
||
authorization_id="auth-exp",
|
||
approved_by="创始人",
|
||
run_id="run-exp",
|
||
source_version="source-v1",
|
||
content_hashes=("sha256:" + "c" * 64,),
|
||
purpose="offline_evaluation",
|
||
retain_until=(base + timedelta(seconds=10)).isoformat(),
|
||
)
|
||
self.manager.write_bytes(lease, "input.json", b"secret")
|
||
clock["now"] = base + timedelta(seconds=20) # 越过 retainUntil
|
||
with self.assertRaises(RawVaultError) as raised:
|
||
self.manager.write_bytes(lease, "late.json", b"secret")
|
||
self.assertEqual(raised.exception.code, "RAW_LEASE_EXPIRED")
|
||
# 过期不能阻断清理：cleanup 仍必须删除 vault 并关闭 lease。
|
||
receipt = self.manager.cleanup(lease)
|
||
self.assertEqual(receipt["status"], "closed")
|
||
self.assertEqual(list(self.vault_root.iterdir()), [])
|
||
|
||
def test_recover_cleans_expired_open_lease(self):
|
||
"""恢复扫描必须能收敛已过期的 open lease，不因过期而卡住清理。"""
|
||
|
||
base = datetime(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||
clock = {"now": base}
|
||
with mock.patch.object(raw_vault, "_utc_now", side_effect=lambda: clock["now"]):
|
||
lease = self.manager.create_vault(
|
||
authorization_id="auth-rec",
|
||
approved_by="用户",
|
||
run_id="run-rec",
|
||
source_version="source-v1",
|
||
content_hashes=("sha256:" + "f" * 64,),
|
||
purpose="offline_evaluation",
|
||
retain_until=(base + timedelta(seconds=10)).isoformat(),
|
||
)
|
||
self.manager.write_bytes(lease, "response.json", b"secret")
|
||
clock["now"] = base + timedelta(seconds=20) # 越过 retainUntil
|
||
recovery = self.manager.recover()
|
||
self.assertIn(lease.vault_id, recovery["cleanedVaultIds"])
|
||
self.assertEqual(list(self.vault_root.iterdir()), [])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|