zizi e36cd57010 框架: 评测合同与提示词去支架化 + skill 三层重组(Phase 0-4)
【评测合同与提示词】
- writer/detector/judge 评测提示词去支架化:删评测身份/输出格式叮嘱/盲化反向叮嘱/机器字段
  (改由运行时 --json-schema/--tools ""/--strict-mcp-config/输入设计强制),装回各角色本业纪律
- writer 加写作纪律:戏剧化节拍禁抄细纲概述句、具体压倒抽象、每场戏三件套、篇幅用场景写够不注水不缩水
- 篇幅自纠环修订指令区分偏短(加场景)/偏长(删冗余);机械门要求清单(章末钩子/硬事件/角色/伏笔)注入写手硬约束
- 引文校验从「必须恰好1次」放宽为「0次失败关闭、≥1次绑首次」(检测+盲评同口径)
- llm 立 chat_governed 为主入口(chat 降为仅调试),clean_detect 改用 chat_governed 弃自带降级链
- rewrite/expansion/polish 输出合同对齐 writer.md(返回文本、写手不读写工作区、主会话落工作区)
- 修复 confirm/replay-eval 探针两组坏自测(夹具适配现行合同、探针测试自包含不硬编码漂移哈希);补 clean_detect 离线测试
- AGENTS.md 新增 §10「Agent 提示词与 Skill 审查标准」

【skill 三层重组:单向依赖 底座→能力→编排,断两环+修生产倒挂】
- Phase 0: 新建 runtime 底座(claude_runtime/file_cas/raw_vault),断环 C1、修 continuation 生产倒挂
- Phase 1: 评分尺+门判(writer_rubric/fine_outline_rubric/writer_gate/gate_input_builder)收进 quality-gate,断环 C2、名实相符
- Phase 2: 升格管线从 parse-book 独立成 upgrade skill(备份审计契约键名稳定、仅改路径定位)
- Phase 3: replay-eval 瘦成纯编排
- read-context 共用簇(build_snapshot/audit_leakage/check_snapshot/load_reference_work)下沉到新 snapshot skill,消除能力层向上引用
- Phase 4: run_writer_replay.py(130KB)拆成包(_common/budget/authorization/sample/blind/execute),__init__ 全量 re-export 测试零改动

【base 配置】三角色 effort 提 high;提示词更新;探针重测绑定 writer 合同;预算 writer 45 次/总 450 美元(含篇幅修订)

全量离线测试 36 个文件全绿;函数逻辑零改动(仅搬位置/改 import/改文档,capture_code_identity 仅改路径定位)。
2026-07-26 03:28:22 +08:00

166 lines
6.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""raw vault 的 lease 顺序、权限、路径边界与恢复测试。"""
from __future__ import annotations
import json
import os
import pathlib
import stat
import sys
import tempfile
import unittest
from datetime import datetime, timedelta, timezone
SCRIPT_DIR = pathlib.Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))
from raw_vault import RawVaultError, RawVaultManager # noqa: E402
def future_time(hours: int = 1) -> str:
"""生成不超过 24 小时的 UTC 保留截止时间。"""
return (datetime.now(timezone.utc) + timedelta(hours=hours)).isoformat()
class RawVaultTest(unittest.TestCase):
"""验证敏感字节只在有效 lease 覆盖下存在。"""
def setUp(self) -> None:
"""为每个测试创建独立 vault 根和非敏感 journal。"""
self.temporary = tempfile.TemporaryDirectory()
root = pathlib.Path(self.temporary.name)
self.vault_root = root / "private-tmp"
self.journal_root = root / "journal"
self.vault_root.mkdir(mode=0o700)
self.manager = RawVaultManager(self.journal_root, vault_root=self.vault_root)
def tearDown(self) -> None:
"""删除测试产生的独立目录。"""
self.temporary.cleanup()
def _create(self):
"""创建字段完整、审批角色合法的单次 lease。"""
return self.manager.create_vault(
authorization_id="auth-1",
approved_by="创始人",
run_id="run-1",
source_version="source-v1",
content_hashes=("sha256:" + "a" * 64,),
purpose="offline_evaluation",
retain_until=future_time(),
)
def test_lease_precedes_vault_and_permissions_are_fixed(self):
"""journal 必须先存在,随后 vault=0700、raw 文件=0600。"""
lease = self._create()
lease_file = self.journal_root / "leases" / f"{lease.vault_id}.json"
self.assertTrue(lease_file.exists())
self.assertEqual(stat.S_IMODE(lease_file.stat().st_mode), 0o600)
lease_json = json.loads(lease_file.read_text(encoding="utf-8"))
self.assertEqual(lease_json["status"], "open")
self.assertNotIn("vaultPath", lease_json)
vaults = list(self.vault_root.iterdir())
self.assertEqual(len(vaults), 1)
self.assertEqual(stat.S_IMODE(vaults[0].stat().st_mode), 0o700)
self.manager.write_bytes(lease, "responses/model.json", b"raw-secret")
raw_file = vaults[0] / "responses" / "model.json"
self.assertEqual(raw_file.read_bytes(), b"raw-secret")
self.assertEqual(stat.S_IMODE(raw_file.stat().st_mode), 0o600)
def test_path_escape_absolute_path_and_symlink_are_rejected(self):
"""调用方不能用绝对路径、.. 或软链接把 raw 写到 vault 外。"""
lease = self._create()
for candidate in ("../escape", "/tmp/escape", "nested/../../escape"):
with self.subTest(candidate=candidate), self.assertRaises(RawVaultError) as raised:
self.manager.write_bytes(lease, candidate, b"secret")
self.assertEqual(raised.exception.code, "RAW_PATH_INVALID")
vault = next(self.vault_root.iterdir())
outside = pathlib.Path(self.temporary.name) / "outside"
outside.write_bytes(b"unchanged")
os.symlink(outside, vault / "link")
with self.assertRaises(RawVaultError) as raised:
self.manager.write_bytes(lease, "link", b"changed")
self.assertEqual(raised.exception.code, "RAW_PATH_INVALID")
self.assertEqual(outside.read_bytes(), b"unchanged")
def test_cleanup_closes_lease_without_disclosing_raw_path(self):
"""清理成功后 vault 消失、lease 关闭,回执不含本机 raw 路径。"""
lease = self._create()
self.manager.write_bytes(lease, "input.json", b"secret")
receipt = self.manager.cleanup(lease)
self.assertEqual(list(self.vault_root.iterdir()), [])
self.assertEqual(receipt["status"], "closed")
self.assertNotIn(str(self.vault_root), json.dumps(receipt, ensure_ascii=False))
lease_json = json.loads(
(self.journal_root / "leases" / f"{lease.vault_id}.json").read_text(encoding="utf-8")
)
self.assertEqual(lease_json["status"], "closed")
def test_recovery_cleans_open_vault_and_closes_missing_lease(self):
"""恢复扫描必须清理未关闭 vault,并补记已经缺失的 lease。"""
existing = self._create()
self.manager.write_bytes(existing, "response.json", b"secret")
missing = self.manager.create_vault(
authorization_id="auth-2",
approved_by="用户",
run_id="run-2",
source_version="source-v2",
content_hashes=("sha256:" + "b" * 64,),
purpose="offline_evaluation",
retain_until=future_time(),
)
missing_dir = [path for path in self.vault_root.iterdir() if missing.vault_id in path.name][0]
missing_dir.rmdir()
recovery = self.manager.recover()
self.assertIn(existing.vault_id, recovery["cleanedVaultIds"])
self.assertIn(missing.vault_id, recovery["closedMissingVaultIds"])
self.assertEqual(list(self.vault_root.iterdir()), [])
def test_recovery_removes_orphan_in_manager_namespace(self):
"""同一管理器命名空间内没有 lease 的 raw 目录不能遗留。"""
orphan = self.vault_root / f"{self.manager.vault_prefix}orphan"
orphan.mkdir(mode=0o700)
(orphan / "raw").write_bytes(b"secret")
os.chmod(orphan / "raw", 0o600)
recovery = self.manager.recover()
self.assertIn("orphan", recovery["cleanedOrphanIds"])
self.assertFalse(orphan.exists())
def test_recovery_removes_raw_directory_left_after_closed_lease(self):
"""lease 已关闭后若同名 raw 目录再次出现,恢复扫描仍必须删除。"""
lease = self._create()
vault_name = next(self.vault_root.iterdir()).name
self.manager.cleanup(lease)
stale = self.vault_root / vault_name
stale.mkdir(mode=0o700)
(stale / "raw").write_bytes(b"secret")
os.chmod(stale / "raw", 0o600)
recovery = self.manager.recover()
self.assertIn(lease.vault_id, recovery["cleanedVaultIds"])
self.assertFalse(stale.exists())
if __name__ == "__main__":
unittest.main()