muse-agent-example/tests/skills/evaluate-frozen-replay/test_pattern_reference_injection.py
zizi c9f69d9d6d 治理: Skill 测试治理第一阶段——harness 控制平面 + 实现测试迁出运行时目录
范围(不含 design-story-foundation、docs/、humanization/README.md 等进行中改动):

1. 新增 harness/ 控制平面
   - skill_harness.py 静态审计:32 个运行时 Skill 的 frontmatter/manifest/文档污染,当前 0 问题
   - run_selected.py 选择性执行器:manifest 与磁盘一一对账、依赖阻断、
     空跑与 skip-only 失败关闭、AST 测试形状门
   - manifests/skills.json:32 个 Skill 的合同责任方与协作领域登记
   - manifests/test-inventory.json:81 个测试资产登记
   - specs/skill-testing.md 与 README.md:测试分层、证据边界与 harness 职责

2. 实现测试从 .claude/skills/*/scripts/ 迁至 tests/skills/<skill>/
   - 71 个测试文件迁移并修复项目根与临时目录运行导入
   - 数据库触发器测试宽泛异常收窄为 psycopg.errors.RaiseException
   - 抽取离线大测试拆出真实 PG smoke(默认阻断,不计入离线通过)
   - 抽取 presence 去重边界拆出独立测试:493 + 78 = 571 项检查不变

3. 运行时文档清理
   - 13 个 SKILL.md 移除自测/离线验证段落、测试命令与测试文件事实源表述,
     只保留运行时合同;业务运行合同、额度、授权与离线模式均保留

4. SoT 同步
   - AGENTS.md:新增 Skill 领域索引(7 个合同责任方分组,覆盖 32 个运行时 Skill)
   - 领域 07:测试入口改由 harness/manifests/ 登记,SKILL.md 不承载测试命令
   - humanization 覆盖矩阵:活动测试路径同步迁移

验证证据: harness 自测 15 项 + runner 自测 13 项通过;静态审计 32 Skill / 0 问题;
73 个非数据库测试通过;8 个集成条目中 6 个 PostgreSQL 项被依赖门明确阻断;
py_compile 与 git diff --check 通过。未连接 PostgreSQL、网络、真实模型或额度。

已知边界: 真正 skill_behavior_eval 仍为 0,尚未验证任何 Skill 自然语言行为;
evaluate-frozen-replay 的 raw 存储边界冲突留待单独治理。
2026-08-19 01:50:20 +08:00

494 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""公共范式卡接入 C 臂写手输入的离线验证。
只验证取数与合同投影,不真连库、不真调 writer/detector/judge:
检索器用 stub 注入;装配器跑真实 dry-run 路径(内部会过 validate_writer_context
的严格合同,从而证明注入的范式引用字段合法)。
"""
from __future__ import annotations
import json
import pathlib
import sys
import unittest
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
SKILLS_DIR = PROJECT_ROOT / ".claude" / "skills"
SCRIPT_DIR = SKILLS_DIR / "evaluate-frozen-replay" / "scripts"
TEST_DIR = PROJECT_ROOT / "tests" / "skills" / "evaluate-frozen-replay"
ASSEMBLE_CONTEXT_TESTS = PROJECT_ROOT / "tests" / "skills" / "assemble-context"
READ_CONTEXT_SCRIPTS = SKILLS_DIR / "assemble-context" / "scripts"
# 端到端链路测试要导入回放包(run_writer_replay.sample),其依赖执行、证据与
# 评分三个 Skill 的 scripts 目录,路径口径与 test_run_writer_replay 保持一致。
EXECUTION_SCRIPTS = SKILLS_DIR / "execute-claude-task" / "scripts"
EVIDENCE_SCRIPTS = SKILLS_DIR / "record-run-evidence" / "scripts"
QUALITY_GATE_SCRIPTS = SKILLS_DIR / "score-content-quality" / "scripts"
for _path in (
SCRIPT_DIR,
TEST_DIR,
ASSEMBLE_CONTEXT_TESTS,
READ_CONTEXT_SCRIPTS,
EXECUTION_SCRIPTS,
EVIDENCE_SCRIPTS,
QUALITY_GATE_SCRIPTS,
):
if str(_path) not in sys.path:
sys.path.insert(0, str(_path))
import load_writer_reference_work as loader # noqa: E402
from load_writer_reference_work import ( # noqa: E402
PATTERN_CARD_TYPES,
PATTERN_TOTAL_CAP,
_pattern_references_for_arm,
_retrieve_pattern_references,
assemble_writer_gate_config,
)
from run_writer_replay.sample import ( # noqa: E402
_build_arm_contexts,
_common_controls,
)
from writer_contract import ( # noqa: E402
PATTERN_NAME_MAX_CHARS,
PATTERN_POINTS_MAX_FIELDS,
PATTERN_POINT_MAX_CHARS,
PATTERN_SUMMARY_MAX_CHARS,
ContractError,
_pattern_source_ref,
_source_ref,
build_writer_creative_input,
normalize_text,
pattern_references_for_arm,
retrieval_identity,
)
# 复用既有测试夹具:真实五章 base 配置 + 纯数据快照,能让装配器跑通真实 dry-run。
from test_load_writer_reference_work import ( # noqa: E402
BASE_CONFIG_PATH,
SELECTOR_DIGEST,
SELECTOR_PATH,
_assembly_rows,
_refresh_self_hash,
)
def _prepared_base_config() -> dict[str, object]:
"""按既有用例的口径准备 base 配置:预算置 pending 并重签自哈希。"""
base_config = json.loads(BASE_CONFIG_PATH.read_text(encoding="utf-8"))
budget = base_config["executionAuthorization"]["budget"]
budget["status"] = "pending"
budget.pop("totalBudgetUsd", None)
_refresh_self_hash(budget)
return base_config
def _stub_searcher(per_type: int = 2, record: list | None = None):
"""构造确定性范式卡检索 stub,签名与真实检索器一致:(intent, *, ttype, top)。
返回的卡含 search_cards 的稳定来源指针字段(sourceId/sourceVersion/type)与内容
字段(name/summary/visibleFields)——后者正是 _pattern_content_projection 读取、
用来验证「写手真正读到卡内容」这条链路的集合。
"""
def _searcher(intent: str, *, ttype: str, top: int) -> list[dict[str, object]]:
if record is not None:
record.append({"intent": intent, "ttype": ttype, "top": top})
cards = []
for index in range(per_type):
cards.append(
{
"cardId": f"{ttype}-{index}",
"type": ttype,
"name": f"范式-{ttype}-{index}",
"score": round(0.9 - 0.01 * index, 4),
"summary": f"{ttype} 摘要 {index}",
"visibleFields": {
"节拍": f"压制—反转—落点({ttype}-{index})",
"钩子": "身份揭破",
},
"sourceId": f"draft:{ttype}-{index}",
"sourceVersion": f"draft-revision:{index + 1}",
"sourceKind": "draft",
"retrievalScope": "public_pattern",
"productionRetrievalEligible": True,
}
)
return cards[:top]
return _searcher
class RetrievePatternReferencesTest(unittest.TestCase):
"""_retrieve_pattern_references 的合同形状、总量控制与失败关闭。"""
def test_projects_pointer_plus_content_fields(self):
"""每张范式卡投影成「来源指针 + 内容字段」,且通过范式专用合同校验。"""
refs = _retrieve_pattern_references(
"本章要完成一场决战", card_searcher=_stub_searcher(per_type=2)
)
self.assertTrue(refs)
pointer_fields = {"sourceId", "sourceVersion", "sourceType"}
content_fields = {"name", "summary", "writingPoints"}
for index, ref in enumerate(refs):
# 来源指针必备;内容字段(stub 全给了)齐备;不允许合同白名单外的未知字段。
self.assertTrue(pointer_fields.issubset(ref), ref)
self.assertTrue(set(ref) <= pointer_fields | content_fields, ref)
for field in ("sourceId", "sourceVersion", "sourceType", "name", "summary"):
self.assertIsInstance(ref[field], str)
self.assertTrue(ref[field].strip())
# 必须已 NFC 归一化,合同 _string 会复核。
self.assertEqual(ref[field], normalize_text(ref[field]))
# writingPoints 是字符串→字符串,字段数与值都在合同上限内。
self.assertIsInstance(ref["writingPoints"], dict)
self.assertTrue(ref["writingPoints"])
self.assertLessEqual(len(ref["writingPoints"]), PATTERN_POINTS_MAX_FIELDS)
for key, value in ref["writingPoints"].items():
self.assertIsInstance(key, str)
self.assertIsInstance(value, str)
self.assertLessEqual(len(value), PATTERN_POINT_MAX_CHARS)
# 直接过范式专用合同 _pattern_source_ref:不抛异常即字段合法。
_pattern_source_ref(ref, f"$.patternReferences[{index}]")
# sourceType 必须落在五型内(成为写手最终看到的 kind)。
self.assertTrue(all(ref["sourceType"] in PATTERN_CARD_TYPES for ref in refs))
def test_content_truncated_to_contract_caps(self):
"""卡内容过长/字段过多时,投影截断到合同上限以内(合同侧再失败关闭复核)。"""
def _huge_searcher(intent: str, *, ttype: str, top: int) -> list[dict[str, object]]:
del intent, top
return [
{
"type": ttype,
"name": "超长范式名" * 20,
"summary": "超长摘要" * 50,
# 10 个字段超过上限 6;值也超长,验证双重截断。
"visibleFields": {f"字段{i:02d}": f"写法要点{i}" * 100 for i in range(10)},
"sourceId": f"draft:{ttype}-huge",
"sourceVersion": "draft-revision:9",
"sourceKind": "draft",
"retrievalScope": "public_pattern",
"productionRetrievalEligible": True,
}
]
refs = _retrieve_pattern_references("意图", card_searcher=_huge_searcher)
self.assertTrue(refs)
for index, ref in enumerate(refs):
self.assertLessEqual(len(ref["name"]), PATTERN_NAME_MAX_CHARS)
self.assertLessEqual(len(ref["summary"]), PATTERN_SUMMARY_MAX_CHARS)
self.assertLessEqual(len(ref["writingPoints"]), PATTERN_POINTS_MAX_FIELDS)
for value in ref["writingPoints"].values():
self.assertLessEqual(len(value), PATTERN_POINT_MAX_CHARS)
# 截断后仍须通过范式专用合同(合同侧也失败关闭)。
_pattern_source_ref(ref, f"$.patternReferences[{index}]")
def test_strict_source_ref_still_rejects_content_fields(self):
"""放宽只针对 patternReferences:严格 _source_ref 仍把 name/summary 当未知字段拒收。"""
refs = _retrieve_pattern_references("决战", card_searcher=_stub_searcher(per_type=1))
self.assertTrue(refs)
# 前提:内容字段确已注入;拿给其它来源指针用的严格校验必须失败关闭。
self.assertIn("name", refs[0])
with self.assertRaises(ContractError):
_source_ref(refs[0], "$.factEvidence[0].sourceRef")
def test_per_type_top_and_total_cap(self):
"""默认每型 2 张合计 10;显式加大每型 top 时总量硬上限 12 生效。"""
default_refs = _retrieve_pattern_references(
"决战意图", card_searcher=_stub_searcher(per_type=5)
)
# 默认 top_per_type=2:五型 × 2 = 10。
self.assertEqual(len(default_refs), len(PATTERN_CARD_TYPES) * 2)
capped_refs = _retrieve_pattern_references(
"决战意图",
card_searcher=_stub_searcher(per_type=5),
top_per_type=5,
total_cap=PATTERN_TOTAL_CAP,
)
# 5 型 × 5 = 25 会被硬上限裁到恰好 12。
self.assertEqual(len(capped_refs), PATTERN_TOTAL_CAP)
self.assertLessEqual(len(capped_refs), PATTERN_TOTAL_CAP)
def test_dedup_and_missing_pointer_skipped(self):
"""跨型同卡只注入一次;缺稳定来源指针的卡被跳过而非混入空引用。"""
def _dup_searcher(intent: str, *, ttype: str, top: int) -> list[dict[str, object]]:
del intent, top
# 所有型都返回同一张卡(同 sourceVersion+sourceId),应去重为 1 张。
return [{
"type": ttype,
"sourceId": "draft:shared",
"sourceVersion": "draft-revision:1",
"sourceKind": "draft",
"retrievalScope": "public_pattern",
"productionRetrievalEligible": True,
}]
refs = _retrieve_pattern_references("意图", card_searcher=_dup_searcher)
self.assertEqual(len(refs), 1)
def _missing_pointer(intent: str, *, ttype: str, top: int) -> list[dict[str, object]]:
del intent, top
# 缺 sourceVersion:不得进入冻结上下文。
return [{
"type": ttype,
"sourceId": f"draft:{ttype}",
"sourceKind": "draft",
"retrievalScope": "public_pattern",
"productionRetrievalEligible": True,
}]
self.assertEqual(_retrieve_pattern_references("意图", card_searcher=_missing_pointer), [])
def test_empty_intent_fails_closed(self):
"""没有检索意图(细纲为空)就不召回。"""
self.assertEqual(_retrieve_pattern_references("", card_searcher=_stub_searcher()), [])
self.assertEqual(_retrieve_pattern_references(" \n ", card_searcher=_stub_searcher()), [])
def test_non_public_or_ineligible_cards_are_skipped(self):
"""替换检索实现缺少公共范围资格时,loader 第二道门必须拒绝注入。"""
def wrong_scope(_intent: str, *, ttype: str, top: int):
del top
return [{
"type": ttype,
"sourceId": "draft:private",
"sourceVersion": "draft-revision:1",
"sourceKind": "draft",
"retrievalScope": "admin",
"productionRetrievalEligible": True,
}]
self.assertEqual(
_retrieve_pattern_references("意图", card_searcher=wrong_scope),
[],
)
class PatternReferencesForArmTest(unittest.TestCase):
"""单变量对照在取数端的直接证据:A 恒空、C 才拿卡。"""
def test_a_empty_c_nonempty(self):
refs = _retrieve_pattern_references("决战", card_searcher=_stub_searcher(per_type=2))
self.assertTrue(refs) # 前提:C 臂候选非空
# C 臂候选携带内容字段(写手真正读到卡内容)。
self.assertTrue(all("name" in ref and "writingPoints" in ref for ref in refs))
self.assertEqual(_pattern_references_for_arm("A", refs), []) # A 臂恒空,对照不破
c_refs = _pattern_references_for_arm("C", refs)
self.assertEqual(c_refs, refs)
self.assertTrue(all("name" in ref and "writingPoints" in ref for ref in c_refs))
# C 臂返回深拷贝,改动不污染源候选。
c_refs[0]["sourceId"] = "被改写"
self.assertNotEqual(refs[0]["sourceId"], "被改写")
class PatternCreativeInputProjectionTest(unittest.TestCase):
"""创作输入层证据:C 臂写手读到 name/summary/写法要点,A 臂为空,指针不外泄。"""
def _creative_input(self, refs: list, arm: str) -> dict:
"""把某臂的范式引用装进最小合法上下文,跑出写手唯一可见的创作输入。"""
from test_writer_contract import valid_context # 复用合同测试的最小合法上下文夹具
context = valid_context()
context["patternReferences"] = _pattern_references_for_arm(arm, refs)
context["contextSnapshot"]["contextSha256"] = retrieval_identity(context)
return build_writer_creative_input(context)
def test_c_arm_content_reaches_writer_and_a_arm_empty(self):
refs = _retrieve_pattern_references("决战", card_searcher=_stub_searcher(per_type=2))
self.assertTrue(refs)
creative_c = self._creative_input(refs, "C")
self.assertTrue(creative_c["patternReferences"])
for index, item in enumerate(creative_c["patternReferences"]):
self.assertEqual(item["referenceId"], f"pattern-{index + 1}")
# 写手真正看到的范式卡内容:名字、一句话摘要、写法要点,加可作范式的型。
self.assertTrue(item["name"])
self.assertTrue(item["summary"])
self.assertTrue(item["writingPoints"])
self.assertIn(item["kind"], PATTERN_CARD_TYPES)
# 来源指针只供审计回读,绝不允许泄进写手输入。
self.assertNotIn("sourceId", item)
self.assertNotIn("sourceVersion", item)
creative_a = self._creative_input(refs, "A")
self.assertEqual(creative_a["patternReferences"], []) # A 臂恒空
class PatternInjectionIntegrationTest(unittest.TestCase):
"""经真实装配 dry-run 验证:C 注入范式卡过严格合同,A 保持空,门禁放行。"""
def setUp(self):
self.base_config = _prepared_base_config()
self.selectors = json.loads(SELECTOR_PATH.read_text(encoding="utf-8"))
def _assemble(self, searcher):
return assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=_assembly_rows(self.base_config),
pattern_card_searcher=searcher,
)
def test_c_arm_pattern_references_pass_strict_contract_and_diff_allowed(self):
"""注入范式卡后装配成功(即过 validate_writer_context),且 A/C 差异落在白名单。"""
record: list = []
config = self._assemble(_stub_searcher(per_type=2, record=record))
# 五个样本都按五型发起了检索(证明 C 臂取数真正接线)。
self.assertTrue(record)
self.assertTrue(all(call["ttype"] in PATTERN_CARD_TYPES for call in record))
receipts = config["writerContextDiffReceipts"]
self.assertEqual(len(receipts), len(config["samples"]))
for sample in config["samples"]:
receipt = receipts[sample["sampleId"]]
self.assertTrue(receipt["ok"])
# A/C 上下文哈希必须不同(C 多了范式卡)。
self.assertNotEqual(receipt["contextSha256"]["A"], receipt["contextSha256"]["C"])
# 差异里必须出现 patternReferences——因 A 臂硬编码为空,差异只可能来自 C 臂非空。
pattern_paths = [
path for path in receipt["allowedDifferencePaths"] if path.startswith("$.patternReferences")
]
self.assertTrue(pattern_paths, receipt["allowedDifferencePaths"])
# 白名单不含任何越界字段。
self.assertTrue(
all(
path.startswith(("$.factConstraints", "$.proseExcerpts", "$.patternReferences"))
for path in receipt["allowedDifferencePaths"]
)
)
def test_default_without_searcher_keeps_pattern_references_empty(self):
"""不传检索器时保持历史行为:A/C 的 patternReferences 均空,差异不含该字段。"""
config = self._assemble(None)
for sample in config["samples"]:
receipt = config["writerContextDiffReceipts"][sample["sampleId"]]
self.assertTrue(receipt["ok"])
self.assertFalse(
any(path.startswith("$.patternReferences") for path in receipt["allowedDifferencePaths"]),
receipt["allowedDifferencePaths"],
)
class PatternEndToEndLinkTest(unittest.TestCase):
"""端到端链路回归:loader 装配 → config.json 序列化 → 回放端重新 assemble。
WHY 单独补这组用例:上一轮单点测试用 stub 直接喂装配器,只覆盖取数与单次
assemble,没覆盖「C 臂候选冻结进 config.json、真写时回放再读回分臂」这条真实
链路——断点正藏在两段之间(loader 没把候选写进 writerContextInput.patternReferences,
回放又把同一份原样喂给所有臂)。这里用 stub 检索器走完整链路:不连库、不调
Claude,只证明范式卡在链路每一段的形态正确。
"""
def setUp(self):
self.base_config = _prepared_base_config()
self.selectors = json.loads(SELECTOR_PATH.read_text(encoding="utf-8"))
def _loader_config(self, searcher):
"""装配后做 JSON 往返,模拟 config.json 落盘再被回放读取的真实路径。"""
config = assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=_assembly_rows(self.base_config),
pattern_card_searcher=searcher,
)
return json.loads(json.dumps(config, ensure_ascii=False))
def _replay(self, config):
"""走回放端重新 assemble 入口;内含与 loader 冻结回执的一致性复核。"""
sample = config["samples"][0]
common = _common_controls(config, sample)
contexts, writer_contexts, diff_receipt, _requirements, _mode = _build_arm_contexts(
config=config,
sample=sample,
common=common,
replay_run_id="replay-e2e-pattern.v1",
)
return writer_contexts, diff_receipt
def test_pattern_references_survive_config_roundtrip_into_c_arm_only(self):
"""C 臂候选经 config.json 往返后,真写重组装只有 C 臂拿到范式卡内容。"""
config = self._loader_config(_stub_searcher(per_type=2))
sample = config["samples"][0]
# 段间证据(上一轮的断点):loader 已把 C 臂候选冻结进 writerContextInput。
frozen_refs = sample["writerContextInput"]["patternReferences"]
self.assertTrue(frozen_refs)
for ref in frozen_refs:
self.assertTrue(ref["name"])
self.assertTrue(ref["summary"])
self.assertTrue(ref["writingPoints"])
writer_contexts, diff_receipt = self._replay(config)
# C 臂真写上下文拿到范式卡内容;A 臂恒空,单变量对照不破。
c_refs = writer_contexts["C"]["patternReferences"]
self.assertTrue(c_refs)
for ref in c_refs:
self.assertTrue(ref["name"])
self.assertTrue(ref["summary"])
self.assertTrue(ref["writingPoints"])
self.assertEqual(writer_contexts["A"]["patternReferences"], [])
# B 臂按合同单一事实源语义同属「有卡」处理组(非 A 即拿候选)。
self.assertEqual(
writer_contexts["B"]["patternReferences"],
pattern_references_for_arm("B", frozen_refs),
)
# 单变量门禁回执放行,且差异确实落在 patternReferences 白名单。
self.assertTrue(diff_receipt["ok"])
self.assertTrue(
any(
path.startswith("$.patternReferences")
for path in diff_receipt["allowedDifferencePaths"]
),
diff_receipt["allowedDifferencePaths"],
)
# 写手唯一可见的创作输入:C 臂含范式内容、A 臂空;来源指针只留冻结上下文。
creative_c = build_writer_creative_input(writer_contexts["C"])["patternReferences"]
self.assertTrue(creative_c)
for item in creative_c:
self.assertTrue(item["name"])
self.assertTrue(item["summary"])
self.assertTrue(item["writingPoints"])
self.assertNotIn("sourceId", item)
self.assertNotIn("sourceVersion", item)
self.assertEqual(
build_writer_creative_input(writer_contexts["A"])["patternReferences"], []
)
def test_replay_keeps_both_arms_empty_for_legacy_config_without_searcher(self):
"""loader 未接检索器的历史 config:writerContextInput 为空列表,回放两臂恒空。"""
config = self._loader_config(None)
sample = config["samples"][0]
self.assertEqual(sample["writerContextInput"]["patternReferences"], [])
writer_contexts, diff_receipt = self._replay(config)
self.assertEqual(writer_contexts["A"]["patternReferences"], [])
self.assertEqual(writer_contexts["C"]["patternReferences"], [])
self.assertFalse(
any(
path.startswith("$.patternReferences")
for path in diff_receipt["allowedDifferencePaths"]
)
)
if __name__ == "__main__":
unittest.main()