muse-agent-example/tests/skills/准备正文回放数据/test_pattern_reference_injection.py

501 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""公共范式卡接入 C 臂写手输入的离线验证。
只验证取数与合同投影,不真连库、不真调 writer/detector/judge:
检索器用 stub 注入;装配器跑真实 dry-run 路径(内部会过 validate_writer_context
的严格合同,从而证明注入的范式引用字段合法)。
"""
from __future__ import annotations
import json
import pathlib
import sys
import unittest
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
SKILLS_DIR = PROJECT_ROOT / ".agent" / "skills"
SCRIPT_DIR = PROJECT_ROOT / "muse" / "lifecycle" / "quality" / "skills" / "replay" / "准备正文回放数据" / "scripts"
TEST_DIR = PROJECT_ROOT / "tests" / "skills" / "准备正文回放数据"
ASSEMBLE_CONTEXT_TESTS = PROJECT_ROOT / "tests" / "skills" / "准备任务上下文"
READ_CONTEXT_SCRIPTS = PROJECT_ROOT / "muse" / "lifecycle" / "context" / "skills" / "准备任务上下文" / "scripts"
# 端到端链路测试要导入回放包,其依赖证据与评分 Skill 的 scripts;角色运行时走已安装的 muse_role 模块。
WRITER_REPLAY_SCRIPTS = PROJECT_ROOT / "muse" / "lifecycle" / "quality" / "skills" / "replay" / "回放评估正文质量" / "scripts"
EVIDENCE_SCRIPTS = PROJECT_ROOT / "muse" / "authority" / "evidence" / "skills" / "记录运行证据" / "scripts"
QUALITY_GATE_SCRIPTS = PROJECT_ROOT / "muse" / "lifecycle" / "quality" / "skills" / "judge" / "评估内容质量" / "scripts"
GATE_ADJUDICATION_SCRIPTS = PROJECT_ROOT / "muse" / "lifecycle" / "quality" / "skills" / "mechanical" / "判定质量是否合格" / "scripts"
REFRESH_RUNTIME_SCRIPTS = PROJECT_ROOT / "muse" / "platform" / "llm" / "skills" / "验证角色运行能力" / "scripts"
for _path in (
SCRIPT_DIR,
TEST_DIR,
ASSEMBLE_CONTEXT_TESTS,
READ_CONTEXT_SCRIPTS,
WRITER_REPLAY_SCRIPTS,
EVIDENCE_SCRIPTS,
QUALITY_GATE_SCRIPTS,
GATE_ADJUDICATION_SCRIPTS,
REFRESH_RUNTIME_SCRIPTS,
):
if str(_path) not in sys.path:
sys.path.insert(0, str(_path))
from load_writer_reference_work import ( # noqa: E402
PATTERN_CARD_TYPES,
PATTERN_TOTAL_CAP,
_pattern_references_for_arm,
_retrieve_pattern_references,
assemble_writer_gate_config,
)
from refresh_runtime_probe import make_dry_run_invoker, refresh_runtime_probe # noqa: E402
from run_writer_replay.sample import ( # noqa: E402
_build_arm_contexts,
_common_controls,
)
# 复用既有测试夹具:真实五章 base 配置 + 纯数据快照,能让装配器跑通真实 dry-run。
from test_load_writer_reference_work import ( # noqa: E402
BASE_CONFIG_PATH,
SELECTOR_DIGEST,
SELECTOR_PATH,
_assembly_rows,
_refresh_self_hash,
)
from writer_contract import ( # noqa: E402
PATTERN_NAME_MAX_CHARS,
PATTERN_POINT_MAX_CHARS,
PATTERN_POINTS_MAX_FIELDS,
PATTERN_SUMMARY_MAX_CHARS,
ContractError,
_pattern_source_ref,
_source_ref,
build_writer_creative_input,
normalize_text,
pattern_references_for_arm,
retrieval_identity,
)
def _prepared_base_config() -> dict[str, object]:
"""按既有用例的口径准备 base 配置:预算置 pending 并重签自哈希。"""
base_config = refresh_runtime_probe(
json.loads(BASE_CONFIG_PATH.read_text(encoding="utf-8")),
invoker=make_dry_run_invoker(),
checked_at="2026-08-21T00:00:00+00:00",
).refreshed_config
budget = base_config["executionAuthorization"]["budget"]
budget["status"] = "pending"
budget.pop("totalBudgetUsd", None)
_refresh_self_hash(budget)
return base_config
def _stub_searcher(per_type: int = 2, record: list | None = None):
"""构造确定性范式卡检索 stub,签名与真实检索器一致:(intent, *, ttype, top)。
返回的卡含 search_cards 的稳定来源指针字段(sourceId/sourceVersion/type)与内容
字段(name/summary/visibleFields)——后者正是 _pattern_content_projection 读取、
用来验证「写手真正读到卡内容」这条链路的集合。
"""
def _searcher(intent: str, *, ttype: str, top: int) -> list[dict[str, object]]:
if record is not None:
record.append({"intent": intent, "ttype": ttype, "top": top})
cards = []
for index in range(per_type):
cards.append(
{
"cardId": f"{ttype}-{index}",
"type": ttype,
"name": f"范式-{ttype}-{index}",
"score": round(0.9 - 0.01 * index, 4),
"summary": f"{ttype} 摘要 {index}",
"visibleFields": {
"节拍": f"压制—反转—落点({ttype}-{index})",
"钩子": "身份揭破",
},
"sourceId": f"draft:{ttype}-{index}",
"sourceVersion": f"draft-revision:{index + 1}",
"sourceKind": "draft",
"retrievalScope": "public_pattern",
"productionRetrievalEligible": True,
}
)
return cards[:top]
return _searcher
class RetrievePatternReferencesTest(unittest.TestCase):
"""_retrieve_pattern_references 的合同形状、总量控制与失败关闭。"""
def test_projects_pointer_plus_content_fields(self):
"""每张范式卡投影成「来源指针 + 内容字段」,且通过范式专用合同校验。"""
refs = _retrieve_pattern_references(
"本章要完成一场决战", card_searcher=_stub_searcher(per_type=2)
)
self.assertTrue(refs)
pointer_fields = {"sourceId", "sourceVersion", "sourceType"}
content_fields = {"name", "summary", "writingPoints"}
for index, ref in enumerate(refs):
# 来源指针必备;内容字段(stub 全给了)齐备;不允许合同白名单外的未知字段。
self.assertTrue(pointer_fields.issubset(ref), ref)
self.assertTrue(set(ref) <= pointer_fields | content_fields, ref)
for field in ("sourceId", "sourceVersion", "sourceType", "name", "summary"):
self.assertIsInstance(ref[field], str)
self.assertTrue(ref[field].strip())
# 必须已 NFC 归一化,合同 _string 会复核。
self.assertEqual(ref[field], normalize_text(ref[field]))
# writingPoints 是字符串→字符串,字段数与值都在合同上限内。
self.assertIsInstance(ref["writingPoints"], dict)
self.assertTrue(ref["writingPoints"])
self.assertLessEqual(len(ref["writingPoints"]), PATTERN_POINTS_MAX_FIELDS)
for key, value in ref["writingPoints"].items():
self.assertIsInstance(key, str)
self.assertIsInstance(value, str)
self.assertLessEqual(len(value), PATTERN_POINT_MAX_CHARS)
# 直接过范式专用合同 _pattern_source_ref:不抛异常即字段合法。
_pattern_source_ref(ref, f"$.patternReferences[{index}]")
# sourceType 必须落在五型内(成为写手最终看到的 kind)。
self.assertTrue(all(ref["sourceType"] in PATTERN_CARD_TYPES for ref in refs))
def test_content_truncated_to_contract_caps(self):
"""卡内容过长/字段过多时,投影截断到合同上限以内(合同侧再失败关闭复核)。"""
def _huge_searcher(intent: str, *, ttype: str, top: int) -> list[dict[str, object]]:
del intent, top
return [
{
"type": ttype,
"name": "超长范式名" * 20,
"summary": "超长摘要" * 50,
# 10 个字段超过上限 6;值也超长,验证双重截断。
"visibleFields": {f"字段{i:02d}": f"写法要点{i}" * 100 for i in range(10)},
"sourceId": f"draft:{ttype}-huge",
"sourceVersion": "draft-revision:9",
"sourceKind": "draft",
"retrievalScope": "public_pattern",
"productionRetrievalEligible": True,
}
]
refs = _retrieve_pattern_references("意图", card_searcher=_huge_searcher)
self.assertTrue(refs)
for index, ref in enumerate(refs):
self.assertLessEqual(len(ref["name"]), PATTERN_NAME_MAX_CHARS)
self.assertLessEqual(len(ref["summary"]), PATTERN_SUMMARY_MAX_CHARS)
self.assertLessEqual(len(ref["writingPoints"]), PATTERN_POINTS_MAX_FIELDS)
for value in ref["writingPoints"].values():
self.assertLessEqual(len(value), PATTERN_POINT_MAX_CHARS)
# 截断后仍须通过范式专用合同(合同侧也失败关闭)。
_pattern_source_ref(ref, f"$.patternReferences[{index}]")
def test_strict_source_ref_still_rejects_content_fields(self):
"""放宽只针对 patternReferences:严格 _source_ref 仍把 name/summary 当未知字段拒收。"""
refs = _retrieve_pattern_references("决战", card_searcher=_stub_searcher(per_type=1))
self.assertTrue(refs)
# 前提:内容字段确已注入;拿给其它来源指针用的严格校验必须失败关闭。
self.assertIn("name", refs[0])
with self.assertRaises(ContractError):
_source_ref(refs[0], "$.factEvidence[0].sourceRef")
def test_per_type_top_and_total_cap(self):
"""默认每型 2 张合计 10;显式加大每型 top 时总量硬上限 12 生效。"""
default_refs = _retrieve_pattern_references(
"决战意图", card_searcher=_stub_searcher(per_type=5)
)
# 默认 top_per_type=2:五型 × 2 = 10。
self.assertEqual(len(default_refs), len(PATTERN_CARD_TYPES) * 2)
capped_refs = _retrieve_pattern_references(
"决战意图",
card_searcher=_stub_searcher(per_type=5),
top_per_type=5,
total_cap=PATTERN_TOTAL_CAP,
)
# 5 型 × 5 = 25 会被硬上限裁到恰好 12。
self.assertEqual(len(capped_refs), PATTERN_TOTAL_CAP)
self.assertLessEqual(len(capped_refs), PATTERN_TOTAL_CAP)
def test_dedup_and_missing_pointer_skipped(self):
"""跨型同卡只注入一次;缺稳定来源指针的卡被跳过而非混入空引用。"""
def _dup_searcher(intent: str, *, ttype: str, top: int) -> list[dict[str, object]]:
del intent, top
# 所有型都返回同一张卡(同 sourceVersion+sourceId),应去重为 1 张。
return [{
"type": ttype,
"sourceId": "draft:shared",
"sourceVersion": "draft-revision:1",
"sourceKind": "draft",
"retrievalScope": "public_pattern",
"productionRetrievalEligible": True,
}]
refs = _retrieve_pattern_references("意图", card_searcher=_dup_searcher)
self.assertEqual(len(refs), 1)
def _missing_pointer(intent: str, *, ttype: str, top: int) -> list[dict[str, object]]:
del intent, top
# 缺 sourceVersion:不得进入冻结上下文。
return [{
"type": ttype,
"sourceId": f"draft:{ttype}",
"sourceKind": "draft",
"retrievalScope": "public_pattern",
"productionRetrievalEligible": True,
}]
self.assertEqual(_retrieve_pattern_references("意图", card_searcher=_missing_pointer), [])
def test_empty_intent_fails_closed(self):
"""没有检索意图(细纲为空)就不召回。"""
self.assertEqual(_retrieve_pattern_references("", card_searcher=_stub_searcher()), [])
self.assertEqual(_retrieve_pattern_references(" \n ", card_searcher=_stub_searcher()), [])
def test_non_public_or_ineligible_cards_are_skipped(self):
"""替换检索实现缺少公共范围资格时,loader 第二道门必须拒绝注入。"""
def wrong_scope(_intent: str, *, ttype: str, top: int):
del top
return [{
"type": ttype,
"sourceId": "draft:private",
"sourceVersion": "draft-revision:1",
"sourceKind": "draft",
"retrievalScope": "admin",
"productionRetrievalEligible": True,
}]
self.assertEqual(
_retrieve_pattern_references("意图", card_searcher=wrong_scope),
[],
)
class PatternReferencesForArmTest(unittest.TestCase):
"""单变量对照在取数端的直接证据:A 恒空、C 才拿卡。"""
def test_a_empty_c_nonempty(self):
refs = _retrieve_pattern_references("决战", card_searcher=_stub_searcher(per_type=2))
self.assertTrue(refs) # 前提:C 臂候选非空
# C 臂候选携带内容字段(写手真正读到卡内容)。
self.assertTrue(all("name" in ref and "writingPoints" in ref for ref in refs))
self.assertEqual(_pattern_references_for_arm("A", refs), []) # A 臂恒空,对照不破
c_refs = _pattern_references_for_arm("C", refs)
self.assertEqual(c_refs, refs)
self.assertTrue(all("name" in ref and "writingPoints" in ref for ref in c_refs))
# C 臂返回深拷贝,改动不污染源候选。
c_refs[0]["sourceId"] = "被改写"
self.assertNotEqual(refs[0]["sourceId"], "被改写")
class PatternCreativeInputProjectionTest(unittest.TestCase):
"""创作输入层证据:C 臂写手读到 name/summary/写法要点,A 臂为空,指针不外泄。"""
def _creative_input(self, refs: list, arm: str) -> dict:
"""把某臂的范式引用装进最小合法上下文,跑出写手唯一可见的创作输入。"""
from test_writer_contract import valid_context # 复用合同测试的最小合法上下文夹具
context = valid_context()
context["patternReferences"] = _pattern_references_for_arm(arm, refs)
context["contextSnapshot"]["contextSha256"] = retrieval_identity(context)
return build_writer_creative_input(context)
def test_c_arm_content_reaches_writer_and_a_arm_empty(self):
refs = _retrieve_pattern_references("决战", card_searcher=_stub_searcher(per_type=2))
self.assertTrue(refs)
creative_c = self._creative_input(refs, "C")
self.assertTrue(creative_c["patternReferences"])
for index, item in enumerate(creative_c["patternReferences"]):
self.assertEqual(item["referenceId"], f"pattern-{index + 1}")
# 写手真正看到的范式卡内容:名字、一句话摘要、写法要点,加可作范式的型。
self.assertTrue(item["name"])
self.assertTrue(item["summary"])
self.assertTrue(item["writingPoints"])
self.assertIn(item["kind"], PATTERN_CARD_TYPES)
# 来源指针只供审计回读,绝不允许泄进写手输入。
self.assertNotIn("sourceId", item)
self.assertNotIn("sourceVersion", item)
creative_a = self._creative_input(refs, "A")
self.assertEqual(creative_a["patternReferences"], []) # A 臂恒空
class PatternInjectionIntegrationTest(unittest.TestCase):
"""经真实装配 dry-run 验证:C 注入范式卡过严格合同,A 保持空,门禁放行。"""
def setUp(self):
self.base_config = _prepared_base_config()
self.selectors = json.loads(SELECTOR_PATH.read_text(encoding="utf-8"))
def _assemble(self, searcher):
return assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=_assembly_rows(self.base_config),
pattern_card_searcher=searcher,
)
def test_c_arm_pattern_references_pass_strict_contract_and_diff_allowed(self):
"""注入范式卡后装配成功(即过 validate_writer_context),且 A/C 差异落在白名单。"""
record: list = []
config = self._assemble(_stub_searcher(per_type=2, record=record))
# 五个样本都按五型发起了检索(证明 C 臂取数真正接线)。
self.assertTrue(record)
self.assertTrue(all(call["ttype"] in PATTERN_CARD_TYPES for call in record))
receipts = config["writerContextDiffReceipts"]
self.assertEqual(len(receipts), len(config["samples"]))
for sample in config["samples"]:
receipt = receipts[sample["sampleId"]]
self.assertTrue(receipt["ok"])
# A/C 上下文哈希必须不同(C 多了范式卡)。
self.assertNotEqual(receipt["contextSha256"]["A"], receipt["contextSha256"]["C"])
# 差异里必须出现 patternReferences——因 A 臂硬编码为空,差异只可能来自 C 臂非空。
pattern_paths = [
path for path in receipt["allowedDifferencePaths"] if path.startswith("$.patternReferences")
]
self.assertTrue(pattern_paths, receipt["allowedDifferencePaths"])
# 白名单不含任何越界字段。
self.assertTrue(
all(
path.startswith(("$.factConstraints", "$.proseExcerpts", "$.patternReferences"))
for path in receipt["allowedDifferencePaths"]
)
)
def test_default_without_searcher_keeps_pattern_references_empty(self):
"""不传检索器时保持历史行为:A/C 的 patternReferences 均空,差异不含该字段。"""
config = self._assemble(None)
for sample in config["samples"]:
receipt = config["writerContextDiffReceipts"][sample["sampleId"]]
self.assertTrue(receipt["ok"])
self.assertFalse(
any(path.startswith("$.patternReferences") for path in receipt["allowedDifferencePaths"]),
receipt["allowedDifferencePaths"],
)
class PatternEndToEndLinkTest(unittest.TestCase):
"""端到端链路回归:loader 装配 → config.json 序列化 → 回放端重新 assemble。
WHY 单独补这组用例:上一轮单点测试用 stub 直接喂装配器,只覆盖取数与单次
assemble,没覆盖「C 臂候选冻结进 config.json、真写时回放再读回分臂」这条真实
链路——断点正藏在两段之间(loader 没把候选写进 writerContextInput.patternReferences,
回放又把同一份原样喂给所有臂)。这里用 stub 检索器走完整链路:不连库、不调
Claude,只证明范式卡在链路每一段的形态正确。
"""
def setUp(self):
self.base_config = _prepared_base_config()
self.selectors = json.loads(SELECTOR_PATH.read_text(encoding="utf-8"))
def _loader_config(self, searcher):
"""装配后做 JSON 往返,模拟 config.json 落盘再被回放读取的真实路径。"""
config = assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=_assembly_rows(self.base_config),
pattern_card_searcher=searcher,
)
return json.loads(json.dumps(config, ensure_ascii=False))
def _replay(self, config):
"""走回放端重新 assemble 入口;内含与 loader 冻结回执的一致性复核。"""
sample = config["samples"][0]
common = _common_controls(config, sample)
contexts, writer_contexts, diff_receipt, _requirements, _mode = _build_arm_contexts(
config=config,
sample=sample,
common=common,
replay_run_id="replay-e2e-pattern.v1",
)
return writer_contexts, diff_receipt
def test_pattern_references_survive_config_roundtrip_into_c_arm_only(self):
"""C 臂候选经 config.json 往返后,真写重组装只有 C 臂拿到范式卡内容。"""
config = self._loader_config(_stub_searcher(per_type=2))
sample = config["samples"][0]
# 段间证据(上一轮的断点):loader 已把 C 臂候选冻结进 writerContextInput。
frozen_refs = sample["writerContextInput"]["patternReferences"]
self.assertTrue(frozen_refs)
for ref in frozen_refs:
self.assertTrue(ref["name"])
self.assertTrue(ref["summary"])
self.assertTrue(ref["writingPoints"])
writer_contexts, diff_receipt = self._replay(config)
# C 臂真写上下文拿到范式卡内容;A 臂恒空,单变量对照不破。
c_refs = writer_contexts["C"]["patternReferences"]
self.assertTrue(c_refs)
for ref in c_refs:
self.assertTrue(ref["name"])
self.assertTrue(ref["summary"])
self.assertTrue(ref["writingPoints"])
self.assertEqual(writer_contexts["A"]["patternReferences"], [])
# B 臂按合同单一事实源语义同属「有卡」处理组(非 A 即拿候选)。
self.assertEqual(
writer_contexts["B"]["patternReferences"],
pattern_references_for_arm("B", frozen_refs),
)
# 单变量门禁回执放行,且差异确实落在 patternReferences 白名单。
self.assertTrue(diff_receipt["ok"])
self.assertTrue(
any(
path.startswith("$.patternReferences")
for path in diff_receipt["allowedDifferencePaths"]
),
diff_receipt["allowedDifferencePaths"],
)
# 写手唯一可见的创作输入:C 臂含范式内容、A 臂空;来源指针只留冻结上下文。
creative_c = build_writer_creative_input(writer_contexts["C"])["patternReferences"]
self.assertTrue(creative_c)
for item in creative_c:
self.assertTrue(item["name"])
self.assertTrue(item["summary"])
self.assertTrue(item["writingPoints"])
self.assertNotIn("sourceId", item)
self.assertNotIn("sourceVersion", item)
self.assertEqual(
build_writer_creative_input(writer_contexts["A"])["patternReferences"], []
)
def test_replay_keeps_both_arms_empty_for_legacy_config_without_searcher(self):
"""loader 未接检索器的历史 config:writerContextInput 为空列表,回放两臂恒空。"""
config = self._loader_config(None)
sample = config["samples"][0]
self.assertEqual(sample["writerContextInput"]["patternReferences"], [])
writer_contexts, diff_receipt = self._replay(config)
self.assertEqual(writer_contexts["A"]["patternReferences"], [])
self.assertEqual(writer_contexts["C"]["patternReferences"], [])
self.assertFalse(
any(
path.startswith("$.patternReferences")
for path in diff_receipt["allowedDifferencePaths"]
)
)
if __name__ == "__main__":
unittest.main()