zizi-al/muse-agent-example
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
muse-agent-example/tests/skills/evaluate-frozen-replay/test_load_writer_reference_work.py
zizi c9f69d9d6d 治理: Skill 测试治理第一阶段——harness 控制平面 + 实现测试迁出运行时目录 
范围（不含 design-story-foundation、docs/、humanization/README.md 等进行中改动）:

1. 新增 harness/ 控制平面
   - skill_harness.py 静态审计：32 个运行时 Skill 的 frontmatter/manifest/文档污染，当前 0 问题
   - run_selected.py 选择性执行器：manifest 与磁盘一一对账、依赖阻断、
     空跑与 skip-only 失败关闭、AST 测试形状门
   - manifests/skills.json：32 个 Skill 的合同责任方与协作领域登记
   - manifests/test-inventory.json：81 个测试资产登记
   - specs/skill-testing.md 与 README.md：测试分层、证据边界与 harness 职责

2. 实现测试从 .claude/skills/*/scripts/ 迁至 tests/skills/<skill>/
   - 71 个测试文件迁移并修复项目根与临时目录运行导入
   - 数据库触发器测试宽泛异常收窄为 psycopg.errors.RaiseException
   - 抽取离线大测试拆出真实 PG smoke（默认阻断，不计入离线通过）
   - 抽取 presence 去重边界拆出独立测试：493 + 78 = 571 项检查不变

3. 运行时文档清理
   - 13 个 SKILL.md 移除自测/离线验证段落、测试命令与测试文件事实源表述，
     只保留运行时合同；业务运行合同、额度、授权与离线模式均保留

4. SoT 同步
   - AGENTS.md：新增 Skill 领域索引（7 个合同责任方分组，覆盖 32 个运行时 Skill）
   - 领域 07：测试入口改由 harness/manifests/ 登记，SKILL.md 不承载测试命令
   - humanization 覆盖矩阵：活动测试路径同步迁移

验证证据: harness 自测 15 项 + runner 自测 13 项通过；静态审计 32 Skill / 0 问题；
73 个非数据库测试通过；8 个集成条目中 6 个 PostgreSQL 项被依赖门明确阻断；
py_compile 与 git diff --check 通过。未连接 PostgreSQL、网络、真实模型或额度。

已知边界: 真正 skill_behavior_eval 仍为 0，尚未验证任何 Skill 自然语言行为；
evaluate-frozen-replay 的 raw 存储边界冲突留待单独治理。
2026-08-19 01:50:20 +08:00

1218 lines
52 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""Writer Gate A 真实五章临时配置装配器测试。"""
from __future__ import annotations
import copy
import json
import pathlib
import stat
import sys
import tempfile
import types
import unittest
from datetime import datetime, timedelta, timezone
from unittest.mock import patch
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
SKILLS_DIR = PROJECT_ROOT / ".claude" / "skills"
SCRIPT_DIR = SKILLS_DIR / "evaluate-frozen-replay" / "scripts"
TEST_DIR = PROJECT_ROOT / "tests" / "skills" / "evaluate-frozen-replay"
READ_CONTEXT_SCRIPTS = SKILLS_DIR / "assemble-context" / "scripts"
DETECT_SCRIPTS = SKILLS_DIR / "check-content-consistency" / "scripts"
CONTINUATION_SCRIPTS = SKILLS_DIR / "write-next-chapter" / "scripts"
CONTINUATION_TESTS = PROJECT_ROOT / "tests" / "skills" / "write-next-chapter"
for _path in (
SCRIPT_DIR,
TEST_DIR,
READ_CONTEXT_SCRIPTS,
DETECT_SCRIPTS,
CONTINUATION_SCRIPTS,
CONTINUATION_TESTS,
):
if str(_path) not in sys.path:
sys.path.insert(0, str(_path))
import load_writer_reference_work as loader # noqa: E402
import run_writer_replay as replay_module # noqa: E402
from load_writer_reference_work import ( # noqa: E402
WriterReferenceWorkError,
_requirement_hard_constraints,
assemble_writer_gate_config,
load_writer_reference_rows,
milestone_reference_chapters,
resolve_card_selectors,
validate_oracle_truth_pack,
write_temporary_config,
)
from run_writer_replay import run_writer_replay # noqa: E402
from check_writer_candidate import check_writer_candidate # noqa: E402
from writer_contract import build_candidate_envelope, han_count # noqa: E402
from test_run_writer import _bound_context # noqa: E402
BASE_CONFIG_PATH = SCRIPT_DIR.parent / "configs" / "writer-gate-a-deep-space-v1.json"
SELECTOR_PATH = (
SCRIPT_DIR.parent / "configs" / "writer-gate-a-deep-space-card-selectors-v1.json"
)
SELECTOR_DIGEST = loader.selector_sha256(SELECTOR_PATH.read_bytes())
FILE_HASH = "02cf1f8c1ca03c26e0b839d88fe536e83c0af20fd8972235b7aedca6a33becf4"
SOURCE_HASH = f"sha256:{FILE_HASH}"
SOURCE_VERSION = f"raw-file-v1:{SOURCE_HASH}"
EXPECTED_CLAUDE_EXECUTABLE_SHA256 = (
"ba790279cab6ef77b713864d4bf5f764fcea87d3a3eb7591a41f741e45212b5c"
)
def _refresh_self_hash(value: dict[str, object]) -> None:
"""按 loader 使用的规范 JSON 算法刷新测试夹具自哈希。"""
value["receiptSha256"] = loader._sha256_value(
{key: item for key, item in value.items() if key != "receiptSha256"}
)
def _card_row(
card_id: int,
card_type: str,
name: str,
*,
aliases: list[str] | None = None,
milestones: list[dict[str, object]] | None = None,
) -> dict[str, object]:
"""构造符合 upgrade_book 冻结投影合同的卡行。"""
return {
"id": card_id,
"status": "pending",
"source_type": "upgrade_book",
"source_id": 8,
"revision": 1,
"deleted": False,
"draft_payload": {
"type": card_type,
"名称": name,
"别名": aliases or [],
"出场章": [],
"字段": {
"演变历程": milestones or [{"章": 1, "台阶": f"{name}历史事实"}],
},
},
}
def _selected_cards() -> list[dict[str, object]]:
"""构造五样本固定选择所需的六张唯一卡。"""
return [
_card_row(
101,
"character",
"安若雪",
milestones=[{"章": 484, "台阶": "早期人物"}, {"章": 485, "台阶": "中期人物"}, {"章": 487, "台阶": "近期人物"}],
),
_card_row(
102,
"character",
"苏铭",
milestones=[{"章": 312, "台阶": "早期人物"}, {"章": 313, "台阶": "中期人物"}, {"章": 320, "台阶": "近期人物"}],
),
_card_row(
103,
"character",
"蒂米",
milestones=[{"章": 454, "台阶": "早期人物"}, {"章": 465, "台阶": "中期人物"}, {"章": 467, "台阶": "近期人物"}],
),
_card_row(104, "character", "璐茜", aliases=["赛莉丝"], milestones=[{"章": 190, "台阶": "历史身份"}]),
_card_row(105, "event", "三校联赛", milestones=[{"章": "188-189", "台阶": "历史事件"}]),
_card_row(106, "character", "伊蕾莉雅", milestones=[{"章": "460-462", "台阶": "历史人物"}]),
]
def _authorization() -> dict[str, object]:
"""构造通过既有授权门禁的真实原文件授权投影。"""
snapshot = {
"id": "1",
"version": "auth-work-8-test-v1",
"immutable": True,
"sourceHash": SOURCE_HASH,
"sourceVersion": SOURCE_VERSION,
"sourceStatus": "active",
"copyrightStatus": "research_only",
"allowedPurpose": ["offline_evaluation"],
"forbiddenPurpose": ["external_distribution"],
"authorizationBasis": "user_authorization",
"authorizedBy": "user:1",
"displaySummary": "仅供离线评测",
"checkedAt": "2026-07-19T00:00:00+00:00",
"expiresAt": None,
"revalidationAt": "2099-07-19T00:00:00+00:00",
}
return {
"sourceStatus": "active",
"copyrightStatus": "research_only",
"sourceHash": SOURCE_HASH,
"sourceVersion": SOURCE_VERSION,
"allowedPurpose": ["offline_evaluation"],
"forbiddenPurpose": ["external_distribution"],
"authorizationSnapshot": snapshot,
}
def _block_row(chapter: int, text: str | None = None) -> dict[str, object]:
"""构造单章唯一 Canonical block；默认正文长度只用于小型测试。"""
body = text if text is not None else f"第{chapter}章历史正文"
return {
"chapter": chapter,
"chapter_id": 10000 + chapter,
"chapter_status": "published",
"block_id": 20000 + chapter,
"block_order": 1,
"revision": 1,
"content_text": body,
}
def _assembly_rows(base_config: dict[str, object]) -> dict[str, object]:
"""按预注册 Han 计数构造可通过真实 dry-run 的纯数据快照。"""
recent_counts: dict[int, int] = {}
target_scaffolds: list[dict[str, object]] = []
for sample in base_config["samples"]:
for chapter, count in zip(
sample["targetLengthBasis"]["sourceChapters"],
sample["frozenRecentHanCounts"],
strict=True,
):
recent_counts[chapter] = count
target = sample["targetChapter"]
target_scaffolds.append(
{
"id": 30000 + target,
"chapter": target,
"title": f"第{target}章目标",
"outline_text": f"第{target}章合法目标细纲，完成预注册场景。",
"entities": [],
"pattern_hints": [],
}
)
extra_card_chapters = {188, 189, 190, 312, 313, 454, 460, 461, 462, 465, 467, 484, 485, 487}
proxy_names = {
188: "联赛系统",
189: "联赛系统",
190: "赛莉丝",
312: "苏铭",
313: "苏铭",
320: "苏铭",
454: "蒂米",
460: "伊蕾莉雅",
461: "伊蕾莉雅",
462: "伊蕾莉雅",
465: "蒂米",
467: "蒂米",
484: "安若雪",
485: "安若雪",
487: "安若雪",
}
block_rows = []
for chapter, count in recent_counts.items():
prefix = proxy_names.get(chapter, "")
block_rows.append(_block_row(chapter, prefix + "文" * (count - loader.han_count(prefix))))
block_rows.extend(
_block_row(chapter, proxy_names[chapter] + "文" * 2500)
for chapter in sorted(extra_card_chapters - set(recent_counts))
)
# oracle 夹具覆盖 1..543 的完整 Canonical 历史；装配所需长近章覆盖同章短文本。
oracle_by_chapter = {
chapter: _block_row(chapter) for chapter in range(1, 544)
}
oracle_by_chapter.update({row["chapter"]: copy.deepcopy(row) for row in block_rows})
return {
"work": {"id": 8, "title": "深空之影", "chapter_count": 594},
"reference": {"id": 8, "imported_chapter_count": 594},
"source": {
"sourceHash": SOURCE_HASH,
"sourceVersion": SOURCE_VERSION,
"fileName": "深空之影_远瞳.txt",
},
"authorization": _authorization(),
"target_scaffolds": target_scaffolds,
"card_rows": _selected_cards(),
"block_rows": block_rows,
"oracle_block_rows": [oracle_by_chapter[chapter] for chapter in range(1, 544)],
"canonical_character_mentions": [
{
"sample_id": "deep-space-489-battle",
"name": "加特朗",
"as_of_chapter": 488,
"first_chapter": None,
"hit_chapters": [],
},
{
"sample_id": "deep-space-321-character-dialogue",
"name": "莫妮卡",
"as_of_chapter": 320,
"first_chapter": 317,
"hit_chapters": [317],
},
{
"sample_id": "deep-space-199-information-reveal",
"name": "赛莉丝",
"as_of_chapter": 198,
"first_chapter": 190,
"hit_chapters": [190],
},
{
"sample_id": "deep-space-523-returning-character",
"name": "伊蕾莉雅",
"as_of_chapter": 522,
"first_chapter": 460,
"hit_chapters": [460, 461, 462],
},
],
}
class FakeQueryResult:
"""提供 psycopg 结果对象的最小 fetch 接口。"""
def __init__(self, rows: object):
self.rows = rows if isinstance(rows, list) else [rows]
def fetchone(self):
"""返回第一行或空值。"""
return self.rows[0] if self.rows else None
def fetchall(self):
"""返回全部测试行。"""
return self.rows
class FakeReadOnlyConnection:
"""记录全部 SQL，证明所有读取都处在同一只读事务。"""
def __init__(self, rows: dict[str, object]):
self.rows = rows
self.queries: list[str] = []
def __enter__(self):
"""模拟 psycopg 连接上下文。"""
return self
def __exit__(self, exc_type, exc_value, traceback):
"""测试连接不吞掉异常。"""
return False
def execute(self, query, params=None):
"""按表名路由固定结果，并保留事务语句供断言。"""
del params
sql = " ".join(str(query).split())
self.queries.append(sql)
if sql.startswith("SET TRANSACTION"):
return FakeQueryResult([])
if "FROM muse_content_work" in sql:
return FakeQueryResult(self.rows["work"])
if "FROM example_reference_work" in sql:
return FakeQueryResult([{**self.rows["reference"], "source_file": "深空之影_远瞳.txt", "deleted": False}])
if "FROM muse_content_import_task" in sql:
return FakeQueryResult(
[{
"id": 11,
"status": "succeeded",
"command_id": f"import-{FILE_HASH[:16]}",
"source_snapshot": {"file": "深空之影_远瞳.txt"},
"deleted": False,
}]
)
if "FROM muse_knowledge_document" in sql:
return FakeQueryResult(
[{"id": 12, "file_name": "深空之影_远瞳.txt", "file_hash": FILE_HASH, "deleted": False}]
)
if "FROM example_reference_authorization_snapshot" in sql:
snapshot = self.rows["authorization"]["authorizationSnapshot"]
return FakeQueryResult(
{
"id": 1,
"snapshot_version": snapshot["version"],
"source_hash": SOURCE_HASH,
"source_version": SOURCE_VERSION,
"copyright_status": "research_only",
"source_status": "active",
"allowed_purpose": ["offline_evaluation"],
"forbidden_purpose": ["external_distribution"],
"authorization_basis": "user_authorization",
"authorized_by": "user:1",
"display_summary": "仅供离线评测",
"checked_at": snapshot["checkedAt"],
"expires_at": None,
"revalidation_at": snapshot["revalidationAt"],
}
)
if "FROM example_parse_scaffold" in sql:
return FakeQueryResult(self.rows["target_scaffolds"])
if "FROM muse_knowledge_draft" in sql:
return FakeQueryResult(self.rows["card_rows"])
if "WITH probes AS" in sql:
return FakeQueryResult(self.rows["canonical_character_mentions"])
if (
"FROM muse_content_chapter ch" in sql
and "muse_content_block" in sql
and "ch.order_no<=%s" in sql
):
return FakeQueryResult(self.rows["oracle_block_rows"])
if "FROM muse_content_chapter ch" in sql and "muse_content_block" in sql:
return FakeQueryResult(self.rows["block_rows"])
raise AssertionError(f"未处理 SQL: {sql}")
class LoadWriterReferenceWorkTest(unittest.TestCase):
"""覆盖选择、冻结、事务和真实 Writer dry-run 合同。"""
def setUp(self):
"""每个测试都从仓内预注册输入与稳定选择器开始。"""
self.base_config = json.loads(BASE_CONFIG_PATH.read_text(encoding="utf-8"))
# loader 仍验证“装配不得补批准”合同；生产 Gate A 配置已单独获得预算批准。
budget = self.base_config["executionAuthorization"]["budget"]
budget["status"] = "pending"
budget.pop("totalBudgetUsd", None)
_refresh_self_hash(budget)
self.selectors = json.loads(SELECTOR_PATH.read_text(encoding="utf-8"))
def test_default_pattern_searcher_binds_loader_dsn_and_tenant(self):
"""公共范式检索不得退回 search 模块的默认连接常量。"""
calls = []
def fake_search_cards(intent, **kwargs):
calls.append((intent, kwargs))
return []
fake_module = types.SimpleNamespace(search_cards=fake_search_cards)
with patch.dict(sys.modules, {"search": fake_module}):
searcher = loader._default_pattern_card_searcher(
dsn="postgresql://test/isolated",
tenant_id=9,
)
self.assertEqual(searcher("转折", ttype="craft", top=2), [])
self.assertEqual(calls[0][0], "转折")
self.assertEqual(
calls[0][1],
{
"scope": "public_pattern",
"ttype": "craft",
"purpose": "generation",
"top": 2,
"dsn": "postgresql://test/isolated",
"tenant_id": 9,
},
)
def test_resolves_exact_canonical_name_and_alias(self):
"""canonical name 与精确 alias 都必须唯一解析，不能模糊挑卡。"""
resolved = resolve_card_selectors(_selected_cards(), self.selectors)
self.assertEqual(resolved["deep-space-489-battle"][0]["id"], 101)
self.assertEqual(
resolved["deep-space-489-battle"][0]["draft_payload"]["名称"],
"安若雪",
)
self.assertEqual(
resolved["deep-space-321-character-dialogue"][0]["draft_payload"]["名称"],
"苏铭",
)
self.assertEqual(
resolved["deep-space-544-turning-point"][0]["draft_payload"]["名称"],
"蒂米",
)
self.assertEqual(resolved["deep-space-199-information-reveal"][0]["id"], 104)
self.assertEqual(resolved["deep-space-199-information-reveal"][1]["id"], 105)
self.assertEqual(
resolved["deep-space-199-information-reveal"][0]["draft_payload"]["名称"],
"璐茜",
)
self.assertEqual(
resolved["deep-space-199-information-reveal"][1]["draft_payload"]["名称"],
"三校联赛",
)
selected_ids = [str(card["id"]) for cards in resolved.values() for card in cards]
self.assertEqual(len(selected_ids), len(set(selected_ids)))
def test_duplicate_or_missing_selector_match_fails_closed(self):
"""同一选择器命中零张或多张卡时都不能猜测。"""
duplicate = _selected_cards() + [_card_row(999, "character", "苏铭")]
with self.assertRaises(WriterReferenceWorkError):
resolve_card_selectors(duplicate, self.selectors)
missing = [row for row in _selected_cards() if row["id"] != 106]
with self.assertRaises(WriterReferenceWorkError):
resolve_card_selectors(missing, self.selectors)
deleted_old_name = [row for row in _selected_cards() if row["id"] != 105]
deleted_old_name.append(
{
**_card_row(11374, "event", "联赛系统被机械一族入侵"),
"deleted": True,
}
)
with self.assertRaisesRegex(WriterReferenceWorkError, "实际 0 张"):
resolve_card_selectors(deleted_old_name, self.selectors)
def test_selector_hash_version_and_frozen_controls_fail_closed_on_tamper(self):
"""选择器内容、版本、scaffold 来源和上下文上限都必须与 base 公共控制一致。"""
self.assertEqual(self.base_config["commonControls"]["selectorSha256"], SELECTOR_DIGEST)
self.assertEqual(
self.base_config["commonControls"]["selectorVersion"],
"writer-gate-a-deep-space-card-selectors-v4",
)
self.assertEqual(self.base_config["strategyVersion"], "writer-abc-single-variable-v2")
probe = self.base_config["executionAuthorization"]["runtimeProbe"]
self.assertEqual(probe["status"], "successful")
self.assertEqual(
self.base_config["commonControls"]["modelVersion"], probe["resolvedModelId"]
)
self.assertEqual(len(probe["claudeExecutableSha256"]), 64)
self.assertEqual(
probe["claudeExecutableSha256"], EXPECTED_CLAUDE_EXECUTABLE_SHA256
)
self.assertEqual(
self.base_config["commonControls"]["adapterVersion"],
"writer-runtime-v1|claude-cli-"
+ probe["claudeCliVersion"]
+ "|binary-sha256-"
+ probe["claudeExecutableSha256"],
)
for control_name in ("runtimeProbe", "budget", "rawRetention"):
control = self.base_config["executionAuthorization"][control_name]
self.assertEqual(
control["receiptSha256"],
loader._sha256_value(
{key: item for key, item in control.items() if key != "receiptSha256"}
),
)
self.assertEqual(self.base_config["executionAuthorization"]["budget"]["status"], "pending")
self.assertEqual(
self.base_config["executionAuthorization"]["rawRetention"]["status"], "approved"
)
self.assertEqual(
self.base_config["commonControls"]["sampling"],
{
"temperature": "unsupported",
"topP": "unsupported",
"seed": "unsupported",
"reproducibilityClaim": "not_claimed",
},
)
self.assertFalse(self.base_config["armPolicies"]["B"]["gateThresholdEligible"])
self.assertEqual(
self.base_config["armPolicies"]["B"]["evidenceStrategy"],
"card_only_diagnostic",
)
tampered = json.loads(json.dumps(self.selectors, ensure_ascii=False))
tampered["samples"][0]["cards"][0]["name"] = "被篡改"
with self.assertRaisesRegex(WriterReferenceWorkError, "SHA-256"):
assemble_writer_gate_config(
base_config=self.base_config,
selector_config=tampered,
selector_digest=loader.selector_sha256(
json.dumps(tampered, ensure_ascii=False).encode("utf-8")
),
rows=_assembly_rows(self.base_config),
)
invalid = json.loads(json.dumps(self.base_config, ensure_ascii=False))
invalid["commonControls"]["writerInputProvenance"] = "runtime_scaffold"
with self.assertRaisesRegex(WriterReferenceWorkError, "writerInputProvenance"):
assemble_writer_gate_config(
base_config=invalid,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=_assembly_rows(self.base_config),
)
invalid = json.loads(json.dumps(self.base_config, ensure_ascii=False))
invalid["samples"][0]["writerContextInput"]["tokenBudget"]["maxContextChars"] += 1
with self.assertRaisesRegex(WriterReferenceWorkError, "原样使用"):
assemble_writer_gate_config(
base_config=invalid,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=_assembly_rows(self.base_config),
)
synchronized_tamper = json.loads(json.dumps(self.base_config, ensure_ascii=False))
synchronized_tamper["commonControls"]["maxContextChars"] = 150000
for sample in synchronized_tamper["samples"]:
sample["writerContextInput"]["tokenBudget"]["maxContextChars"] = 150000
with self.assertRaisesRegex(WriterReferenceWorkError, "严格等于预注册固定值 140000"):
assemble_writer_gate_config(
base_config=synchronized_tamper,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=_assembly_rows(self.base_config),
)
def test_runtime_probe_status_tamper_fails_closed(self):
"""runtime probe 不是成功状态时必须在 loader 控制门阻断。"""
tampered = copy.deepcopy(self.base_config)
tampered["executionAuthorization"]["runtimeProbe"]["status"] = "failed"
with self.assertRaisesRegex(WriterReferenceWorkError, "runtimeProbe.*successful"):
loader._validate_loader_controls(
tampered,
self.selectors,
selector_digest=SELECTOR_DIGEST,
)
def test_runtime_probe_self_hash_tamper_fails_closed(self):
"""runtime probe receipt 的 canonical 自哈希被篡改时必须失败关闭。"""
tampered = copy.deepcopy(self.base_config)
tampered["executionAuthorization"]["runtimeProbe"]["receiptSha256"] = (
"sha256:" + "0" * 64
)
with self.assertRaisesRegex(WriterReferenceWorkError, "runtimeProbe.*自哈希"):
loader._validate_loader_controls(
tampered,
self.selectors,
selector_digest=SELECTOR_DIGEST,
)
def test_model_version_tamper_fails_closed(self):
"""公共 modelVersion 必须精确绑定 probe 解析出的模型 ID。"""
tampered = copy.deepcopy(self.base_config)
tampered["commonControls"]["modelVersion"] = "claude-opus-tampered"
with self.assertRaisesRegex(WriterReferenceWorkError, "modelVersion.*resolvedModelId"):
loader._validate_loader_controls(
tampered,
self.selectors,
selector_digest=SELECTOR_DIGEST,
)
def test_adapter_version_tamper_fails_closed(self):
"""公共 adapterVersion 必须同时绑定 probe CLI 版本和完整 executable hash。"""
tampered = copy.deepcopy(self.base_config)
tampered["commonControls"]["adapterVersion"] = "writer-runtime-v1|tampered"
with self.assertRaisesRegex(WriterReferenceWorkError, "adapterVersion"):
loader._validate_loader_controls(
tampered,
self.selectors,
selector_digest=SELECTOR_DIGEST,
)
def test_planned_calls_are_validated_before_any_database_connection(self):
"""plannedCalls 缺失、低于样本需求或超过 maxCalls 时不得触碰数据库。"""
targets = [int(item["targetChapter"]) for item in self.selectors["samples"]]
cases = (
(
"missing",
lambda budget: budget.pop("plannedCalls"),
"plannedCalls",
),
(
"insufficient",
lambda budget: budget["plannedCalls"].update({"writer": 14}),
"plannedCalls.writer",
),
(
"over-max",
lambda budget: budget["plannedCalls"].update({"writer": 151}),
"maxCalls.writer",
),
)
for name, mutate, message in cases:
with self.subTest(name=name):
tampered = copy.deepcopy(self.base_config)
budget = tampered["executionAuthorization"]["budget"]
mutate(budget)
_refresh_self_hash(budget)
with patch.object(loader.psycopg, "connect") as connect:
with self.assertRaisesRegex(WriterReferenceWorkError, message):
load_writer_reference_rows(
dsn="postgresql://unused",
tenant_id=1,
work_id=8,
targets=targets,
base_config=tampered,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
)
connect.assert_not_called()
def test_approved_budget_preserves_planned_calls_through_assembly(self):
"""loader 只装配真实正文，不把已批准计划改写或降级。"""
approved = json.loads(BASE_CONFIG_PATH.read_text(encoding="utf-8"))
rows = _assembly_rows(approved)
assembled = assemble_writer_gate_config(
base_config=approved,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=rows,
)
budget = assembled["executionAuthorization"]["budget"]
self.assertEqual(budget["status"], "approved")
self.assertEqual(
budget["plannedCalls"],
{"writer": 60, "semantic_detector": 24, "blind_judge": 45},
)
def test_executable_hash_must_remain_full_and_bound(self):
"""probe executable hash 截断后即使重签 receipt 也不能通过。"""
tampered = copy.deepcopy(self.base_config)
probe = tampered["executionAuthorization"]["runtimeProbe"]
probe["claudeExecutableSha256"] = EXPECTED_CLAUDE_EXECUTABLE_SHA256[:-1]
_refresh_self_hash(probe)
with self.assertRaisesRegex(WriterReferenceWorkError, "64 位"):
loader._validate_loader_controls(
tampered,
self.selectors,
selector_digest=SELECTOR_DIGEST,
)
def test_stamp_raw_retention_adds_lease_and_resigns_self_hash(self):
"""盖戳补 retainUntil 并重签自哈希，其余授权字段原样保留。"""
raw = {
"approvedBy": "user",
"authorizationId": "raw-work-8-gate-a-v1",
"purpose": "offline_evaluation",
"reason": "测试夹具",
"status": "approved",
"receiptSha256": "sha256:" + "0" * 64,
}
config = {"executionAuthorization": {"rawRetention": raw}}
original_fields = {
key: value
for key, value in raw.items()
if key not in ("retainUntil", "receiptSha256")
}
loader._stamp_raw_retention(config)
stamped = config["executionAuthorization"]["rawRetention"]
# (a) 出现 retainUntil
self.assertIn("retainUntil", stamped)
# (b) 带时区的 ISO，能 fromisoformat 解析且 tzinfo 非空
parsed = datetime.fromisoformat(stamped["retainUntil"])
self.assertIsNotNone(parsed.tzinfo)
# (c) 解析后在未来、且距 now 不超过 24 小时（实际约等于 23 小时）
now = datetime.now(timezone.utc)
self.assertGreater(parsed, now)
self.assertLessEqual(parsed - now, timedelta(hours=24))
# (d) receiptSha256 == 去掉 receiptSha256 的整段规范自哈希（重签正确）
self.assertEqual(
stamped["receiptSha256"],
loader._sha256_value(
{key: value for key, value in stamped.items() if key != "receiptSha256"}
),
)
# (e) 除 retainUntil 与 receiptSha256 外其余字段不变
self.assertEqual(
{
key: value
for key, value in stamped.items()
if key not in ("retainUntil", "receiptSha256")
},
original_fields,
)
def test_assembly_preserves_execution_authorization_and_blocks_before_side_effects(self):
"""装配不得补批准；真实 execute 应因 pending budget 阻断且不创建 vault/runner。"""
expected_authorization = copy.deepcopy(self.base_config["executionAuthorization"])
base_raw_receipt = self.base_config["executionAuthorization"]["rawRetention"]["receiptSha256"]
config = assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=_assembly_rows(self.base_config),
)
output_authorization = config["executionAuthorization"]
# 盖戳只补 rawRetention 的租约到期时间戳；budget / runtimeProbe 等其余授权原样保留。
self.assertEqual(
{key: value for key, value in output_authorization.items() if key != "rawRetention"},
{key: value for key, value in expected_authorization.items() if key != "rawRetention"},
)
self.assertEqual(output_authorization["budget"]["status"], "pending")
# 输出 rawRetention：授权字段不变，只多 retainUntil，且 receiptSha256 重签有效。
output_raw = output_authorization["rawRetention"]
expected_raw = expected_authorization["rawRetention"]
self.assertEqual(output_raw["status"], "approved")
self.assertIn("retainUntil", output_raw)
self.assertEqual(
{key: value for key, value in output_raw.items() if key not in ("retainUntil", "receiptSha256")},
{key: value for key, value in expected_raw.items() if key != "receiptSha256"},
)
self.assertEqual(
output_raw["receiptSha256"],
loader._sha256_value(
{key: value for key, value in output_raw.items() if key != "receiptSha256"}
),
)
# base_config 的 rawRetention 未被污染：仍无 retainUntil，原 receiptSha256 不变。
base_raw = self.base_config["executionAuthorization"]["rawRetention"]
self.assertNotIn("retainUntil", base_raw)
self.assertEqual(base_raw["receiptSha256"], base_raw_receipt)
with tempfile.TemporaryDirectory(dir="/private/tmp") as directory:
output_dir = pathlib.Path(directory) / "run"
with patch.object(replay_module, "RawVaultManager") as vault_manager, patch.object(
replay_module.subprocess, "run"
) as runner:
result = run_writer_replay(
config,
run_id="loader-pending-budget",
output_dir=output_dir,
execute=True,
)
self.assertFalse(result["ok"])
self.assertEqual(result["status"], "blocked_budget_authorization")
self.assertEqual(result["errors"], ["BUDGET_AUTHORIZATION_REQUIRED"])
vault_manager.assert_not_called()
runner.assert_not_called()
def test_milestone_ranges_expand_and_keep_latest_three_unique_chapters(self):
"""明确区间按绝对章展开，重复章去重后只保留最近三章。"""
chapters = milestone_reference_chapters(
[
{"章": 3, "台阶": "早期"},
{"章": "5-8", "台阶": "区间"},
{"章": 8, "台阶": "重复"},
],
as_of=8,
)
self.assertEqual(chapters, [6, 7, 8])
def test_future_block_or_non_continuous_recent_chapters_fail_closed(self):
"""目标章正文和缺章近章都不能进入装配结果。"""
with self.assertRaises(WriterReferenceWorkError):
loader.index_unique_canonical_blocks([_block_row(9)], allowed_chapters={8})
with self.assertRaises(WriterReferenceWorkError):
loader.select_recent_canonical_blocks(
[_block_row(5), _block_row(6), _block_row(8)],
as_of=8,
)
repository = loader.SnapshotProseRepository({8: _block_row(8)})
with self.assertRaises(WriterReferenceWorkError):
repository.read_source_refs(
work_id=8,
as_of=8,
source_refs=[
{
"sourceId": "chapter:9:block:20009",
"sourceVersion": "chapter:9:block:20009:revision:1",
"chapter": 9,
"blockId": 20009,
"startCodePoint": 0,
"endCodePoint": 1,
}
],
)
def test_han_count_drift_blocks_assembly(self):
"""真实近章 Han 计数只要一章漂移就必须阻断。"""
rows = _assembly_rows(self.base_config)
rows["block_rows"][0]["content_text"] += "文"
with self.assertRaisesRegex(WriterReferenceWorkError, "Han 计数漂移"):
assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=rows,
)
def test_all_database_reads_share_one_repeatable_read_only_transaction(self):
"""来源、授权、scaffold、卡、正文与 oracle 必须共享一个只读快照。"""
rows = _assembly_rows(self.base_config)
connection = FakeReadOnlyConnection(rows)
with patch.object(loader.psycopg, "connect", return_value=connection) as connect:
loaded = load_writer_reference_rows(
dsn="postgresql://unused",
tenant_id=1,
work_id=8,
targets=[199, 321, 489, 523, 544],
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
)
connect.assert_called_once()
self.assertEqual(
connection.queries.count("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"),
1,
)
self.assertEqual(len(loaded["target_scaffolds"]), 5)
self.assertEqual(len(loaded["card_rows"]), 6)
self.assertEqual(
{row["chapter"] for row in loaded["oracle_block_rows"]},
set(range(1, 544)),
)
mentions = {item["name"]: item for item in loaded["canonical_character_mentions"]}
self.assertEqual(mentions["加特朗"]["hit_chapters"], [])
self.assertEqual(mentions["莫妮卡"]["first_chapter"], 317)
def test_complete_real_content_config_passes_writer_dry_run(self):
"""纯数据装配出的 canonical_frozen_prose 五样本配置通过真实 dry-run。"""
config = assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=_assembly_rows(self.base_config),
)
self.assertEqual(len(config["samples"]), 5)
self.assertTrue(
all(
sample["writerContextInput"]["contentMode"] == "canonical_frozen_prose"
for sample in config["samples"]
)
)
for sample in config["samples"]:
context = sample["writerContextInput"]
retrieval = context["retrievalResult"]
self.assertEqual(retrieval["factEvidence"], [])
self.assertEqual(context["tokenBudget"]["maxContextChars"], 140000)
self.assertTrue(all(len(card["sourceRefs"]) <= 3 for card in retrieval["cards"]))
self.assertTrue(
all(
ref["sourceType"] == "card_chapter_proxy"
for card in retrieval["cards"]
for ref in card["sourceRefs"]
)
)
self.assertTrue(
all(
evidence["purpose"] == "card_chapter_proxy"
for evidence in retrieval["proseEvidence"]
if evidence["retrievalArm"] == "C"
)
)
self.assertTrue(
all(
evidence["purpose"] == "generic_historical_prose"
for evidence in retrieval["proseEvidence"]
if evidence["retrievalArm"] == "A"
)
)
self.assertTrue(
all(
evidence["chapter"] <= sample["asOfChapter"]
for evidence in [*context["recentChapters"], *retrieval["proseEvidence"]]
)
)
receipt = config["writerContextDiffReceipts"][sample["sampleId"]]
self.assertTrue(receipt["ok"])
self.assertTrue(receipt["allowedDifferencePaths"])
self.assertNotEqual(receipt["contextSha256"]["A"], receipt["contextSha256"]["C"])
self.assertTrue(
all(
path.startswith(("$.factConstraints", "$.proseExcerpts"))
for path in receipt["allowedDifferencePaths"]
)
)
self.assertEqual(
receipt["proseCharCount"],
{"A": sample["proseCharBudget"], "C": sample["proseCharBudget"]},
)
result = run_writer_replay(config, run_id="writer-loader-test")
self.assertTrue(result["ok"], result)
self.assertEqual(result["status"], "ready")
self.assertTrue(
all(sample["leakageAudit"]["findingCount"] == 0 for sample in result["samples"])
)
ratios = {sample["targetChapter"]: sample["newCharacterRatio"] for sample in config["samples"]}
statuses = {
sample["targetChapter"]: sample["newCharacterRatioStatus"]
for sample in config["samples"]
}
self.assertEqual(ratios, {489: 1.0, 321: 0.0, 544: None, 199: 0.0, 523: 0.0})
self.assertEqual(statuses[544], "unresolved_generic_role")
def test_requirement_hard_constraints_orders_and_guards(self):
"""要求硬约束按固定顺序产出，且对非法锚点/角色/距离做类型保护。"""
# 四类要求齐全：顺序固定为 章末钩子→伏笔动作→硬事件→必须出场角色。
requirements = {
"chapterEndHook": {
"requirementId": "hook-x",
"anchors": ["立场", "态度"],
"maxDistanceFromEnd": 120,
},
"foreshadowingActions": [
{"requirementId": "fs-1", "anchors": ["旧徽章"]},
{"requirementId": "fs-2", "anchors": []}, # 空锚点 → 跳过整条
],
"requiredEvents": [{"requirementId": "ev-1", "anchors": ["内应", "反水"]}],
"requiredCharacters": ["内应", "林澈"],
}
self.assertEqual(
_requirement_hard_constraints(requirements),
[
"章末钩子（硬要求）：正文结尾 120 字内必须出现以下锚点之一：立场、态度",
"伏笔动作（硬要求）：正文必须自然埋入以下锚点之一：旧徽章",
"硬事件（硬要求）：正文必须命中以下锚点之一：内应、反水",
"必须出场角色（硬要求）：正文必须出现以下角色：内应、林澈",
],
)
# 空 requirements / 非对象 → 空列表。
self.assertEqual(_requirement_hard_constraints({}), [])
self.assertEqual(_requirement_hard_constraints(None), [])
self.assertEqual(_requirement_hard_constraints(["不是对象"]), [])
# 类型保护：非字符串锚点跳过；anchors 非列表跳过整条；非正整数距离跳过章末钩子。
guarded = {
"chapterEndHook": {"anchors": ["立场", 123, None], "maxDistanceFromEnd": 120},
"foreshadowingActions": [{"anchors": ["旧徽章", 42]}],
"requiredEvents": [{"anchors": "不是列表"}],
"requiredCharacters": ["内应", 7, ""],
}
self.assertEqual(
_requirement_hard_constraints(guarded),
[
"章末钩子（硬要求）：正文结尾 120 字内必须出现以下锚点之一：立场",
"伏笔动作（硬要求）：正文必须自然埋入以下锚点之一：旧徽章",
"必须出场角色（硬要求）：正文必须出现以下角色：内应",
],
)
self.assertEqual(
_requirement_hard_constraints(
{"chapterEndHook": {"anchors": ["立场"], "maxDistanceFromEnd": 0}}
),
[],
)
self.assertEqual(
_requirement_hard_constraints(
{"chapterEndHook": {"anchors": ["立场"], "maxDistanceFromEnd": True}}
),
[],
)
def test_assembly_surfaces_requirements_into_fine_outline_hard_constraints(self):
"""装配出的写手细纲 hardConstraints 既含大纲文字，也含要求硬约束。"""
config = assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=_assembly_rows(self.base_config),
)
sample_544 = next(
sample
for sample in config["samples"]
if sample["sampleId"] == "deep-space-544-turning-point"
)
hard_constraints = sample_544["writerContextInput"]["fineOutline"]["hardConstraints"]
# 第一条仍是细纲大纲文字，要求硬约束追加在后。
self.assertIn("第544章合法目标细纲", hard_constraints[0])
joined = "\n".join(hard_constraints)
self.assertIn("章末钩子（硬要求）：正文结尾 120 字内必须出现以下锚点之一：立场", hard_constraints)
self.assertIn("硬事件（硬要求）：正文必须命中以下锚点之一：内应、反水", hard_constraints)
self.assertIn("必须出场角色（硬要求）：正文必须出现以下角色：内应", hard_constraints)
self.assertIn("立场", joined)
# requirements 本身未被改写：装配出的 requirements 与 base 一致。
base_544 = next(
sample
for sample in self.base_config["samples"]
if sample["sampleId"] == "deep-space-544-turning-point"
)
self.assertEqual(
sample_544["writerContextInput"]["requirements"],
base_544["writerContextInput"]["requirements"],
)
def test_writer_passes_gate_when_body_hits_hard_constraint_anchors(self):
"""写手只要按 hardConstraints 写到锚点，机械门就放行（章末钩子联动）。"""
sample_544 = next(
sample
for sample in self.base_config["samples"]
if sample["sampleId"] == "deep-space-544-turning-point"
)
requirements = sample_544["writerContextInput"]["requirements"]
# 构造命中全部门禁锚点的候选正文：含硬事件/角色「内应」，结尾 120 字内含章末钩子「立场」。
head = "内应在暗处反水"
tail = "他终于表明立场"
body = head + "文" * (4000 - han_count(head) - han_count(tail)) + tail
context = _bound_context()
candidate = build_candidate_envelope(context, {"candidateBody": body})
report = check_writer_candidate(context, candidate, requirements)
self.assertTrue(report["passed"], report["blockingFailures"])
# 反证：结尾换成无锚点文字，章末钩子门禁必挂 CHAPTER_END_HOOK_MISSING。
bad_tail = "他沉默不语良久"
bad_body = head + "文" * (4000 - han_count(head) - han_count(bad_tail)) + bad_tail
bad_candidate = build_candidate_envelope(context, {"candidateBody": bad_body})
bad_report = check_writer_candidate(context, bad_candidate, requirements)
self.assertFalse(bad_report["passed"])
self.assertIn(
"CHAPTER_END_HOOK_MISSING",
{item["code"] for item in bad_report["blockingFailures"]},
)
def test_loader_rejects_zero_preregistered_prose_budget(self):
"""canonical loader 不得生成 A/C 创作输入相同的零预算正式样本。"""
base_config = copy.deepcopy(self.base_config)
base_config["samples"][0]["proseCharBudget"] = 0
with self.assertRaisesRegex(WriterReferenceWorkError, "proseCharBudget.*正整数"):
assemble_writer_gate_config(
base_config=base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=_assembly_rows(base_config),
)
def test_oracle_truth_pack_uses_only_full_canonical_history_and_target_scaffold(self):
"""oracle pack 只可由冻结历史与目标 scaffold 构建并严格绑定授权和哈希。"""
rows = _assembly_rows(self.base_config)
config = assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=rows,
)
packs = config["oracleTruthPacks"]
self.assertEqual(len(packs), 5)
for sample in config["samples"]:
pack = packs[sample["sampleId"]]
self.assertEqual(pack["schemaVersion"], "oracle-truth-pack-v1")
self.assertEqual(pack["asOf"], sample["asOfChapter"])
self.assertEqual(pack["authorizationSnapshotId"], "1")
self.assertEqual(
{item["chapterBoundary"]["maxChapter"] for item in pack["historicalAssertions"]},
set(range(1, sample["asOfChapter"] + 1)),
)
self.assertTrue(
all(
item["chapterBoundary"]["maxChapter"] <= sample["asOfChapter"]
for item in pack["historicalAssertions"]
)
)
self.assertTrue(
all(
item["chapterBoundary"]["minChapter"] == sample["targetChapter"]
for item in pack["targetAssertions"]
)
)
serialized = json.dumps(pack, ensure_ascii=False, sort_keys=True)
self.assertNotIn("cardId", serialized)
self.assertNotIn("retrievalManifest", serialized)
self.assertTrue(pack["packSha256"].startswith("sha256:"))
def test_oracle_truth_pack_rejects_card_derived_or_incomplete_history(self):
"""oracle 历史缺章或断言来源伪装成卡派生时必须失败关闭。"""
rows = _assembly_rows(self.base_config)
rows["oracle_block_rows"] = [
row for row in rows["oracle_block_rows"] if row["chapter"] != 100
]
with self.assertRaisesRegex(WriterReferenceWorkError, "oracle.*缺章"):
assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=rows,
)
valid_rows = _assembly_rows(self.base_config)
config = assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=valid_rows,
)
tampered = copy.deepcopy(config["oracleTruthPacks"]["deep-space-489-battle"])
tampered["historicalAssertions"][0]["statement"] = "卡派生伪断言"
with self.assertRaisesRegex(WriterReferenceWorkError, "内容哈希"):
validate_oracle_truth_pack(tampered)
extra = copy.deepcopy(config["oracleTruthPacks"]["deep-space-489-battle"])
extra["retrievalManifest"] = {}
with self.assertRaisesRegex(WriterReferenceWorkError, "顶层字段"):
validate_oracle_truth_pack(extra)
def test_character_ratio_comes_from_canonical_text_not_contaminated_card(self):
"""卡内塞入角色名不能把 Canonical 正文从未出现的角色伪装成已知。"""
rows = _assembly_rows(self.base_config)
rows["card_rows"][0]["draft_payload"]["别名"].append("加特朗")
config = assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=rows,
)
samples = {item["targetChapter"]: item for item in config["samples"]}
self.assertEqual(samples[489]["newCharacterBasis"]["absentBeforeAsOf"], ["加特朗"])
self.assertEqual(samples[489]["newCharacterRatio"], 1.0)
self.assertEqual(samples[321]["newCharacterBasis"]["knownBeforeAsOf"], ["莫妮卡"])
def test_derived_chapter_proxy_requires_name_or_preregistered_alias_in_each_chapter(self):
"""整章代理若不能在对应正文命中规范名或预注册别名，必须失败关闭。"""
rows = _assembly_rows(self.base_config)
target = next(row for row in rows["block_rows"] if row["chapter"] == 188)
target["content_text"] = "第188章无关历史正文"
with self.assertRaisesRegex(WriterReferenceWorkError, "规范名或合法别名"):
assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=rows,
)
def test_derived_chapter_proxy_accepts_selector_alias_only(self):
"""整章代理可用 selector 的“联赛”别名命中，不能依赖已删除旧长名。"""
rows = _assembly_rows(self.base_config)
target = next(row for row in rows["block_rows"] if row["chapter"] == 188)
target["content_text"] = target["content_text"].replace("联赛系统", "联赛", 1) + "文" * 2
config = assemble_writer_gate_config(
base_config=self.base_config,
selector_config=self.selectors,
selector_digest=SELECTOR_DIGEST,
rows=rows,
)
event_card = next(
card
for card in config["samples"][3]["writerContextInput"]["retrievalResult"]["cards"]
if card["type"] == "event"
)
self.assertEqual(event_card["name"], "三校联赛")
def test_output_must_be_new_private_tmp_child(self):
"""配置只能写入 /private/tmp 的全新独立子目录。"""
config = {"profile": "writer_replay"}
with tempfile.TemporaryDirectory(dir="/private/tmp") as parent:
output_dir = pathlib.Path(parent) / "writer-config"
path = write_temporary_config(config, output_dir)
self.assertEqual(path, output_dir / "config.json")
self.assertEqual(json.loads(path.read_text(encoding="utf-8")), config)
self.assertEqual(stat.S_IMODE(output_dir.stat().st_mode), 0o700)
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600)
with self.assertRaises(WriterReferenceWorkError):
write_temporary_config(config, output_dir)
with self.assertRaises(WriterReferenceWorkError):
write_temporary_config(config, SCRIPT_DIR / "forbidden-output")
if __name__ == "__main__":
unittest.main()
Reference in New Issue View Git Blame Copy Permalink