c9f69d9d6d
范围（不含 design-story-foundation、docs/、humanization/README.md 等进行中改动）:
1. 新增 harness/ 控制平面
- skill_harness.py 静态审计：32 个运行时 Skill 的 frontmatter/manifest/文档污染，当前 0 问题
- run_selected.py 选择性执行器：manifest 与磁盘一一对账、依赖阻断、
空跑与 skip-only 失败关闭、AST 测试形状门
- manifests/skills.json：32 个 Skill 的合同责任方与协作领域登记
- manifests/test-inventory.json：81 个测试资产登记
- specs/skill-testing.md 与 README.md：测试分层、证据边界与 harness 职责
2. 实现测试从 .claude/skills/*/scripts/ 迁至 tests/skills/<skill>/
- 71 个测试文件迁移并修复项目根与临时目录运行导入
- 数据库触发器测试宽泛异常收窄为 psycopg.errors.RaiseException
- 抽取离线大测试拆出真实 PG smoke（默认阻断，不计入离线通过）
- 抽取 presence 去重边界拆出独立测试：493 + 78 = 571 项检查不变
3. 运行时文档清理
- 13 个 SKILL.md 移除自测/离线验证段落、测试命令与测试文件事实源表述，
只保留运行时合同；业务运行合同、额度、授权与离线模式均保留
4. SoT 同步
- AGENTS.md：新增 Skill 领域索引（7 个合同责任方分组，覆盖 32 个运行时 Skill）
- 领域 07：测试入口改由 harness/manifests/ 登记，SKILL.md 不承载测试命令
- humanization 覆盖矩阵：活动测试路径同步迁移
验证证据: harness 自测 15 项 + runner 自测 13 项通过；静态审计 32 Skill / 0 问题；
73 个非数据库测试通过；8 个集成条目中 6 个 PostgreSQL 项被依赖门明确阻断；
py_compile 与 git diff --check 通过。未连接 PostgreSQL、网络、真实模型或额度。
已知边界: 真正 skill_behavior_eval 仍为 0，尚未验证任何 Skill 自然语言行为；
evaluate-frozen-replay 的 raw 存储边界冲突留待单独治理。
362 lines
15 KiB
Python
362 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""参考作品只读适配器的纯数据组装测试。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import inspect
|
|
import pathlib
|
|
import sys
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
SCRIPT_DIR = PROJECT_ROOT / ".claude" / "skills" / "freeze-context" / "scripts"
|
|
sys.path.insert(0, str(SCRIPT_DIR))
|
|
import load_reference_work as loader # noqa: E402
|
|
from load_reference_work import ( # noqa: E402
|
|
AdapterError,
|
|
build_replay_config,
|
|
project_card,
|
|
)
|
|
|
|
|
|
FILE_HASH = "02cf1f8c1ca03c26e0b839d88fe536e83c0af20fd8972235b7aedca6a33becf4"
|
|
SOURCE_HASH = f"sha256:{FILE_HASH}"
|
|
SOURCE_VERSION = f"raw-file-v1:{SOURCE_HASH}"
|
|
WORK = {
|
|
"id": 8,
|
|
"title": "深空之影",
|
|
"revision": 12,
|
|
"chapter_count": 594,
|
|
"parse_status": "pending",
|
|
}
|
|
REFERENCE = {
|
|
"declared_chapter_count": 595,
|
|
"imported_chapter_count": 594,
|
|
"parse_scope": {"from": 1, "to": 594},
|
|
"parse_status": "parsing",
|
|
"source_file": "深空之影_远瞳.txt",
|
|
"deleted": False,
|
|
}
|
|
IMPORT_TASK = {
|
|
"id": 19,
|
|
"status": "succeeded",
|
|
"command_id": f"import-{FILE_HASH[:16]}",
|
|
"source_snapshot": {"file": REFERENCE["source_file"]},
|
|
"deleted": False,
|
|
}
|
|
DOCUMENT = {
|
|
"id": 7,
|
|
"file_name": REFERENCE["source_file"],
|
|
"file_hash": FILE_HASH,
|
|
"deleted": False,
|
|
}
|
|
AUTHORIZATION_ROW = {
|
|
"id": 1,
|
|
"snapshot_version": "auth-work-8-v1",
|
|
"source_hash": SOURCE_HASH,
|
|
"source_version": SOURCE_VERSION,
|
|
"copyright_status": "research_only",
|
|
"source_status": "active",
|
|
"allowed_purpose": ["offline_evaluation"],
|
|
"forbidden_purpose": ["external_distribution"],
|
|
"authorization_basis": "user_authorization",
|
|
"authorized_by": "user:1",
|
|
"display_summary": "用户授权仅用于内部离线评测",
|
|
"checked_at": "2026-07-19T00:00:00Z",
|
|
"expires_at": None,
|
|
"revalidation_at": "2099-07-19T00:00:00Z",
|
|
}
|
|
|
|
|
|
def card_row(card_id=11126, name="苏铭"):
|
|
return {
|
|
"id": card_id,
|
|
"status": "pending",
|
|
"source_type": "upgrade_book",
|
|
"source_id": 8,
|
|
"revision": 3,
|
|
"update_time": "2026-07-19T00:00:00+00:00",
|
|
"draft_payload": {
|
|
"type": "character",
|
|
"名称": name,
|
|
"别名": [f"{name}别名"],
|
|
"出场章": [1, 430, 489, 550],
|
|
"字段": {
|
|
"演变历程": [
|
|
{"章": 420, "台阶": "历史台阶"},
|
|
{"章": 489, "台阶": "未来台阶"},
|
|
],
|
|
"演变概括": "全书终态摘要，不应透传",
|
|
"境界与能力": "终局能力，不应透传",
|
|
"说话方式": "可选静态字段",
|
|
},
|
|
"一句话摘要": "终局摘要，不应透传",
|
|
},
|
|
}
|
|
|
|
|
|
class FakeQueryResult:
|
|
"""提供 psycopg 查询结果所需的最小 fetch 接口。"""
|
|
|
|
def __init__(self, rows):
|
|
self.rows = rows if isinstance(rows, list) else [rows]
|
|
|
|
def fetchone(self):
|
|
return self.rows[0] if self.rows else None
|
|
|
|
def fetchall(self):
|
|
return self.rows
|
|
|
|
|
|
class FakeReadOnlyConnection:
|
|
"""模拟只读查询，并按 SQL 中的原文件条件过滤导入任务。"""
|
|
|
|
def __init__(self, import_tasks):
|
|
self.import_tasks = import_tasks
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_value, traceback):
|
|
return False
|
|
|
|
def execute(self, query, params=None):
|
|
sql = " ".join(query.split())
|
|
if sql.startswith("SET TRANSACTION"):
|
|
return FakeQueryResult([])
|
|
if "FROM muse_content_work" in sql:
|
|
return FakeQueryResult(WORK)
|
|
if "FROM example_reference_work" in sql:
|
|
return FakeQueryResult([REFERENCE])
|
|
if "FROM muse_content_import_task" in sql:
|
|
rows = self.import_tasks
|
|
if "source_snapshot->>'file' = %s" in sql:
|
|
source_file = params[-1]
|
|
rows = [
|
|
row
|
|
for row in rows
|
|
if row.get("source_snapshot", {}).get("file") == source_file
|
|
]
|
|
return FakeQueryResult(rows)
|
|
if "FROM muse_knowledge_document" in sql:
|
|
return FakeQueryResult([DOCUMENT])
|
|
if "FROM example_reference_authorization_snapshot" in sql:
|
|
return FakeQueryResult(AUTHORIZATION_ROW)
|
|
if "FROM example_parse_outline" in sql:
|
|
return FakeQueryResult([])
|
|
if "FROM example_parse_scaffold" in sql and "ch.order_no<=%s" in sql:
|
|
return FakeQueryResult([])
|
|
if "FROM example_parse_scaffold" in sql and "ch.order_no=%s" in sql:
|
|
return FakeQueryResult(
|
|
{"id": 11, "chapter": 489, "title": "目标", "outline_text": "目标事实"}
|
|
)
|
|
if "FROM muse_knowledge_draft" in sql:
|
|
return FakeQueryResult([card_row(), card_row(11127, "赵宁")])
|
|
raise AssertionError(f"未处理 SQL: {sql}")
|
|
|
|
|
|
class LoadReferenceWorkTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.source = loader.validate_source_records([REFERENCE], [IMPORT_TASK], [DOCUMENT])
|
|
self.authorization = loader.project_authorization(AUTHORIZATION_ROW, self.source)
|
|
|
|
def test_window_freeze_uses_absolute_bounds_and_excludes_target_window(self):
|
|
config = build_replay_config(
|
|
work=WORK,
|
|
reference=REFERENCE,
|
|
outline_rows=[
|
|
{"id": 1, "window_no": 20, "from_order": 401, "to_order": 429, "outline_text": "历史窗", "check_status": "revised"},
|
|
{"id": 2, "window_no": 19, "from_order": 430, "to_order": 438, "outline_text": "目标后窗", "check_status": "revised"},
|
|
],
|
|
scaffold_rows=[
|
|
{"id": 10, "chapter": 428, "title": "前章", "outline_text": "前情", "entities": None, "pattern_hints": []},
|
|
],
|
|
target_scaffold={"id": 11, "chapter": 430, "title": "目标", "outline_text": "目标章事实"},
|
|
card_rows=[card_row(), card_row(11127, "赵宁")],
|
|
card_selection={"correctCardIds": [11126], "placeboCardIds": [11127]},
|
|
as_of=429,
|
|
target=430,
|
|
evaluation_set_version="set-test",
|
|
strategy_version="strategy-test",
|
|
source=self.source,
|
|
authorization=self.authorization,
|
|
)
|
|
windows = config["snapshot"]["data"]["outlineWindows"]
|
|
self.assertEqual([item["from_order"] for item in windows], [401])
|
|
public = json.dumps(
|
|
{
|
|
"snapshot": config["snapshot"],
|
|
"commonContext": config["commonContext"],
|
|
"arms": config["arms"],
|
|
"sources": config["sources"],
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
self.assertNotIn("目标章事实", public)
|
|
|
|
def test_card_is_eval_draft_and_history_is_frozen(self):
|
|
result = project_card(card_row(), as_of=488, source_version=SOURCE_VERSION)
|
|
encoded = json.dumps(result, ensure_ascii=False)
|
|
self.assertEqual(result["evaluationStatus"], "eval_draft")
|
|
self.assertFalse(result["productionRetrievalEligible"])
|
|
self.assertEqual([item["章"] for item in result["milestones"]], [420])
|
|
self.assertNotIn("终局摘要", encoded)
|
|
self.assertNotIn("终局能力", encoded)
|
|
self.assertEqual(result["source"]["sourceId"], "eval-draft:11126")
|
|
self.assertEqual(result["source"]["sourceVersion"], SOURCE_VERSION)
|
|
|
|
def test_missing_history_fails_closed_instead_of_using_static_card_fields(self):
|
|
row = card_row()
|
|
row["draft_payload"]["字段"].pop("演变历程")
|
|
with self.assertRaises(AdapterError):
|
|
project_card(row, as_of=488, source_version=SOURCE_VERSION)
|
|
|
|
def test_sources_have_source_id_and_version(self):
|
|
config = build_replay_config(
|
|
work=WORK,
|
|
reference=REFERENCE,
|
|
outline_rows=[{"id": 1, "from_order": 1, "to_order": 10, "outline_text": "历史窗"}],
|
|
scaffold_rows=[],
|
|
target_scaffold={"id": 11, "chapter": 489, "title": "目标", "outline_text": "目标事实"},
|
|
card_rows=[card_row(), card_row(11127, "赵宁")],
|
|
card_selection={"correctCardIds": [11126], "placeboCardIds": [11127]},
|
|
as_of=488,
|
|
target=489,
|
|
evaluation_set_version="set-test",
|
|
strategy_version="strategy-test",
|
|
source=self.source,
|
|
authorization=self.authorization,
|
|
)
|
|
self.assertTrue(config["sources"])
|
|
self.assertTrue(all(item["sourceId"] and item["sourceVersion"] for item in config["sources"]))
|
|
self.assertEqual(config["referenceWork"]["version"], SOURCE_VERSION)
|
|
self.assertEqual(config["authorization"], self.authorization)
|
|
|
|
def test_source_records_normalize_original_file_version(self):
|
|
self.assertEqual(self.source["documentId"], "7")
|
|
self.assertEqual(self.source["fileName"], REFERENCE["source_file"])
|
|
self.assertEqual(self.source["sourceHash"], SOURCE_HASH)
|
|
self.assertEqual(self.source["sourceVersion"], SOURCE_VERSION)
|
|
|
|
def test_missing_source_record_fails_closed(self):
|
|
for references, tasks, documents in (
|
|
([], [IMPORT_TASK], [DOCUMENT]),
|
|
([REFERENCE], [], [DOCUMENT]),
|
|
([REFERENCE], [IMPORT_TASK], []),
|
|
):
|
|
with self.subTest(references=len(references), tasks=len(tasks), documents=len(documents)):
|
|
with self.assertRaises(AdapterError):
|
|
loader.validate_source_records(references, tasks, documents)
|
|
|
|
def test_duplicate_source_record_fails_closed(self):
|
|
for references, tasks, documents in (
|
|
([REFERENCE, REFERENCE], [IMPORT_TASK], [DOCUMENT]),
|
|
([REFERENCE], [IMPORT_TASK, IMPORT_TASK], [DOCUMENT]),
|
|
([REFERENCE], [IMPORT_TASK], [DOCUMENT, DOCUMENT]),
|
|
):
|
|
with self.subTest(references=len(references), tasks=len(tasks), documents=len(documents)):
|
|
with self.assertRaises(AdapterError):
|
|
loader.validate_source_records(references, tasks, documents)
|
|
|
|
def test_import_task_query_ignores_successful_task_for_other_source_file(self):
|
|
unrelated = {
|
|
**IMPORT_TASK,
|
|
"id": 20,
|
|
"source_snapshot": {"file": "无关作品.txt"},
|
|
}
|
|
connection = FakeReadOnlyConnection([IMPORT_TASK, unrelated])
|
|
with patch.object(loader.psycopg, "connect", return_value=connection):
|
|
rows = loader.load_reference_rows(
|
|
dsn="postgresql://unused",
|
|
tenant_id=1,
|
|
work_id=8,
|
|
as_of=488,
|
|
target=489,
|
|
card_selection={"correctCardIds": [11126], "placeboCardIds": [11127]},
|
|
)
|
|
self.assertEqual(rows["source"]["importTaskId"], "19")
|
|
|
|
def test_import_task_query_blocks_two_tasks_for_same_source_file(self):
|
|
duplicate = {**IMPORT_TASK, "id": 20}
|
|
connection = FakeReadOnlyConnection([IMPORT_TASK, duplicate])
|
|
with patch.object(loader.psycopg, "connect", return_value=connection):
|
|
with self.assertRaises(AdapterError):
|
|
loader.load_reference_rows(
|
|
dsn="postgresql://unused",
|
|
tenant_id=1,
|
|
work_id=8,
|
|
as_of=488,
|
|
target=489,
|
|
card_selection={"correctCardIds": [11126], "placeboCardIds": [11127]},
|
|
)
|
|
|
|
def test_invalid_hash_fails_closed(self):
|
|
with self.assertRaises(AdapterError):
|
|
loader.validate_source_records([REFERENCE], [IMPORT_TASK], [{**DOCUMENT, "file_hash": "xyz"}])
|
|
|
|
def test_filename_mismatch_fails_closed(self):
|
|
bad_task = {**IMPORT_TASK, "source_snapshot": {"file": "别的文件.txt"}}
|
|
with self.assertRaises(AdapterError):
|
|
loader.validate_source_records([REFERENCE], [bad_task], [DOCUMENT])
|
|
with self.assertRaises(AdapterError):
|
|
loader.validate_source_records([REFERENCE], [IMPORT_TASK], [{**DOCUMENT, "file_name": "别的文件.txt"}])
|
|
|
|
def test_command_prefix_mismatch_fails_closed(self):
|
|
with self.assertRaises(AdapterError):
|
|
loader.validate_source_records(
|
|
[REFERENCE],
|
|
[{**IMPORT_TASK, "command_id": "import-0000000000000000"}],
|
|
[DOCUMENT],
|
|
)
|
|
|
|
def test_soft_deleted_source_record_fails_closed(self):
|
|
for references, tasks, documents in (
|
|
([{**REFERENCE, "deleted": True}], [IMPORT_TASK], [DOCUMENT]),
|
|
([REFERENCE], [{**IMPORT_TASK, "deleted": True}], [DOCUMENT]),
|
|
([REFERENCE], [IMPORT_TASK], [{**DOCUMENT, "deleted": True}]),
|
|
):
|
|
with self.subTest(references=references, tasks=tasks, documents=documents):
|
|
with self.assertRaises(AdapterError):
|
|
loader.validate_source_records(references, tasks, documents)
|
|
|
|
def test_authorization_is_projected_or_kept_blocked(self):
|
|
self.assertEqual(self.authorization["copyrightStatus"], "research_only")
|
|
self.assertEqual(self.authorization["sourceHash"], SOURCE_HASH)
|
|
self.assertEqual(self.authorization["authorizationSnapshot"]["sourceVersion"], SOURCE_VERSION)
|
|
blocked = loader.project_authorization(None, self.source)
|
|
self.assertEqual(blocked["sourceStatus"], "missing_authorization_snapshot")
|
|
self.assertEqual(blocked["allowedPurpose"], [])
|
|
self.assertEqual(blocked["authorizationSnapshot"], {})
|
|
|
|
def test_work_revision_change_does_not_change_source_version(self):
|
|
changed_work = {**WORK, "revision": 999}
|
|
config = build_replay_config(
|
|
work=changed_work,
|
|
reference={**REFERENCE, "imported_chapter_count": 593},
|
|
outline_rows=[{"id": 1, "from_order": 1, "to_order": 10, "outline_text": "历史窗"}],
|
|
scaffold_rows=[],
|
|
target_scaffold={"id": 11, "chapter": 489, "title": "目标", "outline_text": "目标事实"},
|
|
card_rows=[card_row(), card_row(11127, "赵宁")],
|
|
card_selection={"correctCardIds": [11126], "placeboCardIds": [11127]},
|
|
as_of=488,
|
|
target=489,
|
|
evaluation_set_version="set-test",
|
|
strategy_version="strategy-test",
|
|
source=self.source,
|
|
authorization=self.authorization,
|
|
)
|
|
self.assertEqual(config["referenceWork"]["version"], SOURCE_VERSION)
|
|
|
|
def test_database_transaction_is_repeatable_read_only(self):
|
|
source = inspect.getsource(loader.load_reference_rows)
|
|
helper = inspect.getsource(loader.begin_read_snapshot)
|
|
self.assertIn("begin_read_snapshot(conn)", source)
|
|
self.assertIn("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY", helper)
|
|
self.assertIn("ORDER BY checked_at DESC,id DESC\n LIMIT 1", source)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|