c9f69d9d6d
范围（不含 design-story-foundation、docs/、humanization/README.md 等进行中改动）:
1. 新增 harness/ 控制平面
- skill_harness.py 静态审计：32 个运行时 Skill 的 frontmatter/manifest/文档污染，当前 0 问题
- run_selected.py 选择性执行器：manifest 与磁盘一一对账、依赖阻断、
空跑与 skip-only 失败关闭、AST 测试形状门
- manifests/skills.json：32 个 Skill 的合同责任方与协作领域登记
- manifests/test-inventory.json：81 个测试资产登记
- specs/skill-testing.md 与 README.md：测试分层、证据边界与 harness 职责
2. 实现测试从 .claude/skills/*/scripts/ 迁至 tests/skills/<skill>/
- 71 个测试文件迁移并修复项目根与临时目录运行导入
- 数据库触发器测试宽泛异常收窄为 psycopg.errors.RaiseException
- 抽取离线大测试拆出真实 PG smoke（默认阻断，不计入离线通过）
- 抽取 presence 去重边界拆出独立测试：493 + 78 = 571 项检查不变
3. 运行时文档清理
- 13 个 SKILL.md 移除自测/离线验证段落、测试命令与测试文件事实源表述，
只保留运行时合同；业务运行合同、额度、授权与离线模式均保留
4. SoT 同步
- AGENTS.md：新增 Skill 领域索引（7 个合同责任方分组，覆盖 32 个运行时 Skill）
- 领域 07：测试入口改由 harness/manifests/ 登记，SKILL.md 不承载测试命令
- humanization 覆盖矩阵：活动测试路径同步迁移
验证证据: harness 自测 15 项 + runner 自测 13 项通过；静态审计 32 Skill / 0 问题；
73 个非数据库测试通过；8 个集成条目中 6 个 PostgreSQL 项被依赖门明确阻断；
py_compile 与 git diff --check 通过。未连接 PostgreSQL、网络、真实模型或额度。
已知边界: 真正 skill_behavior_eval 仍为 0，尚未验证任何 Skill 自然语言行为；
evaluate-frozen-replay 的 raw 存储边界冲突留待单独治理。
300 lines
12 KiB
Python
300 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""GateInputBuilder 的安全输入与同 ID 混淆项测试。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
import pathlib
|
||
import sys
|
||
import unittest
|
||
|
||
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
||
SKILLS_DIR = PROJECT_ROOT / ".claude" / "skills"
|
||
SCRIPT_DIR = SKILLS_DIR / "evaluate-frozen-replay" / "scripts"
|
||
TEST_DIR = PROJECT_ROOT / "tests" / "skills" / "evaluate-frozen-replay"
|
||
QUALITY_GATE_DIR = SKILLS_DIR / "score-content-quality" / "scripts"
|
||
for _import_dir in (SCRIPT_DIR, TEST_DIR, QUALITY_GATE_DIR):
|
||
if str(_import_dir) not in sys.path:
|
||
sys.path.insert(0, str(_import_dir))
|
||
|
||
from gate_input_builder import GateInputBuildError, GateInputBuilder, canonical_sha256
|
||
from test_writer_gate import build_input, source_bundle
|
||
from writer_rubric import RUBRIC_POLICY_VERSION
|
||
|
||
|
||
class GateInputBuilderTest(unittest.TestCase):
|
||
@staticmethod
|
||
def _rehash_wrapper(wrapper):
|
||
"""只重签 wrapper，故意不替攻击者重签生产内层回执。"""
|
||
|
||
wrapper["receiptSha256"] = canonical_sha256(
|
||
{key: value for key, value in wrapper.items() if key != "receiptSha256"}
|
||
)
|
||
|
||
def test_builder_emits_v3_policy_binding_and_complete_confounders(self):
|
||
gate_input = build_input(source_bundle())
|
||
self.assertEqual(gate_input["schemaVersion"], "writer-gate-input-v3")
|
||
self.assertEqual(gate_input["rubricPolicyVersion"], RUBRIC_POLICY_VERSION)
|
||
self.assertTrue(gate_input["builderReceiptSha256"].startswith("sha256:"))
|
||
self.assertEqual(
|
||
set(gate_input["samples"][0]["confounders"]),
|
||
{
|
||
"falseNegative",
|
||
"falsePositive",
|
||
"leakage",
|
||
"reviewerInstability",
|
||
"newCharactersWithoutCards",
|
||
},
|
||
)
|
||
|
||
def test_sensitive_manifest_field_is_rejected(self):
|
||
bundle = source_bundle()
|
||
bundle["manifest"]["samples"][0]["candidateBody"] = "正文不得进入安全 manifest"
|
||
with self.assertRaisesRegex(GateInputBuildError, "敏感字段"):
|
||
GateInputBuilder().build(**bundle)
|
||
|
||
def test_same_id_and_candidate_binding_are_mandatory(self):
|
||
bundle = source_bundle()
|
||
verdict = bundle["detector_reports"]["sample-1"]["A"]["assertionVerdicts"][0]
|
||
verdict["assertionId"] = "forged-id"
|
||
report = bundle["detector_reports"]["sample-1"]["A"]
|
||
from gate_input_builder import canonical_sha256
|
||
report["reportSha256"] = canonical_sha256(
|
||
{key: value for key, value in report.items() if key != "reportSha256"}
|
||
)
|
||
gate_input = build_input(bundle)
|
||
self.assertTrue(gate_input["samples"][0]["systemFailure"])
|
||
self.assertFalse(gate_input["samples"][0]["schemaValid"])
|
||
|
||
def test_detector_unknown_is_quality_signal_and_reduces_c_coverage(self):
|
||
"""合法 needs_evidence 可进入 Gate；unknown 不计入 C 臂硬约束覆盖率。"""
|
||
|
||
bundle = source_bundle(1, gate="A")
|
||
report = bundle["detector_reports"]["sample-1"]["C"]
|
||
report["status"] = "needs_evidence"
|
||
report["hardConstraintVerdicts"][0]["verdict"] = "unknown"
|
||
report["reportSha256"] = canonical_sha256(
|
||
{key: value for key, value in report.items() if key != "reportSha256"}
|
||
)
|
||
|
||
gate_input = build_input(bundle)
|
||
|
||
sample = gate_input["samples"][0]
|
||
self.assertTrue(sample["schemaValid"], sample)
|
||
self.assertFalse(sample["systemFailure"], sample)
|
||
self.assertEqual(sample["cArm"]["hardConstraintCoverage"], 0.0)
|
||
|
||
def test_empty_wrong_role_and_unbound_inner_execution_receipts_fail_closed(self):
|
||
"""wrapper 自哈希不能替代生产 runtime 原始 ExecutionReceipt 绑定。"""
|
||
|
||
mutations = {
|
||
"empty": lambda wrapper: wrapper.update(
|
||
{"executionReceipts": [], "executionReceiptSha256": []}
|
||
),
|
||
"wrong_role": lambda wrapper: wrapper["executionReceipts"][0].update(
|
||
{"adapterRole": "semantic_detector"}
|
||
),
|
||
"unbound": lambda wrapper: wrapper["executionReceipts"][0].update(
|
||
{"inputSha256": "sha256:" + "f" * 64}
|
||
),
|
||
}
|
||
expected_reason = {
|
||
"empty": "A_writer_inner_receipts_empty",
|
||
"wrong_role": "A_writer_inner_receipt_role_mismatch",
|
||
"unbound": "A_writer_inner_receipt_hash_mismatch",
|
||
}
|
||
for name, mutate in mutations.items():
|
||
bundle = source_bundle(1, gate="A")
|
||
wrapper = bundle["execution_receipts"]["sample-1"]["A"]["writer"]
|
||
mutate(wrapper)
|
||
self._rehash_wrapper(wrapper)
|
||
|
||
gate_input = build_input(bundle)
|
||
|
||
sample = gate_input["samples"][0]
|
||
with self.subTest(name=name):
|
||
self.assertTrue(sample["systemFailure"])
|
||
self.assertIn(expected_reason[name], sample["systemFailureReasons"])
|
||
|
||
def test_writer_and_detector_retry_chains_bind_to_final_receipt(self):
|
||
"""1..3 条重试回执均保留，semantic 报告必须绑定最后一条。"""
|
||
|
||
bundle = source_bundle(1, gate="A")
|
||
for arm in ("A", "B", "C"):
|
||
for role in ("writer", "semantic_detector"):
|
||
wrapper = bundle["execution_receipts"]["sample-1"][arm][role]
|
||
retry = copy.deepcopy(wrapper["executionReceipts"][0])
|
||
retry["invocationId"] = f"{role}-{arm}-retry-2"
|
||
wrapper["executionReceipts"].append(retry)
|
||
wrapper["executionReceiptSha256"].append(canonical_sha256(retry))
|
||
self._rehash_wrapper(wrapper)
|
||
detector = bundle["detector_reports"]["sample-1"][arm]
|
||
detector["modelReceiptSha256"] = bundle["execution_receipts"]["sample-1"][arm][
|
||
"semantic_detector"
|
||
]["executionReceiptSha256"][-1]
|
||
detector["reportSha256"] = canonical_sha256(
|
||
{key: value for key, value in detector.items() if key != "reportSha256"}
|
||
)
|
||
|
||
gate_input = build_input(bundle)
|
||
|
||
self.assertFalse(gate_input["samples"][0]["systemFailure"], gate_input["samples"][0])
|
||
|
||
def test_nonfinal_known_cost_api_error_receipt_can_precede_detector_success(self):
|
||
"""前置 API 失败必须保留审计；只有末笔成功且报告绑定末笔时可进入 Gate。"""
|
||
|
||
bundle = source_bundle(1, gate="A")
|
||
wrapper = bundle["execution_receipts"]["sample-1"]["A"]["semantic_detector"]
|
||
transient = copy.deepcopy(wrapper["executionReceipts"][0])
|
||
transient.update(
|
||
{
|
||
"invocationId": "semantic-A-transient-api-error",
|
||
"actualModelId": None,
|
||
"modelMatch": False,
|
||
"totalCostUsd": "0.000000",
|
||
"terminalReason": "api_error",
|
||
"isError": True,
|
||
"apiErrorStatus": None,
|
||
"exitCode": 1,
|
||
"structuredOutputSha256": None,
|
||
}
|
||
)
|
||
wrapper["executionReceipts"].insert(0, transient)
|
||
wrapper["executionReceiptSha256"].insert(0, canonical_sha256(transient))
|
||
self._rehash_wrapper(wrapper)
|
||
|
||
gate_input = build_input(bundle)
|
||
|
||
self.assertFalse(gate_input["samples"][0]["systemFailure"], gate_input["samples"][0])
|
||
|
||
def test_final_api_error_receipt_remains_system_failure(self):
|
||
"""API 错误只能出现在最终成功前；把它放在末笔不得借重试合同放行。"""
|
||
|
||
bundle = source_bundle(1, gate="A")
|
||
wrapper = bundle["execution_receipts"]["sample-1"]["A"]["semantic_detector"]
|
||
final_error = copy.deepcopy(wrapper["executionReceipts"][0])
|
||
final_error.update(
|
||
{
|
||
"actualModelId": None,
|
||
"modelMatch": False,
|
||
"totalCostUsd": "0.000000",
|
||
"terminalReason": "api_error",
|
||
"isError": True,
|
||
"apiErrorStatus": None,
|
||
"exitCode": 1,
|
||
"structuredOutputSha256": None,
|
||
}
|
||
)
|
||
wrapper["executionReceipts"] = [final_error]
|
||
wrapper["executionReceiptSha256"] = [canonical_sha256(final_error)]
|
||
self._rehash_wrapper(wrapper)
|
||
|
||
gate_input = build_input(bundle)
|
||
|
||
sample = gate_input["samples"][0]
|
||
self.assertTrue(sample["systemFailure"])
|
||
self.assertIn(
|
||
"A_semantic_detector_inner_receipt_not_successful",
|
||
sample["systemFailureReasons"],
|
||
)
|
||
|
||
def test_four_writer_receipts_fail_closed(self):
|
||
"""有界修订最多三次，第四条回执不得借审计链绕过。"""
|
||
|
||
bundle = source_bundle(1, gate="A")
|
||
wrapper = bundle["execution_receipts"]["sample-1"]["A"]["writer"]
|
||
for invocation in range(2, 6):
|
||
retry = copy.deepcopy(wrapper["executionReceipts"][0])
|
||
retry["invocationId"] = f"writer-A-retry-{invocation}"
|
||
wrapper["executionReceipts"].append(retry)
|
||
wrapper["executionReceiptSha256"].append(canonical_sha256(retry))
|
||
self._rehash_wrapper(wrapper)
|
||
|
||
gate_input = build_input(bundle)
|
||
|
||
self.assertTrue(gate_input["samples"][0]["systemFailure"])
|
||
self.assertIn(
|
||
"A_writer_inner_receipt_count_invalid",
|
||
gate_input["samples"][0]["systemFailureReasons"],
|
||
)
|
||
|
||
def test_judge_score_tampering_without_original_hash_resign_fails_closed(self):
|
||
"""篡改稳定报告分数但保留生产原始 hash 时不得进入 Gate 分数恢复。"""
|
||
|
||
bundle = source_bundle(1, gate="A")
|
||
report = bundle["judge_reports"]["sample-1"]
|
||
original_report_hash = report["reportSha256"]
|
||
report["candidateScores"][0]["scores"]["setting_entity_fidelity"] = 999
|
||
|
||
gate_input = build_input(bundle)
|
||
|
||
sample = gate_input["samples"][0]
|
||
self.assertEqual(report["reportSha256"], original_report_hash)
|
||
self.assertTrue(sample["systemFailure"])
|
||
self.assertFalse(sample["schemaValid"])
|
||
self.assertTrue(
|
||
any("judgeReports.sample-1.reportSha256 不一致" in reason for reason in sample["systemFailureReasons"])
|
||
)
|
||
|
||
def test_judge_report_resign_cannot_replace_original_panel_receipt_binding(self):
|
||
"""攻击者重签报告自哈希后，仍不能替换生产 panel 的原始模型回执集合。"""
|
||
|
||
bundle = source_bundle(1, gate="A")
|
||
report = bundle["judge_reports"]["sample-1"]
|
||
report["modelReceiptSha256"] = "sha256:" + "e" * 64
|
||
report["reportSha256"] = canonical_sha256(
|
||
{key: value for key, value in report.items() if key != "reportSha256"}
|
||
)
|
||
|
||
gate_input = build_input(bundle)
|
||
|
||
sample = gate_input["samples"][0]
|
||
self.assertTrue(sample["systemFailure"])
|
||
self.assertFalse(sample["schemaValid"])
|
||
self.assertTrue(
|
||
any("未绑定原始 panel 回执" in reason for reason in sample["systemFailureReasons"])
|
||
)
|
||
|
||
def test_judge_report_cannot_swap_preregistered_scenario_policy(self):
|
||
bundle = source_bundle(1, gate="A")
|
||
panel = bundle["judge_reports"]["sample-1"]
|
||
panel["reviewerReports"][0]["scenario"] = "character_dialogue"
|
||
panel["reportSha256"] = canonical_sha256(
|
||
{key: value for key, value in panel.items() if key != "reportSha256"}
|
||
)
|
||
|
||
gate_input = build_input(bundle)
|
||
|
||
sample = gate_input["samples"][0]
|
||
self.assertTrue(sample["systemFailure"])
|
||
self.assertFalse(sample["schemaValid"])
|
||
self.assertTrue(
|
||
any(
|
||
"未绑定当前样本场景评分策略" in reason
|
||
for reason in sample["systemFailureReasons"]
|
||
)
|
||
)
|
||
|
||
def test_legal_score_tampering_and_report_resign_still_fails_receipt_binding(self):
|
||
"""合法 9.5 分即使重签 panel 自哈希，也必须与模型原始 structured output 冲突。"""
|
||
|
||
bundle = source_bundle(1, gate="A")
|
||
report = bundle["judge_reports"]["sample-1"]
|
||
report["candidateScores"][0]["scores"]["setting_entity_fidelity"] = 9.5
|
||
report["reportSha256"] = canonical_sha256(
|
||
{key: value for key, value in report.items() if key != "reportSha256"}
|
||
)
|
||
|
||
gate_input = build_input(bundle)
|
||
|
||
sample = gate_input["samples"][0]
|
||
self.assertTrue(sample["systemFailure"])
|
||
self.assertFalse(sample["schemaValid"])
|
||
self.assertTrue(
|
||
any("模型原始 structured output" in reason for reason in sample["systemFailureReasons"])
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|