091b66a9bb
将角色与 Skill 从 .claude 迁入 .agent，移除 Claude CLI 运行时并接入固定 Opus 角色 profile、完整 schema、预算 deadline、raw 与回执证据链。 同步拆分 Skill 职责、复利 lesson、Gate 回放、Dashboard 人审入口、数据库登记和机械门禁；候选设计正文不包含在本提交中。
293 lines
13 KiB
Python
293 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""正文候选、独立 detector、补证重写和 CAS 的离线测试。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
SKILLS_DIR = PROJECT_ROOT / ".agent" / "skills"
|
|
TEST_DIR = PROJECT_ROOT / "tests" / "skills" / "write-next-chapter"
|
|
CHECK_TEST_DIR = PROJECT_ROOT / "tests" / "skills" / "check-content-consistency"
|
|
SCRIPT_DIR = SKILLS_DIR / "write-next-chapter" / "scripts"
|
|
DETECT_DIR = SKILLS_DIR / "check-content-consistency" / "scripts"
|
|
READ_CONTEXT_DIR = SKILLS_DIR / "assemble-context" / "scripts"
|
|
for path in (READ_CONTEXT_DIR, DETECT_DIR, SCRIPT_DIR, TEST_DIR, CHECK_TEST_DIR):
|
|
sys.path.insert(0, str(path))
|
|
|
|
from test_check_writer_candidate import _candidate_body, _requirements, _valid_pair # noqa: E402
|
|
from run_writer_pipeline import CasToken, InMemoryCasStateStore, PipelineError, atomic_write_json, run_writer_pipeline # noqa: E402
|
|
from run_writer_semantic_detector import canonical_sha256 # noqa: E402
|
|
from writer_contract import build_candidate_envelope, retrieval_identity # noqa: E402
|
|
|
|
|
|
def _advance_context(context: dict, attempt: int, *, add_evidence: bool = False) -> dict:
|
|
advanced = copy.deepcopy(context)
|
|
advanced["attempt"] = attempt
|
|
if add_evidence:
|
|
advanced["factEvidence"].append({
|
|
"evidenceId": "fact-gap-1",
|
|
"fact": "旧徽章属于林澈",
|
|
"sourceType": "canonical_state",
|
|
"sourceRef": {"sourceId": "state:gap", "sourceVersion": "v1"},
|
|
"contentSha256": "sha256:" + "4" * 64,
|
|
"riskLevel": "low",
|
|
})
|
|
advanced["contextSnapshot"]["contextSha256"] = retrieval_identity(advanced)
|
|
return advanced
|
|
|
|
|
|
def _writer_output(context: dict, candidate_version: int, *, body: str | None = None) -> dict:
|
|
return build_candidate_envelope(context, {"candidateBody": body or _candidate_body()}, candidate_version=candidate_version)
|
|
|
|
|
|
def _semantic_report(context: dict, candidate: dict, *, status: str = "passed", gaps: list[dict] | None = None, findings: list[dict] | None = None) -> dict:
|
|
report = {
|
|
"schemaVersion": "semantic-detection-v3",
|
|
"runId": context["runId"],
|
|
"sampleId": "pipeline-sample",
|
|
"opaqueArmId": "pipeline-candidate",
|
|
"inputSha256": "sha256:" + "1" * 64,
|
|
"candidateVersion": candidate["candidateVersion"],
|
|
"candidateSha256": candidate["candidateSha256"],
|
|
"contextSnapshotSha256": context["contextSnapshot"]["contextSha256"],
|
|
"modelReceiptSha256": "sha256:" + "2" * 64,
|
|
"status": status,
|
|
"claims": [],
|
|
"findings": findings or [],
|
|
"assertionVerdicts": [],
|
|
"hardConstraintVerdicts": [],
|
|
"newSettingCandidates": [],
|
|
"evidenceGaps": gaps or [],
|
|
}
|
|
report["reportSha256"] = canonical_sha256(report)
|
|
return report
|
|
|
|
|
|
def _semantic_pass(context: dict, candidate: dict, _mechanical: dict) -> dict:
|
|
return _semantic_report(context, candidate)
|
|
|
|
|
|
class RunWriterPipelineV3Test(unittest.TestCase):
|
|
def _assert_failure(self, *, context: dict, writer, detector, provider, code: str) -> None:
|
|
store = InMemoryCasStateStore()
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
result_path = pathlib.Path(directory) / "result.json"
|
|
with self.assertRaises(PipelineError) as caught:
|
|
run_writer_pipeline(
|
|
context=context,
|
|
requirements=_requirements(),
|
|
writer=writer,
|
|
evidence_provider=provider,
|
|
semantic_detector=detector,
|
|
state_store=store,
|
|
result_path=result_path,
|
|
)
|
|
self.assertEqual(caught.exception.code, code)
|
|
self.assertEqual(store.latest(context["runId"]).state, "REJECTED")
|
|
self.assertEqual(json.loads(result_path.read_text(encoding="utf-8"))["failureCode"], code)
|
|
|
|
def test_writer_returns_candidate_envelope_without_semantic_fields(self) -> None:
|
|
context, _ = _valid_pair()
|
|
result = run_writer_pipeline(
|
|
context=context,
|
|
requirements=_requirements(),
|
|
writer=lambda current, version: _writer_output(current, version),
|
|
evidence_provider=lambda *_args: self.fail("无 gap 不得补证"),
|
|
semantic_detector=_semantic_pass,
|
|
state_store=InMemoryCasStateStore(),
|
|
)
|
|
self.assertEqual(result["status"], "PASSED")
|
|
self.assertEqual(result["candidateArtifact"]["schemaVersion"], "candidate-envelope-v2")
|
|
for forbidden in ("claimLedger", "evidenceRequests", "newSettingDeclarations"):
|
|
self.assertNotIn(forbidden, result["candidateArtifact"])
|
|
|
|
def test_only_detector_evidence_gaps_trigger_fresh_attempt(self) -> None:
|
|
context, _ = _valid_pair()
|
|
writer_calls: list[tuple[int, int, str]] = []
|
|
detector_calls: list[int] = []
|
|
provider_calls: list[list[str]] = []
|
|
|
|
def writer(current: dict, version: int) -> dict:
|
|
writer_calls.append((current["attempt"], version, current["contextSnapshot"]["contextSha256"]))
|
|
return _writer_output(current, version)
|
|
|
|
def detector(current: dict, candidate: dict, _mechanical: dict) -> dict:
|
|
detector_calls.append(candidate["candidateVersion"])
|
|
if candidate["candidateVersion"] == 1:
|
|
return _semantic_report(current, candidate, status="needs_evidence", gaps=[{
|
|
"gapId": "gap-1", "query": "旧徽章来源", "reason": "证据不足", "priority": "high",
|
|
"candidateSha256": candidate["candidateSha256"], "candidateQuote": "旧徽章",
|
|
"startCodePoint": 11, "endCodePoint": 14,
|
|
}])
|
|
return _semantic_report(current, candidate)
|
|
|
|
def provider(current: dict, gaps: list[dict], attempt: int) -> dict:
|
|
provider_calls.append([item["gapId"] for item in gaps])
|
|
return _advance_context(current, attempt, add_evidence=True)
|
|
|
|
result = run_writer_pipeline(
|
|
context=context,
|
|
requirements=_requirements(),
|
|
writer=writer,
|
|
evidence_provider=provider,
|
|
semantic_detector=detector,
|
|
state_store=InMemoryCasStateStore(),
|
|
)
|
|
self.assertEqual(result["status"], "PASSED")
|
|
self.assertEqual(writer_calls[0][:2], (1, 1))
|
|
self.assertEqual(writer_calls[1][:2], (2, 2))
|
|
self.assertNotEqual(writer_calls[0][2], writer_calls[1][2])
|
|
self.assertEqual(detector_calls, [1, 2])
|
|
self.assertEqual(provider_calls, [["gap-1"]])
|
|
self.assertEqual(result["evidenceRequestCount"], 1)
|
|
|
|
def test_writer_cannot_trigger_evidence_provider(self) -> None:
|
|
context, _ = _valid_pair()
|
|
|
|
def writer(current: dict, version: int) -> dict:
|
|
candidate = _writer_output(current, version)
|
|
candidate["evidenceRequests"] = [{"gapId": "forged"}]
|
|
return candidate
|
|
|
|
self._assert_failure(
|
|
context=context,
|
|
writer=writer,
|
|
detector=lambda *_args: self.fail("非法候选不得进入 detector"),
|
|
provider=lambda *_args: self.fail("writer 字段不得触发补证"),
|
|
code="MECHANICAL_DETECTION_REJECTED",
|
|
)
|
|
|
|
def test_reassembled_context_must_have_new_snapshot_and_attempt(self) -> None:
|
|
context, _ = _valid_pair()
|
|
|
|
def detector(current: dict, candidate: dict, _mechanical: dict) -> dict:
|
|
return _semantic_report(current, candidate, status="needs_evidence", gaps=[{
|
|
"gapId": "gap-1", "query": "补证", "reason": "缺口", "priority": "high",
|
|
"candidateSha256": candidate["candidateSha256"], "candidateQuote": "旧徽章",
|
|
"startCodePoint": 11, "endCodePoint": 14,
|
|
}])
|
|
|
|
same_snapshot = copy.deepcopy(context)
|
|
same_snapshot["attempt"] = 2
|
|
self._assert_failure(
|
|
context=context,
|
|
writer=lambda current, version: _writer_output(current, version),
|
|
detector=detector,
|
|
provider=lambda *_args: same_snapshot,
|
|
code="REASSEMBLED_CONTEXT_INVALID",
|
|
)
|
|
|
|
def test_old_semantic_v2_report_fails_closed(self) -> None:
|
|
context, _ = _valid_pair()
|
|
self._assert_failure(
|
|
context=context,
|
|
writer=lambda current, version: _writer_output(current, version),
|
|
detector=lambda *_args: {"schemaVersion": "semantic-detector-report-v2", "status": "passed"},
|
|
provider=lambda *_args: self.fail("旧报告不得触发补证"),
|
|
code="SEMANTIC_DETECTOR_INVALID",
|
|
)
|
|
|
|
def test_semantic_failure_without_gap_rejects_current_candidate(self) -> None:
|
|
context, _ = _valid_pair()
|
|
calls: list[int] = []
|
|
|
|
def writer(current: dict, version: int) -> dict:
|
|
calls.append(version)
|
|
return _writer_output(current, version)
|
|
|
|
def detector(current: dict, candidate: dict, _mechanical: dict) -> dict:
|
|
finding = {
|
|
"findingId": "finding-1", "severity": "high", "category": "fact_conflict",
|
|
"candidateSha256": candidate["candidateSha256"], "candidateQuote": "旧徽章",
|
|
"startCodePoint": 11, "endCodePoint": 14, "evidenceIds": [], "message": "冲突",
|
|
}
|
|
return _semantic_report(current, candidate, status="failed", findings=[finding])
|
|
|
|
self._assert_failure(
|
|
context=context,
|
|
writer=writer,
|
|
detector=detector,
|
|
provider=lambda *_args: self.fail("无 gap 不得补证"),
|
|
code="SEMANTIC_DETECTION_REJECTED",
|
|
)
|
|
self.assertEqual(calls, [1])
|
|
|
|
def test_cas_rejects_late_result(self) -> None:
|
|
store = InMemoryCasStateStore()
|
|
first = store.create("run-cas", attempt=1, candidate_version=1)
|
|
checking = store.transition(first, "CHECKING")
|
|
rejected = store.transition(checking, "REJECTED")
|
|
second = store.start_next(rejected, attempt=2, candidate_version=2)
|
|
self.assertIsNone(store.transition(checking, "PASSED"))
|
|
self.assertEqual(store.latest("run-cas"), second)
|
|
|
|
def test_atomic_result_fsyncs_before_replace(self) -> None:
|
|
events: list[str] = []
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
target = pathlib.Path(directory) / "result.json"
|
|
real_fsync = os.fsync
|
|
real_replace = os.replace
|
|
|
|
def recording_fsync(fd: int) -> None:
|
|
events.append("fsync")
|
|
real_fsync(fd)
|
|
|
|
def recording_replace(source: str, destination: str) -> None:
|
|
events.append("replace")
|
|
real_replace(source, destination)
|
|
|
|
with mock.patch("run_writer_pipeline.os.fsync", side_effect=recording_fsync), mock.patch("run_writer_pipeline.os.replace", side_effect=recording_replace):
|
|
atomic_write_json(target, {"status": "PASSED"})
|
|
self.assertLess(events.index("fsync"), events.index("replace"))
|
|
|
|
def test_initial_candidate_version_offsets_same_chapter_rerun(self) -> None:
|
|
"""同章重跑从既有最大版本接续，避免与留库被拒候选撞版本。"""
|
|
|
|
context, _ = _valid_pair()
|
|
seen_versions: list[int] = []
|
|
|
|
def writer(current: dict, version: int) -> dict:
|
|
seen_versions.append(version)
|
|
return _writer_output(current, version)
|
|
|
|
result = run_writer_pipeline(
|
|
context=context,
|
|
requirements=_requirements(),
|
|
writer=writer,
|
|
evidence_provider=lambda *_args: self.fail("无 gap 不得补证"),
|
|
semantic_detector=_semantic_pass,
|
|
state_store=InMemoryCasStateStore(),
|
|
initial_candidate_version=4,
|
|
)
|
|
self.assertEqual(result["status"], "PASSED")
|
|
self.assertEqual(seen_versions, [4])
|
|
self.assertEqual(result["candidateVersion"], 4)
|
|
self.assertEqual(result["candidateArtifact"]["candidateVersion"], 4)
|
|
|
|
def test_invalid_initial_candidate_version_fails_closed(self) -> None:
|
|
context, _ = _valid_pair()
|
|
for bad in (0, -1, True, "1"):
|
|
with self.subTest(bad=bad), self.assertRaises(PipelineError) as caught:
|
|
run_writer_pipeline(
|
|
context=context,
|
|
requirements=_requirements(),
|
|
writer=lambda current, version: _writer_output(current, version),
|
|
evidence_provider=lambda *_args: self.fail("不得补证"),
|
|
semantic_detector=_semantic_pass,
|
|
state_store=InMemoryCasStateStore(),
|
|
initial_candidate_version=bad,
|
|
)
|
|
self.assertEqual(caught.exception.code, "PIPELINE_INPUT_INVALID")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|