303 lines
14 KiB
Python
303 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""正文候选、独立 detector、补证重写和 CAS 的离线测试。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
import json
|
||
import os
|
||
import pathlib
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from unittest import mock
|
||
|
||
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
||
SKILLS_DIR = PROJECT_ROOT / ".agent" / "skills"
|
||
TEST_DIR = PROJECT_ROOT / "tests" / "skills" / "write-next-chapter"
|
||
CHECK_TEST_DIR = PROJECT_ROOT / "tests" / "skills" / "check-content-consistency"
|
||
SCRIPT_DIR = PROJECT_ROOT / "muse" / "content" / "work" / "skills" / "generate" / "write-next-chapter" / "scripts"
|
||
DETECT_DIR = PROJECT_ROOT / "muse" / "lifecycle" / "quality" / "skills" / "semantic" / "check-content-consistency" / "scripts"
|
||
READ_CONTEXT_DIR = PROJECT_ROOT / "muse" / "lifecycle" / "context" / "skills" / "assemble-context" / "scripts"
|
||
for path in (READ_CONTEXT_DIR, DETECT_DIR, SCRIPT_DIR, TEST_DIR, CHECK_TEST_DIR):
|
||
sys.path.insert(0, str(path))
|
||
|
||
from test_check_writer_candidate import _candidate_body, _requirements, _valid_pair # noqa: E402
|
||
from run_writer_pipeline import CasToken, InMemoryCasStateStore, PipelineError, atomic_write_json, run_writer_pipeline # noqa: E402
|
||
from run_writer_semantic_detector import canonical_sha256 # noqa: E402
|
||
from writer_contract import build_candidate_envelope, retrieval_identity # noqa: E402
|
||
|
||
|
||
def _advance_context(context: dict, attempt: int, *, add_evidence: bool = False) -> dict:
|
||
advanced = copy.deepcopy(context)
|
||
advanced["attempt"] = attempt
|
||
if add_evidence:
|
||
advanced["factEvidence"].append({
|
||
"evidenceId": "fact-gap-1",
|
||
"fact": "旧徽章属于林澈",
|
||
"sourceType": "canonical_state",
|
||
"sourceRef": {"sourceId": "state:gap", "sourceVersion": "v1"},
|
||
"contentSha256": "sha256:" + "4" * 64,
|
||
"riskLevel": "low",
|
||
})
|
||
advanced["contextSnapshot"]["contextSha256"] = retrieval_identity(advanced)
|
||
return advanced
|
||
|
||
|
||
def _writer_output(context: dict, candidate_version: int, *, body: str | None = None) -> dict:
|
||
return build_candidate_envelope(context, {"candidateBody": body or _candidate_body()}, candidate_version=candidate_version)
|
||
|
||
|
||
def _semantic_report(context: dict, candidate: dict, *, status: str = "passed", gaps: list[dict] | None = None, findings: list[dict] | None = None) -> dict:
|
||
report = {
|
||
"schemaVersion": "semantic-detection-v3",
|
||
"runId": context["runId"],
|
||
"sampleId": "pipeline-sample",
|
||
"opaqueArmId": "pipeline-candidate",
|
||
"inputSha256": "sha256:" + "1" * 64,
|
||
"candidateVersion": candidate["candidateVersion"],
|
||
"candidateSha256": candidate["candidateSha256"],
|
||
"contextSnapshotSha256": context["contextSnapshot"]["contextSha256"],
|
||
"modelReceiptSha256": "sha256:" + "2" * 64,
|
||
"status": status,
|
||
"claims": [],
|
||
"findings": findings or [],
|
||
"assertionVerdicts": [],
|
||
"hardConstraintVerdicts": [],
|
||
"newSettingCandidates": [],
|
||
"evidenceGaps": gaps or [],
|
||
}
|
||
report["reportSha256"] = canonical_sha256(report)
|
||
return report
|
||
|
||
|
||
def _semantic_pass(context: dict, candidate: dict, _mechanical: dict) -> dict:
|
||
return _semantic_report(context, candidate)
|
||
|
||
|
||
class RunWriterPipelineV3Test(unittest.TestCase):
|
||
def _assert_failure(self, *, context: dict, writer, detector, provider, code: str) -> None:
|
||
store = InMemoryCasStateStore()
|
||
with tempfile.TemporaryDirectory() as directory:
|
||
result_path = pathlib.Path(directory) / "result.json"
|
||
with self.assertRaises(PipelineError) as caught:
|
||
run_writer_pipeline(
|
||
context=context,
|
||
requirements=_requirements(),
|
||
writer=writer,
|
||
evidence_provider=provider,
|
||
semantic_detector=detector,
|
||
state_store=store,
|
||
result_path=result_path,
|
||
)
|
||
self.assertEqual(caught.exception.code, code)
|
||
self.assertEqual(store.latest(context["runId"]).state, "REJECTED")
|
||
self.assertEqual(json.loads(result_path.read_text(encoding="utf-8"))["failureCode"], code)
|
||
|
||
def test_writer_returns_candidate_envelope_without_semantic_fields(self) -> None:
|
||
context, _ = _valid_pair()
|
||
result = run_writer_pipeline(
|
||
context=context,
|
||
requirements=_requirements(),
|
||
writer=lambda current, version: _writer_output(current, version),
|
||
evidence_provider=lambda *_args: self.fail("无 gap 不得补证"),
|
||
semantic_detector=_semantic_pass,
|
||
state_store=InMemoryCasStateStore(),
|
||
)
|
||
self.assertEqual(result["status"], "PASSED")
|
||
self.assertEqual(result["candidateArtifact"]["schemaVersion"], "candidate-envelope-v2")
|
||
for forbidden in ("claimLedger", "evidenceRequests", "newSettingDeclarations"):
|
||
self.assertNotIn(forbidden, result["candidateArtifact"])
|
||
|
||
def test_evidence_gaps_stop_at_authorization_terminal(self) -> None:
|
||
"""新合同：补证检索命中后收敛为 AUTHORIZATION_REQUIRED，重写等人授权。"""
|
||
context, _ = _valid_pair()
|
||
writer_calls: list[tuple[int, int]] = []
|
||
|
||
def writer(current: dict, version: int) -> dict:
|
||
writer_calls.append((current["attempt"], version))
|
||
return _writer_output(current, version)
|
||
|
||
def detector(current: dict, candidate: dict, _mechanical: dict) -> dict:
|
||
return _semantic_report(current, candidate, status="needs_evidence", gaps=[{
|
||
"gapId": "gap-1", "query": "旧徽章来源", "reason": "证据不足", "priority": "high",
|
||
"candidateSha256": candidate["candidateSha256"], "candidateQuote": "旧徽章",
|
||
"startCodePoint": 11, "endCodePoint": 14,
|
||
}])
|
||
|
||
store = InMemoryCasStateStore()
|
||
with tempfile.TemporaryDirectory() as directory:
|
||
result_path = pathlib.Path(directory) / "result.json"
|
||
with self.assertRaises(PipelineError) as caught:
|
||
run_writer_pipeline(
|
||
context=context,
|
||
requirements=_requirements(),
|
||
writer=writer,
|
||
evidence_provider=lambda current, _gaps, attempt: _advance_context(current, attempt, add_evidence=True),
|
||
semantic_detector=detector,
|
||
state_store=store,
|
||
result_path=result_path,
|
||
)
|
||
result = json.loads(result_path.read_text(encoding="utf-8"))
|
||
self.assertEqual(caught.exception.code, "AUTHORIZATION_REQUIRED")
|
||
# 只跑单次收敛：写手只被调用一次，无自动第二轮。
|
||
self.assertEqual(writer_calls, [(1, 1)])
|
||
self.assertEqual(store.latest(context["runId"]).state, "REJECTED")
|
||
# 授权终态携带结构化缺口报告，供主代理转述给人。
|
||
gaps = caught.exception.details["evidenceGaps"]
|
||
self.assertEqual([item["gapId"] for item in gaps], ["gap-1"])
|
||
self.assertEqual(result["failureCode"], "AUTHORIZATION_REQUIRED")
|
||
self.assertEqual(result["status"], "REJECTED")
|
||
|
||
def test_writer_cannot_trigger_evidence_provider(self) -> None:
|
||
context, _ = _valid_pair()
|
||
|
||
def writer(current: dict, version: int) -> dict:
|
||
candidate = _writer_output(current, version)
|
||
candidate["evidenceRequests"] = [{"gapId": "forged"}]
|
||
return candidate
|
||
|
||
self._assert_failure(
|
||
context=context,
|
||
writer=writer,
|
||
detector=lambda *_args: self.fail("非法候选不得进入 detector"),
|
||
provider=lambda *_args: self.fail("writer 字段不得触发补证"),
|
||
code="MECHANICAL_DETECTION_REJECTED",
|
||
)
|
||
|
||
def test_authorization_terminal_publishes_gap_report(self) -> None:
|
||
"""授权终态结果文件必须带缺口与检查快照，编排方据此向人请求授权。"""
|
||
context, _ = _valid_pair()
|
||
|
||
def detector(current: dict, candidate: dict, _mechanical: dict) -> dict:
|
||
return _semantic_report(current, candidate, status="needs_evidence", gaps=[{
|
||
"gapId": "gap-1", "query": "补证", "reason": "缺口", "priority": "high",
|
||
"candidateSha256": candidate["candidateSha256"], "candidateQuote": "旧徽章",
|
||
"startCodePoint": 11, "endCodePoint": 14,
|
||
}])
|
||
|
||
store = InMemoryCasStateStore()
|
||
with tempfile.TemporaryDirectory() as directory:
|
||
result_path = pathlib.Path(directory) / "result.json"
|
||
with self.assertRaises(PipelineError) as caught:
|
||
run_writer_pipeline(
|
||
context=context,
|
||
requirements=_requirements(),
|
||
writer=lambda current, version: _writer_output(current, version),
|
||
evidence_provider=lambda current, _gaps, attempt: _advance_context(current, attempt, add_evidence=True),
|
||
semantic_detector=detector,
|
||
state_store=store,
|
||
result_path=result_path,
|
||
)
|
||
result = json.loads(result_path.read_text(encoding="utf-8"))
|
||
self.assertEqual(caught.exception.code, "AUTHORIZATION_REQUIRED")
|
||
self.assertEqual(result["candidateSha256"], caught.exception.result["candidateSha256"])
|
||
self.assertEqual(caught.exception.details["mechanicalPassed"], True)
|
||
self.assertEqual(caught.exception.details["semanticStatus"], "needs_evidence")
|
||
|
||
def test_old_semantic_v2_report_fails_closed(self) -> None:
|
||
context, _ = _valid_pair()
|
||
self._assert_failure(
|
||
context=context,
|
||
writer=lambda current, version: _writer_output(current, version),
|
||
detector=lambda *_args: {"schemaVersion": "semantic-detector-report-v2", "status": "passed"},
|
||
provider=lambda *_args: self.fail("旧报告不得触发补证"),
|
||
code="SEMANTIC_DETECTOR_INVALID",
|
||
)
|
||
|
||
def test_semantic_failure_without_gap_rejects_current_candidate(self) -> None:
|
||
context, _ = _valid_pair()
|
||
calls: list[int] = []
|
||
|
||
def writer(current: dict, version: int) -> dict:
|
||
calls.append(version)
|
||
return _writer_output(current, version)
|
||
|
||
def detector(current: dict, candidate: dict, _mechanical: dict) -> dict:
|
||
finding = {
|
||
"findingId": "finding-1", "severity": "high", "category": "fact_conflict",
|
||
"candidateSha256": candidate["candidateSha256"], "candidateQuote": "旧徽章",
|
||
"startCodePoint": 11, "endCodePoint": 14, "evidenceIds": [], "message": "冲突",
|
||
}
|
||
return _semantic_report(current, candidate, status="failed", findings=[finding])
|
||
|
||
self._assert_failure(
|
||
context=context,
|
||
writer=writer,
|
||
detector=detector,
|
||
provider=lambda *_args: self.fail("无 gap 不得补证"),
|
||
code="SEMANTIC_DETECTION_REJECTED",
|
||
)
|
||
self.assertEqual(calls, [1])
|
||
|
||
def test_cas_rejects_late_result(self) -> None:
|
||
store = InMemoryCasStateStore()
|
||
first = store.create("run-cas", attempt=1, candidate_version=1)
|
||
checking = store.transition(first, "CHECKING")
|
||
rejected = store.transition(checking, "REJECTED")
|
||
second = store.start_next(rejected, attempt=2, candidate_version=2)
|
||
self.assertIsNone(store.transition(checking, "PASSED"))
|
||
self.assertEqual(store.latest("run-cas"), second)
|
||
|
||
def test_atomic_result_fsyncs_before_replace(self) -> None:
|
||
events: list[str] = []
|
||
with tempfile.TemporaryDirectory() as directory:
|
||
target = pathlib.Path(directory) / "result.json"
|
||
real_fsync = os.fsync
|
||
real_replace = os.replace
|
||
|
||
def recording_fsync(fd: int) -> None:
|
||
events.append("fsync")
|
||
real_fsync(fd)
|
||
|
||
def recording_replace(source: str, destination: str) -> None:
|
||
events.append("replace")
|
||
real_replace(source, destination)
|
||
|
||
with mock.patch("run_writer_pipeline.os.fsync", side_effect=recording_fsync), mock.patch("run_writer_pipeline.os.replace", side_effect=recording_replace):
|
||
atomic_write_json(target, {"status": "PASSED"})
|
||
self.assertLess(events.index("fsync"), events.index("replace"))
|
||
|
||
def test_initial_candidate_version_offsets_same_chapter_rerun(self) -> None:
|
||
"""同章重跑从既有最大版本接续，避免与留库被拒候选撞版本。"""
|
||
|
||
context, _ = _valid_pair()
|
||
seen_versions: list[int] = []
|
||
|
||
def writer(current: dict, version: int) -> dict:
|
||
seen_versions.append(version)
|
||
return _writer_output(current, version)
|
||
|
||
result = run_writer_pipeline(
|
||
context=context,
|
||
requirements=_requirements(),
|
||
writer=writer,
|
||
evidence_provider=lambda *_args: self.fail("无 gap 不得补证"),
|
||
semantic_detector=_semantic_pass,
|
||
state_store=InMemoryCasStateStore(),
|
||
initial_candidate_version=4,
|
||
)
|
||
self.assertEqual(result["status"], "PASSED")
|
||
self.assertEqual(seen_versions, [4])
|
||
self.assertEqual(result["candidateVersion"], 4)
|
||
self.assertEqual(result["candidateArtifact"]["candidateVersion"], 4)
|
||
|
||
def test_invalid_initial_candidate_version_fails_closed(self) -> None:
|
||
context, _ = _valid_pair()
|
||
for bad in (0, -1, True, "1"):
|
||
with self.subTest(bad=bad), self.assertRaises(PipelineError) as caught:
|
||
run_writer_pipeline(
|
||
context=context,
|
||
requirements=_requirements(),
|
||
writer=lambda current, version: _writer_output(current, version),
|
||
evidence_provider=lambda *_args: self.fail("不得补证"),
|
||
semantic_detector=_semantic_pass,
|
||
state_store=InMemoryCasStateStore(),
|
||
initial_candidate_version=bad,
|
||
)
|
||
self.assertEqual(caught.exception.code, "PIPELINE_INPUT_INVALID")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|