600 lines
28 KiB
Python
600 lines
28 KiB
Python
#!/usr/bin/env python3
|
||
"""blind judge v4 输入/报告、v3 模型输出和 fresh panel 测试。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
import hashlib
|
||
import json
|
||
import pathlib
|
||
import sys
|
||
import unittest
|
||
from collections.abc import Mapping
|
||
from typing import Any
|
||
|
||
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
||
SCRIPT_DIR = PROJECT_ROOT / "muse" / "lifecycle" / "quality" / "skills" / "judge" / "评估内容质量" / "scripts"
|
||
if str(SCRIPT_DIR) not in sys.path:
|
||
sys.path.insert(0, str(SCRIPT_DIR))
|
||
|
||
from run_writer_blind_judge import ( # noqa: E402
|
||
BLIND_JUDGE_REPORT_JSON_SCHEMA,
|
||
BlindJudgeContractError,
|
||
_quote_location,
|
||
canonical_sha256,
|
||
run_writer_blind_judge,
|
||
run_writer_blind_judge_panel,
|
||
validate_blind_judge_input,
|
||
validate_blind_judge_report,
|
||
)
|
||
from writer_rubric import ( # noqa: E402
|
||
COMMON_DIMENSIONS,
|
||
DIMENSIONS,
|
||
RUBRIC_POLICY_VERSION,
|
||
SCENARIO_DIMENSION,
|
||
)
|
||
|
||
|
||
def _candidate_hash(body: str) -> str:
|
||
return "sha256:" + hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def _truth_pack() -> dict[str, Any]:
|
||
pack: dict[str, Any] = {
|
||
"schemaVersion": "oracle-truth-pack-v1",
|
||
"evaluationSetVersion": "writer-gate-a-v1",
|
||
"sampleId": "sample-489",
|
||
"workId": 8,
|
||
"asOf": 488,
|
||
"sourceSnapshotSha256": "sha256:" + "1" * 64,
|
||
"authorizationSnapshotId": "authorization-offline-1",
|
||
"historicalAssertions": [{
|
||
"assertionId": "assertion-history-1", "text": "林澈在城门", "sourceVersion": "v488",
|
||
"chapterStart": 488, "chapterEnd": 488, "contentSha256": "sha256:" + "2" * 64,
|
||
}],
|
||
"targetAssertions": [],
|
||
}
|
||
pack["packSha256"] = canonical_sha256(pack)
|
||
return pack
|
||
|
||
|
||
def blind_input(reviewer_id: str, *, order: tuple[str, ...] = ("blind-1", "blind-2")) -> dict[str, Any]:
|
||
bodies = {"blind-1": "林澈守住城门。", "blind-2": "林澈退回城内。"}
|
||
pack = _truth_pack()
|
||
payload: dict[str, Any] = {
|
||
"schemaVersion": "writer-blind-input-v4",
|
||
"runId": "run-judge-1",
|
||
"sampleId": "sample-489",
|
||
"scenario": "turning_point",
|
||
"rubricPolicyVersion": RUBRIC_POLICY_VERSION,
|
||
"reviewerInvocationId": reviewer_id,
|
||
"candidateOrder": list(order),
|
||
"candidates": [{"blindCandidateId": item, "candidateSha256": _candidate_hash(bodies[item]), "candidateBody": bodies[item]} for item in order],
|
||
"fineOutline": {
|
||
"sourceRef": {"sourceId": "fine-outline:489", "sourceVersion": "v1", "chapter": 489},
|
||
"hardConstraints": [{"constraintId": "constraint-1", "text": "必须守住城门"}],
|
||
"adjustableBeats": [],
|
||
"declaredNewFacts": [],
|
||
},
|
||
"oracleTruthPack": pack,
|
||
"oracleTruthPackSha256": pack["packSha256"],
|
||
"authorizationSnapshotId": "authorization-offline-1",
|
||
}
|
||
payload["blindInputSha256"] = canonical_sha256(payload)
|
||
return payload
|
||
|
||
|
||
def blind_draft(payload: Mapping[str, Any], *, score: float = 8.0) -> dict[str, Any]:
|
||
ids = list(payload["candidateOrder"])
|
||
quotes = {item["blindCandidateId"]: item["candidateBody"][:2] for item in payload["candidates"]}
|
||
return {
|
||
"schemaVersion": "blind-judge-draft-v3",
|
||
"candidateScores": [{
|
||
"blindCandidateId": blind_id,
|
||
"scores": {dimension: {
|
||
"score": score,
|
||
"reason": "候选正文支持该维判断",
|
||
"candidateQuote": quotes[blind_id],
|
||
"evidenceRefs": [{"sourceType": "candidate", "sourceId": blind_id}],
|
||
} for dimension in DIMENSIONS},
|
||
} for blind_id in ids],
|
||
"dimensionPreferences": [{"dimension": dimension, "orderedCandidateIds": ids, "reason": "第一候选更符合共同目标"} for dimension in DIMENSIONS],
|
||
"oracleAssertionVerdicts": [{
|
||
"assertionId": "assertion-history-1", "blindCandidateId": blind_id,
|
||
"verdict": "pass", "reason": "未违背冻结事实", "candidateQuote": quotes[blind_id],
|
||
"evidenceRefs": [{"sourceType": "oracle_assertion", "sourceId": "assertion-history-1"}],
|
||
} for blind_id in ids],
|
||
"hardConstraintVerdicts": [{
|
||
"constraintId": "constraint-1", "blindCandidateId": blind_id,
|
||
"verdict": "pass", "reason": "满足守城目标", "candidateQuote": quotes[blind_id],
|
||
"evidenceRefs": [{"sourceType": "fine_outline", "sourceId": "constraint-1"}],
|
||
} for blind_id in ids],
|
||
}
|
||
|
||
|
||
class SequenceRunner:
|
||
def __init__(self, outputs: list[Any]) -> None:
|
||
self.outputs = [
|
||
item if isinstance(item, BaseException) else copy.deepcopy(dict(item))
|
||
for item in outputs
|
||
]
|
||
self.calls: list[dict[str, Any]] = []
|
||
|
||
def run(self, *, adapter_role: str, model_input: Mapping[str, Any], output_schema: Mapping[str, Any]) -> Mapping[str, Any]:
|
||
self.calls.append({"role": adapter_role, "input": copy.deepcopy(dict(model_input)), "schema": output_schema})
|
||
output = self.outputs[len(self.calls) - 1]
|
||
if isinstance(output, BaseException):
|
||
raise output
|
||
return {"structuredOutput": output, "modelReceiptSha256": "sha256:" + str(len(self.calls)) * 64}
|
||
|
||
|
||
class BlindJudgeV3Test(unittest.TestCase):
|
||
def test_model_schema_excludes_hash_offset_identity_and_receipt(self) -> None:
|
||
schema = BLIND_JUDGE_REPORT_JSON_SCHEMA
|
||
self.assertFalse(schema["additionalProperties"])
|
||
forbidden = {"runId", "sampleId", "reviewerInvocationId", "candidateSha256", "blindInputSha256", "modelReceiptSha256", "reportSha256", "startCodePoint", "endCodePoint"}
|
||
self.assertTrue(forbidden.isdisjoint(schema["properties"]))
|
||
|
||
forged = blind_draft(blind_input("reviewer-1"))
|
||
forged["candidateSha256"] = "sha256:" + "f" * 64
|
||
result = run_writer_blind_judge(
|
||
blind_input("reviewer-1"),
|
||
model_runner=SequenceRunner([forged]),
|
||
max_corrections=0,
|
||
)
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(result["primaryCode"], "BLIND_JUDGE_MODEL_OUTPUT_INVALID")
|
||
|
||
def test_adapter_sends_anonymous_common_input_and_binds_report(self) -> None:
|
||
payload = blind_input("reviewer-1")
|
||
runner = SequenceRunner([blind_draft(payload)])
|
||
result = run_writer_blind_judge(payload, model_runner=runner)
|
||
self.assertTrue(result["ok"])
|
||
report = result["report"]
|
||
self.assertEqual(report["schemaVersion"], "blind-judge-report-v4")
|
||
self.assertEqual(report["scenario"], "turning_point")
|
||
self.assertEqual(report["rubricPolicyVersion"], RUBRIC_POLICY_VERSION)
|
||
self.assertEqual(report["reviewerInvocationId"], "reviewer-1")
|
||
self.assertEqual(report["candidateScores"][0]["candidateSha256"], payload["candidates"][0]["candidateSha256"])
|
||
self.assertEqual(report["reportSha256"], canonical_sha256({k: v for k, v in report.items() if k != "reportSha256"}))
|
||
|
||
model_input = runner.calls[0]["input"]
|
||
self.assertEqual(
|
||
set(model_input),
|
||
{
|
||
"candidates",
|
||
"fineOutline",
|
||
"oracleTruthPack",
|
||
"rubric",
|
||
"outputContract",
|
||
"candidateQuoteHints",
|
||
},
|
||
)
|
||
self.assertEqual(
|
||
model_input["outputContract"]["expectedCounts"],
|
||
{
|
||
"candidateScores": 2,
|
||
"dimensionPreferences": len(DIMENSIONS),
|
||
"oracleAssertionVerdicts": 2,
|
||
"hardConstraintVerdicts": 2,
|
||
},
|
||
)
|
||
self.assertEqual(
|
||
model_input["candidateQuoteHints"]["blind-1"], ["林澈守住城门。"]
|
||
)
|
||
self.assertEqual(model_input["rubric"]["policyVersion"], RUBRIC_POLICY_VERSION)
|
||
self.assertEqual(model_input["rubric"]["scenarioType"], "turning_point")
|
||
self.assertEqual(
|
||
[item["dimensionId"] for item in model_input["rubric"]["commonDimensions"]],
|
||
list(COMMON_DIMENSIONS),
|
||
)
|
||
self.assertEqual(
|
||
model_input["rubric"]["scenarioDimension"]["dimensionId"],
|
||
SCENARIO_DIMENSION,
|
||
)
|
||
self.assertEqual(
|
||
model_input["rubric"]["scenarioDimension"]["displayName"],
|
||
"转折执行",
|
||
)
|
||
serialized = str(model_input)
|
||
for forbidden in ("runId", "sampleId", "reviewerInvocationId", "candidateSha256", "blindInputSha256", "authorizationSnapshotId"):
|
||
self.assertNotIn(forbidden, serialized)
|
||
|
||
def test_unregistered_scenario_and_policy_drift_fail_before_model_call(self) -> None:
|
||
for field, value in (
|
||
("scenario", "romance"),
|
||
("rubricPolicyVersion", "writer-replay-rubric-v1"),
|
||
):
|
||
payload = blind_input("reviewer-1")
|
||
payload[field] = value
|
||
payload["blindInputSha256"] = canonical_sha256(
|
||
{key: item for key, item in payload.items() if key != "blindInputSha256"}
|
||
)
|
||
runner = SequenceRunner([])
|
||
|
||
result = run_writer_blind_judge(payload, model_runner=runner)
|
||
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(result["primaryCode"], "BLIND_JUDGE_INPUT_SCHEMA_INVALID")
|
||
self.assertEqual(runner.calls, [])
|
||
|
||
def test_adapter_computes_quote_offsets(self) -> None:
|
||
payload = blind_input("reviewer-1")
|
||
result = run_writer_blind_judge(payload, model_runner=SequenceRunner([blind_draft(payload)]))
|
||
evidence = result["report"]["candidateScores"][0]["scores"][DIMENSIONS[0]]["evidence"][0]
|
||
self.assertEqual(evidence["sourceRef"], "candidate:0-2")
|
||
self.assertEqual(evidence["excerpt"], "林澈")
|
||
|
||
def test_model_id_order_is_normalized_without_relaxing_coverage(self) -> None:
|
||
"""完整 ID 集乱序时确定性重排，缺失/重复仍失败关闭。"""
|
||
|
||
payload = blind_input("reviewer-1")
|
||
draft = blind_draft(payload)
|
||
draft["candidateScores"].reverse()
|
||
draft["dimensionPreferences"].reverse()
|
||
draft["oracleAssertionVerdicts"].reverse()
|
||
draft["hardConstraintVerdicts"].reverse()
|
||
|
||
result = run_writer_blind_judge(payload, model_runner=SequenceRunner([draft]))
|
||
|
||
self.assertTrue(result["ok"], result)
|
||
self.assertEqual(
|
||
[item["blindCandidateId"] for item in result["report"]["candidateScores"]],
|
||
list(payload["candidateOrder"]),
|
||
)
|
||
self.assertEqual(
|
||
[item["dimension"] for item in result["report"]["dimensionPreferences"]],
|
||
list(DIMENSIONS),
|
||
)
|
||
|
||
def test_invalid_quote_gets_one_bounded_correction(self) -> None:
|
||
"""首轮编造引文时回喂原稿与错误，第二轮合法即可完成。"""
|
||
|
||
payload = blind_input("reviewer-1")
|
||
invalid = blind_draft(payload)
|
||
invalid["candidateScores"][0]["scores"][DIMENSIONS[0]][
|
||
"candidateQuote"
|
||
] = "候选正文中不存在的引文"
|
||
runner = SequenceRunner([invalid, blind_draft(payload)])
|
||
|
||
result = run_writer_blind_judge(
|
||
payload,
|
||
model_runner=runner,
|
||
max_corrections=1,
|
||
)
|
||
|
||
self.assertTrue(result["ok"], result)
|
||
self.assertEqual(result["attemptCount"], 2)
|
||
self.assertEqual(len(runner.calls), 2)
|
||
self.assertNotIn("correction", runner.calls[0]["input"])
|
||
correction = runner.calls[1]["input"]["correction"]
|
||
self.assertEqual(correction["previousDraft"], invalid)
|
||
self.assertIn("引文未在对应候选中出现", correction["error"])
|
||
self.assertIn("林澈守住城门。", correction["validCandidateQuoteHints"]["blind-1"])
|
||
|
||
def test_invalid_quote_correction_exhaustion_fails_closed(self) -> None:
|
||
"""一次纠错仍编造引文时保留稳定错误码，不继续第三次调用。"""
|
||
|
||
payload = blind_input("reviewer-1")
|
||
invalid = blind_draft(payload)
|
||
invalid["candidateScores"][0]["scores"][DIMENSIONS[0]][
|
||
"candidateQuote"
|
||
] = "候选正文中不存在的引文"
|
||
runner = SequenceRunner([invalid, invalid])
|
||
|
||
result = run_writer_blind_judge(
|
||
payload,
|
||
model_runner=runner,
|
||
max_corrections=1,
|
||
)
|
||
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(result["primaryCode"], "BLIND_JUDGE_QUOTE_NOT_FOUND")
|
||
self.assertEqual(result["attemptCount"], 2)
|
||
self.assertEqual(len(runner.calls), 2)
|
||
diagnostic = result["safeDiagnostic"]
|
||
self.assertEqual(diagnostic["errorCode"], "BLIND_JUDGE_QUOTE_NOT_FOUND")
|
||
self.assertEqual(diagnostic["arrayCounts"]["candidateScores"], 2)
|
||
self.assertNotIn("候选正文中不存在的引文", json.dumps(diagnostic, ensure_ascii=False))
|
||
|
||
def test_judge_inference_refs_bind_to_current_dimension_or_stable_id(self) -> None:
|
||
"""推断证据必须引用当前维度/断言/约束 ID，不能引用字段名占位符。"""
|
||
|
||
payload = blind_input("reviewer-1")
|
||
draft = blind_draft(payload)
|
||
for candidate in draft["candidateScores"]:
|
||
for dimension in DIMENSIONS:
|
||
candidate["scores"][dimension]["evidenceRefs"] = [
|
||
{"sourceType": "judge_inference", "sourceId": dimension}
|
||
]
|
||
for verdict in draft["oracleAssertionVerdicts"]:
|
||
verdict["evidenceRefs"] = [
|
||
{"sourceType": "judge_inference", "sourceId": verdict["assertionId"]}
|
||
]
|
||
for verdict in draft["hardConstraintVerdicts"]:
|
||
verdict["evidenceRefs"] = [
|
||
{"sourceType": "judge_inference", "sourceId": verdict["constraintId"]}
|
||
]
|
||
|
||
result = run_writer_blind_judge(
|
||
payload, model_runner=SequenceRunner([draft]), max_corrections=0
|
||
)
|
||
|
||
self.assertTrue(result["ok"], result)
|
||
|
||
invalid = copy.deepcopy(draft)
|
||
invalid["oracleAssertionVerdicts"][0]["evidenceRefs"][0]["sourceId"] = (
|
||
"assertionId"
|
||
)
|
||
failed = run_writer_blind_judge(
|
||
payload, model_runner=SequenceRunner([invalid]), max_corrections=0
|
||
)
|
||
self.assertFalse(failed["ok"])
|
||
self.assertEqual(failed["primaryCode"], "BLIND_JUDGE_MODEL_OUTPUT_INVALID")
|
||
|
||
def test_transient_api_error_retries_same_blind_input_once(self) -> None:
|
||
"""API 瞬时错误与格式纠错共用两次总调用上限，不伪造 correction。"""
|
||
|
||
payload = blind_input("reviewer-1")
|
||
runner = SequenceRunner(
|
||
[
|
||
BlindJudgeContractError("BLIND_JUDGE_API_ERROR", "瞬时 API 错误"),
|
||
blind_draft(payload),
|
||
]
|
||
)
|
||
|
||
result = run_writer_blind_judge(payload, model_runner=runner)
|
||
|
||
self.assertTrue(result["ok"], result)
|
||
self.assertEqual(result["attemptCount"], 2)
|
||
self.assertEqual(len(runner.calls), 2)
|
||
self.assertNotIn("correction", runner.calls[0]["input"])
|
||
self.assertNotIn("correction", runner.calls[1]["input"])
|
||
|
||
def test_old_v3_v2_and_v1_reports_fail_closed(self) -> None:
|
||
payload = blind_input("reviewer-1")
|
||
for version in (
|
||
"blind-judge-report-v3",
|
||
"blind-judge-report-v2",
|
||
"blind-judge-report-v1",
|
||
):
|
||
with self.subTest(version=version), self.assertRaises(BlindJudgeContractError):
|
||
validate_blind_judge_report({"schemaVersion": version}, payload)
|
||
|
||
def test_input_rejects_real_arm_and_stale_candidate_hash(self) -> None:
|
||
leaked = blind_input("reviewer-1")
|
||
leaked["candidateOrder"][0] = "A"
|
||
leaked["candidates"][0]["blindCandidateId"] = "A"
|
||
leaked["blindInputSha256"] = canonical_sha256({k: v for k, v in leaked.items() if k != "blindInputSha256"})
|
||
with self.assertRaises(BlindJudgeContractError):
|
||
validate_blind_judge_input(leaked)
|
||
|
||
stale = blind_input("reviewer-1")
|
||
stale["candidates"][0]["candidateBody"] += "旧"
|
||
stale["blindInputSha256"] = canonical_sha256({k: v for k, v in stale.items() if k != "blindInputSha256"})
|
||
with self.assertRaises(BlindJudgeContractError) as caught:
|
||
validate_blind_judge_input(stale)
|
||
self.assertEqual(caught.exception.code, "BLIND_JUDGE_CANDIDATE_HASH_MISMATCH")
|
||
|
||
def test_fine_outline_and_oracle_freeze_validation_remain_fail_closed(self) -> None:
|
||
raw_source = blind_input("reviewer-1")
|
||
raw_source["fineOutline"]["sourceRef"]["sourceId"] = "/private/tmp/raw/outline.json"
|
||
raw_source["blindInputSha256"] = canonical_sha256({k: v for k, v in raw_source.items() if k != "blindInputSha256"})
|
||
with self.assertRaises(BlindJudgeContractError) as caught:
|
||
validate_blind_judge_input(raw_source)
|
||
self.assertEqual(caught.exception.code, "BLIND_JUDGE_LEAKAGE_DETECTED")
|
||
|
||
future = blind_input("reviewer-1")
|
||
future["oracleTruthPack"]["historicalAssertions"][0]["chapterEnd"] = 489
|
||
future["oracleTruthPack"]["packSha256"] = canonical_sha256({k: v for k, v in future["oracleTruthPack"].items() if k != "packSha256"})
|
||
future["oracleTruthPackSha256"] = future["oracleTruthPack"]["packSha256"]
|
||
future["blindInputSha256"] = canonical_sha256({k: v for k, v in future.items() if k != "blindInputSha256"})
|
||
with self.assertRaises(BlindJudgeContractError):
|
||
validate_blind_judge_input(future)
|
||
|
||
nested_extra = blind_input("reviewer-1")
|
||
nested_extra["fineOutline"]["hardConstraints"][0]["debug"] = True
|
||
nested_extra["blindInputSha256"] = canonical_sha256({k: v for k, v in nested_extra.items() if k != "blindInputSha256"})
|
||
with self.assertRaises(BlindJudgeContractError):
|
||
validate_blind_judge_input(nested_extra)
|
||
|
||
def test_model_must_cover_full_verdict_cartesian_product(self) -> None:
|
||
payload = blind_input("reviewer-1")
|
||
draft = blind_draft(payload)
|
||
draft["oracleAssertionVerdicts"].pop()
|
||
result = run_writer_blind_judge(
|
||
payload,
|
||
model_runner=SequenceRunner([draft]),
|
||
max_corrections=0,
|
||
)
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(result["primaryCode"], "BLIND_JUDGE_MODEL_OUTPUT_INVALID")
|
||
|
||
def test_two_reviewers_are_fresh_reversed_and_share_no_state(self) -> None:
|
||
first = blind_input("reviewer-1")
|
||
second = blind_input("reviewer-2", order=("blind-2", "blind-1"))
|
||
runner = SequenceRunner([blind_draft(first), blind_draft(second)])
|
||
result = run_writer_blind_judge_panel(first, second, model_runner=runner, persist_lesson=False)
|
||
self.assertTrue(result["ok"])
|
||
self.assertEqual(result["reviewCount"], 2)
|
||
self.assertEqual(len(runner.calls), 2)
|
||
self.assertEqual(runner.calls[0]["input"]["candidates"][0]["blindCandidateId"], "blind-1")
|
||
self.assertEqual(runner.calls[1]["input"]["candidates"][0]["blindCandidateId"], "blind-2")
|
||
self.assertNotIn("previousReport", runner.calls[1]["input"])
|
||
|
||
def test_panel_allows_two_corrections_per_reviewer(self) -> None:
|
||
"""面板必须把 Gate A 的三次总调用合同传给每位评委。"""
|
||
|
||
first = blind_input("reviewer-1")
|
||
second = blind_input("reviewer-2", order=("blind-2", "blind-1"))
|
||
invalid = blind_draft(first)
|
||
invalid["candidateScores"][0]["scores"][DIMENSIONS[0]][
|
||
"candidateQuote"
|
||
] = "候选正文中不存在的引文"
|
||
runner = SequenceRunner([
|
||
invalid,
|
||
invalid,
|
||
blind_draft(first),
|
||
blind_draft(second),
|
||
])
|
||
|
||
result = run_writer_blind_judge_panel(first, second, model_runner=runner, persist_lesson=False)
|
||
|
||
self.assertTrue(result["ok"], result)
|
||
self.assertEqual(result["reviewCount"], 2)
|
||
self.assertEqual(len(runner.calls), 4)
|
||
self.assertIn("correction", runner.calls[1]["input"])
|
||
self.assertIn("correction", runner.calls[2]["input"])
|
||
self.assertNotIn("correction", runner.calls[3]["input"])
|
||
|
||
def test_reviewer_identity_reuse_fails_before_call(self) -> None:
|
||
first = blind_input("reviewer-1")
|
||
reused = blind_input("reviewer-1", order=("blind-2", "blind-1"))
|
||
runner = SequenceRunner([])
|
||
result = run_writer_blind_judge_panel(first, reused, model_runner=runner, persist_lesson=False)
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(result["primaryCode"], "JUDGE_INVALID")
|
||
self.assertEqual(runner.calls, [])
|
||
|
||
def test_reviewers_with_different_scenarios_fail_before_call(self) -> None:
|
||
first = blind_input("reviewer-1")
|
||
second = blind_input("reviewer-2", order=("blind-2", "blind-1"))
|
||
second["scenario"] = "battle"
|
||
second["blindInputSha256"] = canonical_sha256(
|
||
{key: value for key, value in second.items() if key != "blindInputSha256"}
|
||
)
|
||
runner = SequenceRunner([])
|
||
|
||
result = run_writer_blind_judge_panel(first, second, model_runner=runner, persist_lesson=False)
|
||
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(result["primaryCode"], "JUDGE_INVALID")
|
||
self.assertEqual(runner.calls, [])
|
||
|
||
def test_pre_call_input_failure_preserves_specific_contract_code(self) -> None:
|
||
first = blind_input("reviewer-1")
|
||
second = blind_input("reviewer-2", order=("blind-2", "blind-1"))
|
||
first["candidates"][0]["candidateBody"] += "篡改"
|
||
first["blindInputSha256"] = canonical_sha256(
|
||
{key: value for key, value in first.items() if key != "blindInputSha256"}
|
||
)
|
||
runner = SequenceRunner([])
|
||
|
||
result = run_writer_blind_judge_panel(first, second, model_runner=runner, persist_lesson=False)
|
||
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(result["status"], "failed_judge_invalid")
|
||
self.assertEqual(result["primaryCode"], "BLIND_JUDGE_CANDIDATE_HASH_MISMATCH")
|
||
self.assertEqual(result["reviewCount"], 0)
|
||
self.assertEqual(runner.calls, [])
|
||
|
||
def test_score_instability_triggers_exactly_one_third_reviewer(self) -> None:
|
||
first = blind_input("reviewer-1")
|
||
second = blind_input("reviewer-2", order=("blind-2", "blind-1"))
|
||
third = blind_input("reviewer-3")
|
||
runner = SequenceRunner([blind_draft(first, score=8.0), blind_draft(second, score=6.5), blind_draft(third, score=8.0)])
|
||
result = run_writer_blind_judge_panel(first, second, third_input=third, model_runner=runner, persist_lesson=False)
|
||
self.assertTrue(result["ok"])
|
||
self.assertEqual(result["reviewCount"], 3)
|
||
self.assertEqual(len(runner.calls), 3)
|
||
|
||
def test_runner_must_supply_receipt_hash(self) -> None:
|
||
class DirectRunner:
|
||
def run(self, **_kwargs: Any) -> Mapping[str, Any]:
|
||
return blind_draft(blind_input("reviewer-1"))
|
||
|
||
result = run_writer_blind_judge(blind_input("reviewer-1"), model_runner=DirectRunner())
|
||
self.assertFalse(result["ok"])
|
||
self.assertEqual(result["primaryCode"], "BLIND_JUDGE_RECEIPT_BINDING_MISMATCH")
|
||
|
||
def test_ellipsis_in_free_text_not_killed_but_real_traversal_blocked(self) -> None:
|
||
# 候选正文、评委理由、引文均含省略号 `...`：旧的 `".." in value` 子串判定会把它们误判为
|
||
# 路径穿越而冤杀；改用精确路径分量判定后应正常通过校验，不报 BLIND_JUDGE_LEAKAGE_DETECTED。
|
||
bodies = {"blind-1": "林澈说：等等...别走。", "blind-2": "林澈说：慢着...且听我说完。"}
|
||
payload = blind_input("reviewer-1")
|
||
for item in payload["candidates"]:
|
||
item["candidateBody"] = bodies[item["blindCandidateId"]]
|
||
item["candidateSha256"] = _candidate_hash(bodies[item["blindCandidateId"]])
|
||
payload["blindInputSha256"] = canonical_sha256({k: v for k, v in payload.items() if k != "blindInputSha256"})
|
||
|
||
# 输入校验放行：含省略号的正文不被当路径穿越。
|
||
normalized = validate_blind_judge_input(payload)
|
||
self.assertEqual(normalized["_candidateBodies"]["blind-1"], bodies["blind-1"])
|
||
|
||
# 评委理由与引文也含省略号：完整跑通盲评，不报泄露、正常出报告。
|
||
ellipsis_quotes = {"blind-1": "等等...", "blind-2": "慢着..."}
|
||
draft = blind_draft(payload)
|
||
for cand in draft["candidateScores"]:
|
||
for dimension in DIMENSIONS:
|
||
cand["scores"][dimension]["reason"] = "节奏张力足够...情绪递进自然"
|
||
cand["scores"][dimension]["candidateQuote"] = ellipsis_quotes[cand["blindCandidateId"]]
|
||
result = run_writer_blind_judge(payload, model_runner=SequenceRunner([draft]))
|
||
self.assertTrue(result["ok"])
|
||
self.assertNotEqual(result.get("primaryCode"), "BLIND_JUDGE_LEAKAGE_DETECTED")
|
||
first_dim = DIMENSIONS[0]
|
||
self.assertEqual(result["report"]["candidateScores"][0]["scores"][first_dim]["evidence"][0]["excerpt"], "等等...")
|
||
|
||
# 反例：sourceId 含真实路径穿越 `a/../b`（`..` 作为独立路径分量）仍被精确判定拦下。
|
||
traversal = blind_input("reviewer-1")
|
||
traversal["fineOutline"]["sourceRef"]["sourceId"] = "a/../b"
|
||
traversal["blindInputSha256"] = canonical_sha256({k: v for k, v in traversal.items() if k != "blindInputSha256"})
|
||
with self.assertRaises(BlindJudgeContractError) as caught:
|
||
validate_blind_judge_input(traversal)
|
||
self.assertEqual(caught.exception.code, "BLIND_JUDGE_LEAKAGE_DETECTED")
|
||
|
||
def test_three_reviewers_pairwise_unstable_fail_closed_without_winner(self) -> None:
|
||
# 三评评分两两差 >0.5（7.0 / 8.0 / 9.0），无任何稳定配对：必须失败关闭为
|
||
# JUDGE_UNSTABLE / failed_judge_unstable，不得强行给出报告、胜负或方向结论。
|
||
first = blind_input("reviewer-1")
|
||
second = blind_input("reviewer-2", order=("blind-2", "blind-1"))
|
||
third = blind_input("reviewer-3")
|
||
runner = SequenceRunner([
|
||
blind_draft(first, score=7.0),
|
||
blind_draft(second, score=8.0),
|
||
blind_draft(third, score=9.0),
|
||
])
|
||
result = run_writer_blind_judge_panel(first, second, third_input=third, model_runner=runner, persist_lesson=False)
|
||
self.assertFalse(result["ok"])
|
||
self.assertFalse(result["acceptanceEligible"])
|
||
self.assertEqual(result["primaryCode"], "JUDGE_UNSTABLE")
|
||
self.assertEqual(result["status"], "failed_judge_unstable")
|
||
self.assertEqual(result["reviewCount"], 3)
|
||
self.assertEqual(len(runner.calls), 3)
|
||
# 失败关闭：不产出报告，也没有任何稳定仲裁/方向结论字段。
|
||
self.assertNotIn("report", result)
|
||
self.assertNotIn("candidateScores", result)
|
||
self.assertNotEqual(result.get("status"), "adjudicated_report")
|
||
|
||
|
||
class QuoteLocationTest(unittest.TestCase):
|
||
"""盲评 _quote_location 的引文定位：0 次失败关闭，≥1 次绑定首次出现。"""
|
||
|
||
def test_quote_appearing_once_returns_exact_offsets(self) -> None:
|
||
# 情形①：引文在候选正文里恰好出现 1 次 → 正常返回精确 start/end。
|
||
body = "林澈守住城门。"
|
||
text, start, end = _quote_location(body, "城门", "$.candidateScores[0].scores.x.candidateQuote")
|
||
self.assertEqual(text, "城门")
|
||
self.assertEqual(start, body.index("城门"))
|
||
self.assertEqual((start, end), (4, 6))
|
||
self.assertEqual(body[start:end], "城门")
|
||
|
||
def test_quote_appearing_multiple_times_binds_to_first_occurrence(self) -> None:
|
||
# 情形②：引文出现 ≥2 次 → 不再报错，由确定性代码绑定到首次出现位置。
|
||
body = "城门连着城门。"
|
||
text, start, end = _quote_location(body, "城门", "$.candidateScores[0].scores.x.candidateQuote")
|
||
self.assertEqual(text, "城门")
|
||
self.assertEqual(start, body.index("城门")) # 首次出现处
|
||
self.assertEqual((start, end), (0, 2))
|
||
self.assertEqual(body[start:end], "城门")
|
||
|
||
def test_quote_absent_fails_closed_as_fabricated_evidence(self) -> None:
|
||
# 情形③：引文 0 次 = 编造证据 → 失败关闭，码 BLIND_JUDGE_QUOTE_NOT_FOUND。
|
||
body = "林澈守住城门。"
|
||
with self.assertRaises(BlindJudgeContractError) as caught:
|
||
_quote_location(body, "退路", "$.candidateScores[0].scores.x.candidateQuote")
|
||
self.assertEqual(caught.exception.code, "BLIND_JUDGE_QUOTE_NOT_FOUND")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|