183 lines
7.0 KiB
Python
183 lines
7.0 KiB
Python
#!/usr/bin/env python3
|
|
"""正文五维盲评量表的离线回归测试。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pathlib
|
|
import sys
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
|
|
|
from writer_rubric import ( # noqa: E402
|
|
DIMENSIONS,
|
|
RUBRIC_PROFILE,
|
|
adjudicate_reviews,
|
|
deblind_reports,
|
|
validate_blind_pair,
|
|
validate_report,
|
|
)
|
|
|
|
|
|
def scorecard(score: float = 8.0) -> dict[str, dict[str, object]]:
|
|
"""构造包含四类证据归因的合法五维评分。"""
|
|
|
|
source_types = ("fine_outline", "historical_prose", "judge_inference")
|
|
return {
|
|
dimension: {
|
|
"score": score,
|
|
"evidence": [
|
|
{
|
|
"sourceType": source_types[index % len(source_types)],
|
|
"sourceRef": f"source:{dimension}",
|
|
"excerpt": f"证据-{dimension}",
|
|
}
|
|
],
|
|
}
|
|
for index, dimension in enumerate(DIMENSIONS)
|
|
}
|
|
|
|
|
|
def report(
|
|
reviewer_id: str,
|
|
scores: dict[str, dict[str, object]] | None = None,
|
|
*,
|
|
order: tuple[str, ...] = ("blind-1", "blind-2", "blind-3"),
|
|
) -> dict[str, object]:
|
|
"""构造不携带真实臂名的单评委报告。"""
|
|
|
|
return {
|
|
"profile": RUBRIC_PROFILE,
|
|
"reviewerId": reviewer_id,
|
|
"sampleId": "deep-space-489",
|
|
"blindCandidateId": "blind-1",
|
|
"candidateOrder": list(order),
|
|
"scores": scores or scorecard(),
|
|
}
|
|
|
|
|
|
class WriterRubricValidationTest(unittest.TestCase):
|
|
"""验证量表结构与盲评纪律。"""
|
|
|
|
def test_five_dimensions_use_zero_to_ten_half_steps(self):
|
|
self.assertEqual(len(DIMENSIONS), 5)
|
|
self.assertEqual(validate_report(report("judge-1", scorecard(0.0))), [])
|
|
self.assertEqual(validate_report(report("judge-1", scorecard(10.0))), [])
|
|
|
|
bad = scorecard(8.0)
|
|
bad[DIMENSIONS[0]]["score"] = 8.25
|
|
self.assertTrue(any("0.5" in error for error in validate_report(report("judge-1", bad))))
|
|
|
|
def test_each_dimension_requires_typed_evidence(self):
|
|
bad = scorecard()
|
|
bad[DIMENSIONS[2]]["evidence"] = []
|
|
self.assertTrue(any("证据" in error for error in validate_report(report("judge-1", bad))))
|
|
|
|
bad = scorecard()
|
|
bad[DIMENSIONS[3]]["evidence"][0]["sourceType"] = "target_answer"
|
|
self.assertTrue(any("来源类型" in error for error in validate_report(report("judge-1", bad))))
|
|
|
|
hidden_card = scorecard()
|
|
hidden_card[DIMENSIONS[0]]["evidence"][0]["sourceType"] = "card_index"
|
|
self.assertTrue(
|
|
any("来源类型" in error for error in validate_report(report("judge-1", hidden_card)))
|
|
)
|
|
|
|
def test_pair_must_be_independent_blind_and_reverse_ordered(self):
|
|
first = report("judge-1")
|
|
second = report("judge-2", order=("blind-3", "blind-2", "blind-1"))
|
|
self.assertEqual(validate_blind_pair(first, second), [])
|
|
|
|
same_order = report("judge-2")
|
|
self.assertTrue(any("反序" in error for error in validate_blind_pair(first, same_order)))
|
|
|
|
different_candidate = report(
|
|
"judge-2", order=("blind-3", "blind-2", "blind-1")
|
|
)
|
|
different_candidate["blindCandidateId"] = "blind-2"
|
|
self.assertTrue(
|
|
any("同一盲化候选" in error for error in validate_blind_pair(first, different_candidate))
|
|
)
|
|
|
|
leaked = report("judge-2", order=("blind-3", "blind-2", "blind-1"))
|
|
leaked["arm"] = "C"
|
|
self.assertTrue(any("去盲" in error for error in validate_blind_pair(first, leaked)))
|
|
|
|
def test_deblind_uses_preregistered_mapping_not_position(self):
|
|
reports = [
|
|
report("judge-1"),
|
|
{**report("judge-1"), "blindCandidateId": "blind-3"},
|
|
]
|
|
mapping = {"blind-1": "C", "blind-2": "A", "blind-3": "B"}
|
|
|
|
deblinded = deblind_reports(reports, mapping)
|
|
self.assertEqual([item["arm"] for item in deblinded], ["C", "B"])
|
|
|
|
|
|
class WriterRubricAdjudicationTest(unittest.TestCase):
|
|
"""验证双评稳定性与最多一次第三评委仲裁。"""
|
|
|
|
def test_gap_above_half_requires_third_reviewer_once(self):
|
|
first = report("judge-1", scorecard(8.0))
|
|
second_scores = scorecard(8.0)
|
|
second_scores[DIMENSIONS[0]]["score"] = 7.0
|
|
second = report("judge-2", second_scores, order=("blind-3", "blind-2", "blind-1"))
|
|
|
|
result = adjudicate_reviews(first, second)
|
|
self.assertEqual(result["status"], "needs_third_reviewer")
|
|
self.assertEqual(result["unstableDimensions"], [DIMENSIONS[0]])
|
|
|
|
def test_third_reviewer_uses_median_when_any_stable_pair_exists(self):
|
|
first = report("judge-1", scorecard(8.0))
|
|
second_scores = scorecard(8.0)
|
|
second_scores[DIMENSIONS[1]]["score"] = 6.5
|
|
second = report("judge-2", second_scores, order=("blind-3", "blind-2", "blind-1"))
|
|
third_scores = scorecard(8.0)
|
|
third_scores[DIMENSIONS[1]]["score"] = 7.5
|
|
third = report("judge-3", third_scores)
|
|
|
|
result = adjudicate_reviews(first, second, third)
|
|
self.assertEqual(result["status"], "adjudicated_report")
|
|
self.assertEqual(result["scores"][DIMENSIONS[1]], 7.5)
|
|
|
|
def test_third_reviewer_must_bind_same_candidate_and_candidate_set(self):
|
|
"""第三评不得用另一候选或漂移后的候选集合参与仲裁。"""
|
|
|
|
first = report("judge-1", scorecard(8.0))
|
|
second_scores = scorecard(8.0)
|
|
second_scores[DIMENSIONS[1]]["score"] = 6.5
|
|
second = report("judge-2", second_scores, order=("blind-3", "blind-2", "blind-1"))
|
|
|
|
different_candidate = report("judge-3", scorecard(7.5))
|
|
different_candidate["blindCandidateId"] = "blind-2"
|
|
candidate_result = adjudicate_reviews(first, second, different_candidate)
|
|
self.assertEqual(candidate_result["status"], "invalid_report")
|
|
self.assertTrue(any("同一盲化候选" in error for error in candidate_result["errors"]))
|
|
|
|
different_set = report(
|
|
"judge-3",
|
|
scorecard(7.5),
|
|
order=("blind-1", "blind-2", "blind-4"),
|
|
)
|
|
set_result = adjudicate_reviews(first, second, different_set)
|
|
self.assertEqual(set_result["status"], "invalid_report")
|
|
self.assertTrue(any("相同的盲化候选集合" in error for error in set_result["errors"]))
|
|
|
|
def test_no_stable_pair_marks_sample_invalid(self):
|
|
first = report("judge-1", scorecard(8.0))
|
|
second_scores = scorecard(8.0)
|
|
second_scores[DIMENSIONS[4]]["score"] = 6.5
|
|
second = report("judge-2", second_scores, order=("blind-3", "blind-2", "blind-1"))
|
|
third_scores = scorecard(8.0)
|
|
third_scores[DIMENSIONS[4]]["score"] = 9.5
|
|
third = report("judge-3", third_scores)
|
|
|
|
result = adjudicate_reviews(first, second, third)
|
|
self.assertEqual(result["status"], "invalid_unstable")
|
|
self.assertEqual(result["unstableDimensions"], [DIMENSIONS[4]])
|
|
self.assertNotIn("winner", result)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|