337 lines
13 KiB
Python
337 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""正文五维盲评量表的离线回归测试。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pathlib
|
|
import sys
|
|
import unittest
|
|
|
|
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
SCRIPT_DIR = PROJECT_ROOT / "muse" / "lifecycle" / "quality" / "skills" / "judge" / "评估内容质量" / "scripts"
|
|
if str(SCRIPT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPT_DIR))
|
|
|
|
from writer_rubric import ( # noqa: E402
|
|
COMMON_DIMENSIONS,
|
|
DIMENSIONS,
|
|
RUBRIC_POLICY_VERSION,
|
|
RUBRIC_PROFILE,
|
|
SCENARIO_DIMENSION,
|
|
SCENARIO_TYPES,
|
|
adjudicate_reviews,
|
|
adjudicate_structured_reviews,
|
|
deblind_reports,
|
|
rubric_for_scenario,
|
|
validate_blind_pair,
|
|
validate_report,
|
|
validate_structured_report,
|
|
)
|
|
|
|
|
|
def scorecard(score: float = 8.0) -> dict[str, dict[str, object]]:
|
|
"""构造包含四类证据归因的合法五维评分。"""
|
|
|
|
source_types = ("fine_outline", "historical_prose", "judge_inference")
|
|
return {
|
|
dimension: {
|
|
"score": score,
|
|
"evidence": [
|
|
{
|
|
"sourceType": source_types[index % len(source_types)],
|
|
"sourceRef": f"source:{dimension}",
|
|
"excerpt": f"证据-{dimension}",
|
|
}
|
|
],
|
|
}
|
|
for index, dimension in enumerate(DIMENSIONS)
|
|
}
|
|
|
|
|
|
def report(
|
|
reviewer_id: str,
|
|
scores: dict[str, dict[str, object]] | None = None,
|
|
*,
|
|
order: tuple[str, ...] = ("blind-1", "blind-2", "blind-3"),
|
|
) -> dict[str, object]:
|
|
"""构造不携带真实臂名的单评委报告。"""
|
|
|
|
return {
|
|
"profile": RUBRIC_PROFILE,
|
|
"reviewerId": reviewer_id,
|
|
"sampleId": "deep-space-489",
|
|
"blindCandidateId": "blind-1",
|
|
"candidateOrder": list(order),
|
|
"scores": scores or scorecard(),
|
|
}
|
|
|
|
|
|
class WriterRubricValidationTest(unittest.TestCase):
|
|
"""验证量表结构与盲评纪律。"""
|
|
|
|
def test_five_dimensions_use_zero_to_ten_half_steps(self):
|
|
self.assertEqual(len(DIMENSIONS), 5)
|
|
self.assertEqual(validate_report(report("judge-1", scorecard(0.0))), [])
|
|
self.assertEqual(validate_report(report("judge-1", scorecard(10.0))), [])
|
|
|
|
bad = scorecard(8.0)
|
|
bad[DIMENSIONS[0]]["score"] = 8.25
|
|
self.assertTrue(any("0.5" in error for error in validate_report(report("judge-1", bad))))
|
|
|
|
def test_policy_combines_common_dimensions_with_registered_scenario_profile(self):
|
|
expected_bands = ["9-10", "7-8.5", "5-6.5", "3-4.5", "0-2.5"]
|
|
display_names: set[str] = set()
|
|
for scenario in SCENARIO_TYPES:
|
|
policy = rubric_for_scenario(scenario)
|
|
self.assertEqual(policy["policyVersion"], RUBRIC_POLICY_VERSION)
|
|
self.assertEqual(policy["scenarioType"], scenario)
|
|
self.assertEqual(policy["dimensions"], list(DIMENSIONS))
|
|
self.assertEqual(
|
|
[item["dimensionId"] for item in policy["commonDimensions"]],
|
|
list(COMMON_DIMENSIONS),
|
|
)
|
|
for item in policy["commonDimensions"]:
|
|
self.assertEqual(
|
|
[anchor["scoreBand"] for anchor in item["anchors"]],
|
|
expected_bands,
|
|
)
|
|
scenario_dimension = policy["scenarioDimension"]
|
|
self.assertEqual(scenario_dimension["dimensionId"], SCENARIO_DIMENSION)
|
|
self.assertEqual(
|
|
[anchor["scoreBand"] for anchor in scenario_dimension["anchors"]],
|
|
expected_bands,
|
|
)
|
|
self.assertGreaterEqual(len(scenario_dimension["criteria"]), 4)
|
|
display_names.add(scenario_dimension["displayName"])
|
|
self.assertEqual(len(display_names), len(SCENARIO_TYPES))
|
|
|
|
with self.assertRaisesRegex(ValueError, "未登记场景类型"):
|
|
rubric_for_scenario("unknown")
|
|
|
|
def test_each_dimension_requires_typed_evidence(self):
|
|
bad = scorecard()
|
|
bad[DIMENSIONS[2]]["evidence"] = []
|
|
self.assertTrue(any("证据" in error for error in validate_report(report("judge-1", bad))))
|
|
|
|
bad = scorecard()
|
|
bad[DIMENSIONS[3]]["evidence"][0]["sourceType"] = "target_answer"
|
|
self.assertTrue(any("来源类型" in error for error in validate_report(report("judge-1", bad))))
|
|
|
|
hidden_card = scorecard()
|
|
hidden_card[DIMENSIONS[0]]["evidence"][0]["sourceType"] = "card_index"
|
|
self.assertTrue(
|
|
any("来源类型" in error for error in validate_report(report("judge-1", hidden_card)))
|
|
)
|
|
|
|
def test_pair_must_be_independent_blind_and_reverse_ordered(self):
|
|
first = report("judge-1")
|
|
second = report("judge-2", order=("blind-3", "blind-2", "blind-1"))
|
|
self.assertEqual(validate_blind_pair(first, second), [])
|
|
|
|
same_order = report("judge-2")
|
|
self.assertTrue(any("反序" in error for error in validate_blind_pair(first, same_order)))
|
|
|
|
different_candidate = report(
|
|
"judge-2", order=("blind-3", "blind-2", "blind-1")
|
|
)
|
|
different_candidate["blindCandidateId"] = "blind-2"
|
|
self.assertTrue(
|
|
any("同一盲化候选" in error for error in validate_blind_pair(first, different_candidate))
|
|
)
|
|
|
|
leaked = report("judge-2", order=("blind-3", "blind-2", "blind-1"))
|
|
leaked["arm"] = "C"
|
|
self.assertTrue(any("去盲" in error for error in validate_blind_pair(first, leaked)))
|
|
|
|
def test_deblind_uses_preregistered_mapping_not_position(self):
|
|
reports = [
|
|
report("judge-1"),
|
|
{**report("judge-1"), "blindCandidateId": "blind-3"},
|
|
]
|
|
mapping = {"blind-1": "C", "blind-2": "A", "blind-3": "B"}
|
|
|
|
deblinded = deblind_reports(reports, mapping)
|
|
self.assertEqual([item["arm"] for item in deblinded], ["C", "B"])
|
|
|
|
|
|
class WriterRubricAdjudicationTest(unittest.TestCase):
|
|
"""验证双评稳定性与最多一次第三评委仲裁。"""
|
|
|
|
def test_gap_above_half_requires_third_reviewer_once(self):
|
|
first = report("judge-1", scorecard(8.0))
|
|
second_scores = scorecard(8.0)
|
|
second_scores[DIMENSIONS[0]]["score"] = 7.0
|
|
second = report("judge-2", second_scores, order=("blind-3", "blind-2", "blind-1"))
|
|
|
|
result = adjudicate_reviews(first, second)
|
|
self.assertEqual(result["status"], "needs_third_reviewer")
|
|
self.assertEqual(result["unstableDimensions"], [DIMENSIONS[0]])
|
|
|
|
def test_third_reviewer_uses_median_when_any_stable_pair_exists(self):
|
|
first = report("judge-1", scorecard(8.0))
|
|
second_scores = scorecard(8.0)
|
|
second_scores[DIMENSIONS[1]]["score"] = 6.5
|
|
second = report("judge-2", second_scores, order=("blind-3", "blind-2", "blind-1"))
|
|
third_scores = scorecard(8.0)
|
|
third_scores[DIMENSIONS[1]]["score"] = 7.5
|
|
third = report("judge-3", third_scores)
|
|
|
|
result = adjudicate_reviews(first, second, third)
|
|
self.assertEqual(result["status"], "adjudicated_report")
|
|
self.assertEqual(result["scores"][DIMENSIONS[1]], 7.5)
|
|
|
|
def test_third_reviewer_must_bind_same_candidate_and_candidate_set(self):
|
|
"""第三评不得用另一候选或漂移后的候选集合参与仲裁。"""
|
|
|
|
first = report("judge-1", scorecard(8.0))
|
|
second_scores = scorecard(8.0)
|
|
second_scores[DIMENSIONS[1]]["score"] = 6.5
|
|
second = report("judge-2", second_scores, order=("blind-3", "blind-2", "blind-1"))
|
|
|
|
different_candidate = report("judge-3", scorecard(7.5))
|
|
different_candidate["blindCandidateId"] = "blind-2"
|
|
candidate_result = adjudicate_reviews(first, second, different_candidate)
|
|
self.assertEqual(candidate_result["status"], "invalid_report")
|
|
self.assertTrue(any("同一盲化候选" in error for error in candidate_result["errors"]))
|
|
|
|
different_set = report(
|
|
"judge-3",
|
|
scorecard(7.5),
|
|
order=("blind-1", "blind-2", "blind-4"),
|
|
)
|
|
set_result = adjudicate_reviews(first, second, different_set)
|
|
self.assertEqual(set_result["status"], "invalid_report")
|
|
self.assertTrue(any("相同的盲化候选集合" in error for error in set_result["errors"]))
|
|
|
|
def test_no_stable_pair_marks_sample_invalid(self):
|
|
first = report("judge-1", scorecard(8.0))
|
|
second_scores = scorecard(8.0)
|
|
second_scores[DIMENSIONS[4]]["score"] = 6.5
|
|
second = report("judge-2", second_scores, order=("blind-3", "blind-2", "blind-1"))
|
|
third_scores = scorecard(8.0)
|
|
third_scores[DIMENSIONS[4]]["score"] = 9.5
|
|
third = report("judge-3", third_scores)
|
|
|
|
result = adjudicate_reviews(first, second, third)
|
|
self.assertEqual(result["status"], "invalid_unstable")
|
|
self.assertEqual(result["unstableDimensions"], [DIMENSIONS[4]])
|
|
self.assertNotIn("winner", result)
|
|
|
|
|
|
def structured_report(
|
|
reviewer_id: str,
|
|
*,
|
|
order: tuple[str, ...] = ("blind-1", "blind-2"),
|
|
score: float = 8.0,
|
|
verdict: str = "pass",
|
|
) -> dict[str, object]:
|
|
"""构造已完成输入绑定后的 v3 结构化报告，专测 rubric 稳定仲裁。"""
|
|
|
|
candidate_hashes = {
|
|
"blind-1": "sha256:" + "1" * 64,
|
|
"blind-2": "sha256:" + "2" * 64,
|
|
}
|
|
return {
|
|
"schemaVersion": "blind-judge-report-v4",
|
|
"runId": "run-1",
|
|
"sampleId": "sample-1",
|
|
"scenario": "battle",
|
|
"rubricPolicyVersion": RUBRIC_POLICY_VERSION,
|
|
"reviewerInvocationId": reviewer_id,
|
|
"blindInputSha256": "sha256:" + "3" * 64,
|
|
"oracleTruthPackSha256": "sha256:" + "4" * 64,
|
|
"candidateOrder": list(order),
|
|
"candidateScores": [
|
|
{
|
|
"blindCandidateId": blind_id,
|
|
"candidateSha256": candidate_hashes[blind_id],
|
|
"scores": scorecard(score),
|
|
}
|
|
for blind_id in order
|
|
],
|
|
"dimensionPreferences": [
|
|
{
|
|
"dimension": dimension,
|
|
"orderedCandidateIds": list(order),
|
|
"reason": "共同 rubric 下的匿名偏好顺序",
|
|
}
|
|
for dimension in DIMENSIONS
|
|
],
|
|
"oracleAssertionVerdicts": [
|
|
{
|
|
"assertionId": "assertion-1",
|
|
"candidateSha256": candidate_hashes[blind_id],
|
|
"verdict": verdict,
|
|
"reason": "共同 oracle 断言核对结果",
|
|
"evidenceRefs": [
|
|
{"sourceType": "oracle_assertion", "sourceRef": "assertion-1"}
|
|
],
|
|
}
|
|
for blind_id in order
|
|
],
|
|
"hardConstraintVerdicts": [
|
|
{
|
|
"constraintId": "constraint-1",
|
|
"candidateSha256": candidate_hashes[blind_id],
|
|
"verdict": "pass",
|
|
"reason": "共同细纲硬约束核对结果",
|
|
"evidenceRefs": [
|
|
{
|
|
"sourceType": "preregistered_constraint",
|
|
"sourceRef": "constraint-1",
|
|
}
|
|
],
|
|
}
|
|
for blind_id in order
|
|
],
|
|
"modelReceiptSha256": "sha256:" + "5" * 64,
|
|
"status": "completed",
|
|
"reportSha256": "sha256:" + "6" * 64,
|
|
}
|
|
|
|
|
|
class WriterStructuredRubricAdjudicationTest(unittest.TestCase):
|
|
"""验证 v3 报告把逐 verdict 分歧纳入第三评委触发和最终裁决。"""
|
|
|
|
def test_structured_report_rejects_unknown_and_extra_field(self):
|
|
unknown = structured_report("reviewer-1", verdict="unknown")
|
|
self.assertTrue(any("unknown" in error for error in validate_structured_report(unknown)))
|
|
|
|
leaked = structured_report("reviewer-1")
|
|
leaked["arm"] = "C"
|
|
self.assertTrue(any("额外字段" in error for error in validate_structured_report(leaked)))
|
|
|
|
def test_verdict_disagreement_requires_and_uses_third_reviewer(self):
|
|
first = structured_report("reviewer-1")
|
|
second = structured_report(
|
|
"reviewer-2",
|
|
order=("blind-2", "blind-1"),
|
|
verdict="fail",
|
|
)
|
|
|
|
pending = adjudicate_structured_reviews(first, second)
|
|
self.assertEqual(pending["status"], "needs_third_reviewer")
|
|
self.assertTrue(pending["unstableOracleVerdicts"])
|
|
|
|
third = structured_report("reviewer-3", verdict="pass")
|
|
resolved = adjudicate_structured_reviews(first, second, third)
|
|
self.assertEqual(resolved["status"], "adjudicated_report")
|
|
self.assertEqual(resolved["oracleAssertionVerdicts"][0]["verdict"], "pass")
|
|
|
|
def test_third_reviewer_cannot_hide_score_instability(self):
|
|
first = structured_report("reviewer-1", score=8.0)
|
|
second = structured_report(
|
|
"reviewer-2",
|
|
order=("blind-2", "blind-1"),
|
|
score=6.5,
|
|
)
|
|
third = structured_report("reviewer-3", score=9.5)
|
|
|
|
result = adjudicate_structured_reviews(first, second, third)
|
|
self.assertEqual(result["status"], "failed_judge_unstable")
|
|
self.assertTrue(result["unstableScores"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|