255 lines
9.5 KiB
Python
255 lines
9.5 KiB
Python
#!/usr/bin/env python3
|
|
"""正文回放五维量表与确定性稳定性仲裁。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from statistics import median
|
|
from typing import Any, Mapping, Sequence
|
|
|
|
|
|
RUBRIC_PROFILE = "writer_replay"
|
|
DIMENSIONS = (
|
|
"setting_entity_fidelity",
|
|
"fine_outline_fidelity",
|
|
"style_consistency",
|
|
"narrative_tension",
|
|
"prose_readability",
|
|
)
|
|
EVIDENCE_SOURCE_TYPES = frozenset(
|
|
{"fine_outline", "historical_prose", "judge_inference"}
|
|
)
|
|
REPORT_FIELDS = frozenset(
|
|
{"profile", "reviewerId", "sampleId", "blindCandidateId", "candidateOrder", "scores"}
|
|
)
|
|
|
|
|
|
def _is_half_step(value: Any) -> bool:
|
|
"""只接受 0 到 10 的数字和 0.5 步长，布尔值不算数字。"""
|
|
|
|
return (
|
|
not isinstance(value, bool)
|
|
and isinstance(value, (int, float))
|
|
and 0 <= float(value) <= 10
|
|
and float(value) * 2 == int(float(value) * 2)
|
|
)
|
|
|
|
|
|
def validate_scores(scores: Any) -> list[str]:
|
|
"""校验五维分数、步长和逐项证据归因。"""
|
|
|
|
if not isinstance(scores, Mapping):
|
|
return ["scores 必须是对象"]
|
|
errors: list[str] = []
|
|
missing = [dimension for dimension in DIMENSIONS if dimension not in scores]
|
|
unexpected = sorted(set(scores) - set(DIMENSIONS))
|
|
if missing:
|
|
errors.append(f"缺少正文 rubric 维度: {','.join(missing)}")
|
|
if unexpected:
|
|
errors.append(f"存在未登记正文 rubric 维度: {','.join(unexpected)}")
|
|
for dimension in DIMENSIONS:
|
|
item = scores.get(dimension)
|
|
if not isinstance(item, Mapping):
|
|
errors.append(f"维度必须包含 score/evidence 对象: {dimension}")
|
|
continue
|
|
if set(item) != {"score", "evidence"}:
|
|
errors.append(f"维度字段必须精确为 score/evidence: {dimension}")
|
|
if not _is_half_step(item.get("score")):
|
|
errors.append(f"分数必须在 0-10 且使用 0.5 步长: {dimension}")
|
|
evidence = item.get("evidence")
|
|
if not isinstance(evidence, list) or not evidence:
|
|
errors.append(f"分数缺少证据: {dimension}")
|
|
continue
|
|
for index, raw in enumerate(evidence):
|
|
if not isinstance(raw, Mapping):
|
|
errors.append(f"证据必须是对象: {dimension}[{index}]")
|
|
continue
|
|
if set(raw) != {"sourceType", "sourceRef", "excerpt"}:
|
|
errors.append(f"证据字段必须精确为 sourceType/sourceRef/excerpt: {dimension}[{index}]")
|
|
continue
|
|
if raw.get("sourceType") not in EVIDENCE_SOURCE_TYPES:
|
|
errors.append(f"证据来源类型未登记: {dimension}[{index}]")
|
|
for field in ("sourceRef", "excerpt"):
|
|
if not isinstance(raw.get(field), str) or not raw[field].strip():
|
|
errors.append(f"证据 {field} 不能为空: {dimension}[{index}]")
|
|
return errors
|
|
|
|
|
|
def validate_report(report: Any) -> list[str]:
|
|
"""校验单个盲评报告，真实臂名或额外字段一律失败关闭。"""
|
|
|
|
if not isinstance(report, Mapping):
|
|
return ["评委报告必须是对象"]
|
|
errors: list[str] = []
|
|
if set(report) != REPORT_FIELDS:
|
|
errors.append("评委报告字段非法，去盲前不得包含真实臂名或额外信息")
|
|
if report.get("profile") != RUBRIC_PROFILE:
|
|
errors.append(f"profile 必须是 {RUBRIC_PROFILE}")
|
|
for field in ("reviewerId", "sampleId", "blindCandidateId"):
|
|
if not isinstance(report.get(field), str) or not report[field].strip():
|
|
errors.append(f"{field} 必须是非空字符串")
|
|
order = report.get("candidateOrder")
|
|
if (
|
|
not isinstance(order, list)
|
|
or len(order) < 2
|
|
or any(not isinstance(item, str) or not item for item in order)
|
|
or len(set(order)) != len(order)
|
|
):
|
|
errors.append("candidateOrder 必须是无重复盲化候选 ID 数组")
|
|
elif report.get("blindCandidateId") not in order:
|
|
errors.append("blindCandidateId 不在本轮 candidateOrder 中")
|
|
errors.extend(validate_scores(report.get("scores")))
|
|
return errors
|
|
|
|
|
|
def validate_blind_pair(first: Any, second: Any) -> list[str]:
|
|
"""确认双评独立、同样本、同候选集合且第二评委严格反序。"""
|
|
|
|
errors = [*validate_report(first), *validate_report(second)]
|
|
if errors or not isinstance(first, Mapping) or not isinstance(second, Mapping):
|
|
return errors
|
|
if first["reviewerId"] == second["reviewerId"]:
|
|
errors.append("双评委必须使用独立 reviewerId")
|
|
if first["sampleId"] != second["sampleId"]:
|
|
errors.append("双评委必须评审同一样本")
|
|
if first["blindCandidateId"] != second["blindCandidateId"]:
|
|
errors.append("双评委必须评审同一盲化候选")
|
|
if set(first["candidateOrder"]) != set(second["candidateOrder"]):
|
|
errors.append("双评委的 candidateOrder 必须包含同一盲化候选集合")
|
|
if second["candidateOrder"] != list(reversed(first["candidateOrder"])):
|
|
errors.append("第二评委必须对相同盲化候选严格反序")
|
|
return errors
|
|
|
|
|
|
def deblind_reports(
|
|
reports: Sequence[Mapping[str, Any]], mapping: Mapping[str, str]
|
|
) -> list[dict[str, Any]]:
|
|
"""评分结束后按预注册映射去盲，绝不依赖展示位置推断真实臂。"""
|
|
|
|
if set(mapping.values()) != {"A", "B", "C"} or len(mapping) != 3:
|
|
raise ValueError("去盲映射必须将三个盲 ID 一一映射到 A/B/C")
|
|
result: list[dict[str, Any]] = []
|
|
for index, report in enumerate(reports):
|
|
errors = validate_report(report)
|
|
if errors:
|
|
raise ValueError(f"reports[{index}] 非法: {'; '.join(errors)}")
|
|
blind_id = str(report["blindCandidateId"])
|
|
if blind_id not in mapping:
|
|
raise ValueError(f"reports[{index}] 的 blindCandidateId 未预注册")
|
|
result.append({**dict(report), "arm": mapping[blind_id]})
|
|
return result
|
|
|
|
|
|
def _numeric_scores(report: Mapping[str, Any]) -> dict[str, float]:
|
|
"""从已校验报告提取确定性浮点分数。"""
|
|
|
|
return {
|
|
dimension: float(report["scores"][dimension]["score"])
|
|
for dimension in DIMENSIONS
|
|
}
|
|
|
|
|
|
def _stable_pair_exists(values: Sequence[float], threshold: float) -> bool:
|
|
"""判断三个评分中是否至少存在一对落在稳定阈值内。"""
|
|
|
|
return any(
|
|
abs(values[left] - values[right]) <= threshold
|
|
for left in range(len(values))
|
|
for right in range(left + 1, len(values))
|
|
)
|
|
|
|
|
|
def adjudicate_reviews(
|
|
first: Mapping[str, Any],
|
|
second: Mapping[str, Any],
|
|
third: Mapping[str, Any] | None = None,
|
|
*,
|
|
threshold: float = 0.5,
|
|
) -> dict[str, Any]:
|
|
"""按双评差异触发最多一次第三评委，并输出稳定终态。"""
|
|
|
|
pair_errors = validate_blind_pair(first, second)
|
|
if pair_errors:
|
|
return {"status": "invalid_report", "errors": pair_errors}
|
|
first_scores = _numeric_scores(first)
|
|
second_scores = _numeric_scores(second)
|
|
unstable = [
|
|
dimension
|
|
for dimension in DIMENSIONS
|
|
if abs(first_scores[dimension] - second_scores[dimension]) > threshold
|
|
]
|
|
if not unstable:
|
|
return {
|
|
"status": "stable_report",
|
|
"scores": {
|
|
dimension: (first_scores[dimension] + second_scores[dimension]) / 2
|
|
for dimension in DIMENSIONS
|
|
},
|
|
"unstableDimensions": [],
|
|
"reviewCount": 2,
|
|
}
|
|
if third is None:
|
|
return {
|
|
"status": "needs_third_reviewer",
|
|
"unstableDimensions": unstable,
|
|
"reviewCount": 2,
|
|
}
|
|
third_errors = validate_report(third)
|
|
if third_errors:
|
|
return {"status": "invalid_report", "errors": third_errors}
|
|
if third["sampleId"] != first["sampleId"]:
|
|
return {"status": "invalid_report", "errors": ["第三评委必须评审同一样本"]}
|
|
if third["blindCandidateId"] != first["blindCandidateId"]:
|
|
return {"status": "invalid_report", "errors": ["第三评委必须评审同一盲化候选"]}
|
|
allowed_third_orders = (
|
|
first["candidateOrder"],
|
|
list(reversed(first["candidateOrder"])),
|
|
)
|
|
if third["candidateOrder"] not in allowed_third_orders:
|
|
return {
|
|
"status": "invalid_report",
|
|
"errors": ["第三评委必须使用与双评相同的盲化候选集合及第一评或反序顺序"],
|
|
}
|
|
if third["reviewerId"] in {first["reviewerId"], second["reviewerId"]}:
|
|
return {"status": "invalid_report", "errors": ["第三评委必须使用独立 reviewerId"]}
|
|
|
|
third_scores = _numeric_scores(third)
|
|
unresolved = [
|
|
dimension
|
|
for dimension in unstable
|
|
if not _stable_pair_exists(
|
|
[first_scores[dimension], second_scores[dimension], third_scores[dimension]],
|
|
threshold,
|
|
)
|
|
]
|
|
if unresolved:
|
|
return {
|
|
"status": "invalid_unstable",
|
|
"unstableDimensions": unresolved,
|
|
"reviewCount": 3,
|
|
}
|
|
return {
|
|
"status": "adjudicated_report",
|
|
"scores": {
|
|
dimension: float(
|
|
median(
|
|
[first_scores[dimension], second_scores[dimension], third_scores[dimension]]
|
|
)
|
|
)
|
|
for dimension in DIMENSIONS
|
|
},
|
|
"unstableDimensions": unstable,
|
|
"reviewCount": 3,
|
|
}
|
|
|
|
|
|
__all__ = [
|
|
"RUBRIC_PROFILE",
|
|
"DIMENSIONS",
|
|
"EVIDENCE_SOURCE_TYPES",
|
|
"validate_scores",
|
|
"validate_report",
|
|
"validate_blind_pair",
|
|
"deblind_reports",
|
|
"adjudicate_reviews",
|
|
]
|