d909d1bd1b
实现侧： - 上下文：任务范围拆分为 范围校验/范围授权；索引按可发现口径重建、索引新鲜度改对称差；依赖校验统一快照漂移说明。 - 知识方法：方法与材料读取口径统一；超限方法材料按可选省略，核对路径不再二次计费；删除无合同的读时重算。 - 任务运行：新增 context.usage/tool.denied 事件类型；连接池常驻并在装配生命周期内开关；调用结算与核对分列。 - 效果评测/审校修订/交付连载/作者经验/作品规划：凭据冻结、标定消费、导出补证、事实引文核对等收尾修复。 - 资源加载：能力正文不再夹带索引用的导航注记（该注记此前进入角色与技能的模型提示）。 - 元数据：受保护骨架与代码保护属性对齐；字段校验与内置结构口径同步。 - 基础设施：环境预检进入装配生命周期；数据库连接运行期字段不参与相等比较；索引指纹归一化 jsonb 浮点。 - 删除被替代实现：7 份旧提示词模板与空壳 资料来源 读取器。 用例侧： - 用例身份与导航元信息迁移；夹具补生命周期、同库暴露与模板封存； - 本轮定向修复：方法材料省略、事实引文、迁移回执、额度与暂停用例、慢用例超时预算等。
1328 lines
54 KiB
Python
1328 lines
54 KiB
Python
"""匿名分配、独立评委与原字引文的消费合同；不冒充真实模型执行。"""
|
||
|
||
import hashlib
|
||
import json
|
||
from copy import deepcopy
|
||
from dataclasses import replace
|
||
from pathlib import Path
|
||
from uuid import UUID
|
||
|
||
import pytest
|
||
import yaml
|
||
|
||
from muse.任务运行.接口 import (
|
||
校验模型输出,
|
||
模型协议错误,
|
||
模型结果,
|
||
模型请求,
|
||
角色策略目录,
|
||
)
|
||
from muse.审校修订.接口 import 审校错误, 核对语义检测, 组装语义检测材料
|
||
from muse.效果评测.接口 import (
|
||
仲裁文学评分,
|
||
核对匿名分配,
|
||
核对实验顺序,
|
||
核对成对判断,
|
||
核对文学判断,
|
||
比较未执行,
|
||
汇总独立选择,
|
||
生成匿名分配,
|
||
生成实验顺序,
|
||
组装成对材料,
|
||
组装文学材料,
|
||
评测错误,
|
||
)
|
||
from muse.正式变更.接口 import 固定哈希
|
||
|
||
实验ID = "b0473aae-56d1-4485-9f6e-49042b02b130"
|
||
仓库根 = Path(__file__).resolve().parents[2]
|
||
|
||
|
||
def _语义材料():
|
||
return 组装语义检测材料(
|
||
"旅者等在城门。",
|
||
{
|
||
"sources": [{"source_id": "outline", "kind": "fine_outline", "text": "旅者守门。"}],
|
||
"assertions": [{"statement_id": "guard", "text": "旅者是守卫。"}],
|
||
"constraints": [{"statement_id": "stay", "text": "不得离开城门。"}],
|
||
},
|
||
)
|
||
|
||
|
||
def _语义输出(material):
|
||
quote = material["candidate"]
|
||
return {
|
||
"claims": [
|
||
{
|
||
"claim_id": "claim-1",
|
||
"text": "候选中人物采取行动。",
|
||
"candidate_quote": quote,
|
||
"state": "supported",
|
||
"evidence_refs": ["source:outline"],
|
||
"reason": "参照细纲核对。",
|
||
}
|
||
],
|
||
"findings": [],
|
||
"assertion_verdicts": [
|
||
{
|
||
"statement_id": "guard",
|
||
"verdict": "pass",
|
||
"candidate_quote": quote,
|
||
"evidence_refs": ["assertion:guard"],
|
||
"reason": "与给定命题相符。",
|
||
}
|
||
],
|
||
"constraint_verdicts": [
|
||
{
|
||
"statement_id": "stay",
|
||
"verdict": "pass",
|
||
"candidate_quote": quote,
|
||
"evidence_refs": ["constraint:stay"],
|
||
"reason": "符合给定约束。",
|
||
}
|
||
],
|
||
"new_setting_candidates": [],
|
||
}
|
||
|
||
|
||
文学甲 = "甲候选正文：茧撕开舱门。"
|
||
文学乙 = "乙候选正文：林深盯着深渊。"
|
||
|
||
|
||
def _文学材料(*, reverse=False, scenario="turning_point"):
|
||
return 组装文学材料(
|
||
文学乙 if reverse else 文学甲,
|
||
文学甲 if reverse else 文学乙,
|
||
{
|
||
"scenario": scenario,
|
||
"sources": [
|
||
{
|
||
"source_id": "outline",
|
||
"kind": "fine_outline",
|
||
"text": "硬约束甲；硬约束乙；拍一。",
|
||
},
|
||
{
|
||
"source_id": "history",
|
||
"kind": "historical_prose",
|
||
"text": "林深先前一直守着城门。",
|
||
},
|
||
],
|
||
"assertions": [{"statement_id": "fact-one", "text": "新事实"}],
|
||
"constraints": [{"statement_id": "must-stay", "text": "硬约束甲"}],
|
||
},
|
||
)
|
||
|
||
|
||
def _文学引文(material, side, source_type="candidate", identity="q", source_id=None):
|
||
if source_type == "candidate":
|
||
source_id, text = side, material[side]["text"]
|
||
elif source_type in {"fine_outline", "historical_prose"}:
|
||
source = next(s for s in material["basis"]["sources"] if s["kind"] == source_type)
|
||
source_id, text = source["source_id"], source["text"]
|
||
else:
|
||
field = "assertions" if source_type == "oracle_assertion" else "constraints"
|
||
source = next(
|
||
s
|
||
for s in material["basis"][field]
|
||
if source_id is None or s["statement_id"] == source_id
|
||
)
|
||
source_id, text = source["statement_id"], source["text"]
|
||
return {
|
||
"evidence_id": identity,
|
||
"source_type": source_type,
|
||
"source_id": source_id,
|
||
"quote": text,
|
||
}
|
||
|
||
|
||
def _文学输出(material, score=4.0):
|
||
cards = []
|
||
for side in ("left", "right"):
|
||
scores = []
|
||
for dimension in material["dimensions"]:
|
||
evidence = [_文学引文(material, side, identity="candidate")]
|
||
if dimension != "prose_readability":
|
||
evidence.append(
|
||
_文学引文(
|
||
material,
|
||
side,
|
||
"historical_prose" if dimension == "style_consistency" else "fine_outline",
|
||
identity="context",
|
||
)
|
||
)
|
||
scores.append(
|
||
{
|
||
"dimension": dimension,
|
||
"score": score,
|
||
"rationale": "按候选及共同依据核对。",
|
||
"evidence": evidence,
|
||
"inferences": [],
|
||
}
|
||
)
|
||
cards.append({"side": side, "scores": scores})
|
||
output = {
|
||
"choice": "tie",
|
||
"rationale": "两侧按本次材料均可成立。",
|
||
"candidate_scores": cards,
|
||
"preferences": [
|
||
{"dimension": d, "choice": "tie", "rationale": "两侧表现相近。"}
|
||
for d in material["dimensions"]
|
||
],
|
||
}
|
||
for field, source_type, definitions in (
|
||
("assertion_verdicts", "oracle_assertion", "assertions"),
|
||
("constraint_verdicts", "preregistered_constraint", "constraints"),
|
||
):
|
||
output[field] = [
|
||
{
|
||
"side": side,
|
||
"statement_id": statement["statement_id"],
|
||
"verdict": "pass",
|
||
"rationale": "根据两侧正文与共同命题核对。",
|
||
"evidence": [
|
||
_文学引文(material, side, identity="candidate"),
|
||
_文学引文(
|
||
material,
|
||
side,
|
||
source_type,
|
||
identity="statement",
|
||
source_id=statement["statement_id"],
|
||
),
|
||
],
|
||
"inferences": [],
|
||
}
|
||
for side in ("left", "right")
|
||
for statement in material["basis"][definitions]
|
||
]
|
||
return output
|
||
|
||
|
||
def _核对文学(material, output):
|
||
return 核对文学判断(material, output, writer_model="writer-model", judge_model="judge-model")
|
||
|
||
|
||
def _文学评审(index, *, score=4.0, change=None, verdict=None):
|
||
material = _文学材料(reverse=index == 2)
|
||
raw = _文学输出(material, score)
|
||
side = "right" if index == 2 else "left"
|
||
if change is not None:
|
||
dimension, value = change
|
||
card = next(c for c in raw["candidate_scores"] if c["side"] == side)
|
||
next(s for s in card["scores"] if s["dimension"] == dimension)["score"] = value
|
||
if verdict is not None:
|
||
next(v for v in raw["assertion_verdicts"] if v["side"] == side)["verdict"] = verdict
|
||
return {
|
||
"judge_ref": f"judge-run-judge-test-judge-v{index}",
|
||
"model": f"judge-model-{index}",
|
||
"scope_hash": 固定哈希([文学甲, 文学乙, material["basis"], material["rubric"]]),
|
||
"sides": {
|
||
"left": "blind-2" if index == 2 else "blind-1",
|
||
"right": "blind-1" if index == 2 else "blind-2",
|
||
},
|
||
"judgment": _核对文学(material, raw),
|
||
}
|
||
|
||
|
||
def _角色目录():
|
||
return 角色策略目录(yaml.safe_load((仓库根 / "配置/角色策略.yaml").read_text()))
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-8925d954a09e",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
given="独立固定样本、合成模型身份或原字引文；每个原用例仅使用自身参数",
|
||
when="调用该原例对应的公开预注册或成对判断核验",
|
||
then=[
|
||
"五样本按数据集版本加样本ID的哈希排序，运行表循环ABC/BCA/CAB，两张表每个位置的臂数量差不超过一。"
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_五样本运行和盲化表分别哈希排序且循环平衡__8925d9():
|
||
from collections import Counter
|
||
|
||
ids = tuple(f"sample-{i}" for i in range(5))
|
||
registration, blind = 生成实验顺序(实验ID, "evaluation-set-v1", ids, ("A", "B", "C"))
|
||
execution = registration["execution_order"]
|
||
assert [r["sample_id"] for r in execution] == sorted(
|
||
ids, key=lambda sid: hashlib.sha256(("evaluation-set-v1" + sid).encode()).hexdigest()
|
||
)
|
||
rotations = [
|
||
["A", "B", "C"],
|
||
["B", "C", "A"],
|
||
["C", "A", "B"],
|
||
["A", "B", "C"],
|
||
["B", "C", "A"],
|
||
]
|
||
assert [r["arm_order"] for r in execution] == rotations
|
||
for table in ([r["arm_order"] for r in execution], [list(r.order) for r in blind]):
|
||
for position in range(3):
|
||
counts = Counter(row[position] for row in table)
|
||
assert (
|
||
max(counts[a] for a in ("A", "B", "C")) - min(counts[a] for a in ("A", "B", "C"))
|
||
<= 1
|
||
)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-91eae14dfc8a",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
given="独立固定样本、合成模型身份或原字引文；每个原用例仅使用自身参数",
|
||
when="调用该原例对应的公开预注册或成对判断核验",
|
||
then=["输入样本顺序变化不影响预注册；盲化使用独立命名空间和排序，不复用运行顺序表。"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_盲化顺序可重放且不复用运行排序命名空间__91eae1():
|
||
ids = tuple(f"sample-{i}" for i in range(5))
|
||
first = 生成实验顺序(实验ID, "evaluation-set-v1", ids, ("A", "B", "C"))
|
||
assert first == 生成实验顺序(实验ID, "evaluation-set-v1", tuple(reversed(ids)), ("A", "B", "C"))
|
||
registration, blind = first
|
||
assert [r["sample_id"] for r in registration["execution_order"]] != [r.sample_id for r in blind]
|
||
assert registration["blind_namespace"] == "writer-blind-assignment-v1"
|
||
assert "blind_assignment" not in registration
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-7a0ffd64f665",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
given="独立固定样本、合成模型身份或原字引文；每个原用例仅使用自身参数",
|
||
when="调用该原例对应的公开预注册或成对判断核验",
|
||
then=["重复样本、运行表或盲化表失衡均拒绝，不允许对局部表重新签名后放行。"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("changed", ["duplicate", "unbalanced_execution", "unbalanced_blind"])
|
||
def test_预注册拒绝重复样本和任何一张失衡表__7a0ffd(changed):
|
||
ids = ("one", "two", "three")
|
||
arms = ("A", "B", "C")
|
||
with pytest.raises(评测错误):
|
||
if changed == "duplicate":
|
||
生成实验顺序(实验ID, "evaluation-set-v1", ("same", "same"), arms)
|
||
else:
|
||
registration, blind = 生成实验顺序(实验ID, "evaluation-set-v1", ids, arms)
|
||
if changed == "unbalanced_execution":
|
||
for row in registration["execution_order"]:
|
||
row["arm_order"] = list(arms)
|
||
else:
|
||
blind = tuple(replace(r, order=arms) for r in blind)
|
||
核对实验顺序(
|
||
registration,
|
||
blind,
|
||
experiment_id=实验ID,
|
||
dataset_version_id="evaluation-set-v1",
|
||
sample_ids=ids,
|
||
arms=arms,
|
||
)
|
||
|
||
|
||
def _成对输入():
|
||
return 组装成对材料("甲推开门。", "甲走了进去。", ("叙事", "声音"))
|
||
|
||
|
||
def _成对报告(material, choice="left"):
|
||
return {
|
||
"choice": choice,
|
||
"rationale": "按下列维度比较两个匿名文本。",
|
||
"dimensions": [
|
||
{
|
||
"dimension": d,
|
||
"choice": choice,
|
||
"rationale": "根据两侧原文核对叙述方式。",
|
||
"evidence": [
|
||
{
|
||
"side": side,
|
||
"quote": material[side]["text"],
|
||
}
|
||
for side in ("left", "right")
|
||
],
|
||
}
|
||
for d in material["dimensions"]
|
||
],
|
||
}
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-a77c4d10ee77",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
given="独立固定样本、合成模型身份或原字引文；每个原用例仅使用自身参数",
|
||
when="调用该原例对应的公开预注册或成对判断核验",
|
||
then=["同一实际模型身份不得作为独立选择模型；合成身份只证明拒绝条件。"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_同一模型的选择不能当独立判断__a77c4d():
|
||
m = _成对输入()
|
||
with pytest.raises(比较未执行):
|
||
核对成对判断(m, _成对报告(m), writer_model="same-model", judge_model="same-model")
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-f131fa327bf0",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
given="独立固定样本、合成模型身份或原字引文；每个原用例仅使用自身参数",
|
||
when="调用该原例对应的公开预注册或成对判断核验",
|
||
then=["成对判断总体理由为空白时拒绝，不把结构不完整的输出当执行成功。"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_无选择理由不能执行成对判断__f131fa():
|
||
m = _成对输入()
|
||
report = _成对报告(m)
|
||
report["rationale"] = " "
|
||
with pytest.raises(比较未执行):
|
||
核对成对判断(m, report, writer_model="writer-model", judge_model="judge-model")
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w25-250001",
|
||
environment="离线纯合同",
|
||
given="固定合成样本、实验身份和两侧文本",
|
||
when="匿名分配、逐维码点引文校验或选择汇总",
|
||
then=["拒绝重复/失衡/伪引文/身份泄露；保留原文、平局、都差、未知和分歧，执行认证保持false"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("count", [1, 2, 5, 11])
|
||
def test_匿名分配可复现且三臂各位置均衡__250001(count):
|
||
from collections import Counter
|
||
|
||
ids = tuple(f"sample-{i}" for i in range(count))
|
||
arms = ("control", "reference", "method")
|
||
a = 生成匿名分配(实验ID, ids, arms)
|
||
assert a == 生成匿名分配(实验ID, tuple(reversed(ids)), arms)
|
||
assert a == 生成匿名分配(实验ID.upper(), ids, arms)
|
||
assert tuple(r.sample_id for r in a) == tuple(
|
||
sorted(ids, key=lambda s: hashlib.sha256(s.encode()).hexdigest())
|
||
)
|
||
for p in range(3):
|
||
values = Counter(r.order[p] for r in a)
|
||
assert max(values[x] for x in arms) - min(values[x] for x in arms) <= 1
|
||
other = 生成匿名分配("f4d59b1f-21cb-45ee-a294-97a4f8d0469a", ids, arms)
|
||
assert not {r.assignment_id for r in a} & {r.assignment_id for r in other}
|
||
assert all(UUID(r.assignment_id) for r in a)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w25-250002",
|
||
environment="离线纯合同",
|
||
given="固定合成样本、实验身份和两侧文本",
|
||
when="匿名分配、逐维码点引文校验或选择汇总",
|
||
then=["拒绝重复/失衡/伪引文/身份泄露；保留原文、平局、都差、未知和分歧，执行认证保持false"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize(
|
||
"bad",
|
||
["duplicate_samples", "duplicate_arms", "unbalanced", "foreign_namespace", "missing_sample"],
|
||
)
|
||
def test_匿名分配拒绝重复失衡和跨实验复用__250002(bad):
|
||
ids = ("one", "two", "three", "four", "five")
|
||
arms = ("control", "reference", "method")
|
||
with pytest.raises(评测错误):
|
||
if bad == "duplicate_samples":
|
||
生成匿名分配(实验ID, ids + ("one",), arms)
|
||
elif bad == "duplicate_arms":
|
||
生成匿名分配(实验ID, ids, arms + ("control",))
|
||
else:
|
||
rows = 生成匿名分配(实验ID, ids, arms)
|
||
if bad == "unbalanced":
|
||
rows = tuple(replace(r, order=arms) for r in rows)
|
||
elif bad == "foreign_namespace":
|
||
rows = 生成匿名分配("f4d59b1f-21cb-45ee-a294-97a4f8d0469a", ids, arms)
|
||
else:
|
||
rows = rows[:-1]
|
||
核对匿名分配(rows, ids, arms, experiment_id=实验ID)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w25-250003",
|
||
environment="离线纯合同",
|
||
given="固定合成样本、实验身份和两侧文本",
|
||
when="匿名分配、逐维码点引文校验或选择汇总",
|
||
then=["拒绝重复/失衡/伪引文/身份泄露；保留原文、平局、都差、未知和分歧，执行认证保持false"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize(
|
||
"bad",
|
||
[
|
||
"quote",
|
||
"side",
|
||
"dimension",
|
||
"duplicate_dimension",
|
||
"arm_leak",
|
||
"model_leak",
|
||
"boolean_offset",
|
||
"no_evidence",
|
||
"empty_rationale",
|
||
"unsupported_winner",
|
||
],
|
||
)
|
||
def test_逐维引文与匿名合同拒绝伪造或泄露__250003(bad):
|
||
m = _成对输入()
|
||
r = _成对报告(m)
|
||
if bad == "quote":
|
||
r["dimensions"][0]["evidence"][0]["quote"] = "不在原文里。"
|
||
elif bad == "side":
|
||
r["dimensions"][0]["evidence"][0]["side"] = "original"
|
||
elif bad == "dimension":
|
||
r["dimensions"][0]["dimension"] = "编排信息"
|
||
elif bad == "duplicate_dimension":
|
||
r["dimensions"][1]["dimension"] = r["dimensions"][0]["dimension"]
|
||
elif bad == "arm_leak":
|
||
r["dimensions"][0]["arm"] = "method"
|
||
elif bad == "model_leak":
|
||
r["selection_model"] = "claimed-judge"
|
||
elif bad == "boolean_offset":
|
||
r["dimensions"][0]["evidence"][0]["start"] = False
|
||
elif bad == "no_evidence":
|
||
r["dimensions"][1]["evidence"] = []
|
||
elif bad == "empty_rationale":
|
||
r["dimensions"][0]["rationale"] = " "
|
||
else:
|
||
r["choice"] = "right"
|
||
with pytest.raises(比较未执行):
|
||
核对成对判断(m, r, writer_model="writer-model", judge_model="judge-model")
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w25-250004",
|
||
environment="离线纯合同",
|
||
given="固定合成样本、实验身份和两侧文本",
|
||
when="匿名分配、逐维码点引文校验或选择汇总",
|
||
then=["拒绝重复/失衡/伪引文/身份泄露；保留原文、平局、都差、未知和分歧，执行认证保持false"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("choice", ["left", "right", "tie", "both_bad", "unknown"])
|
||
def test_成对结构合同保留五种合法判断但不认证调用__250004(choice):
|
||
m = _成对输入()
|
||
r = _成对报告(m, choice)
|
||
result = 核对成对判断(m, r, writer_model="writer-model", judge_model="judge-model")
|
||
assert result["choice"] == r["choice"]
|
||
assert all(
|
||
e["start"] == 0 and e["end"] == len(e["quote"])
|
||
for d in result["dimensions"]
|
||
for e in d["evidence"]
|
||
)
|
||
result = 汇总独立选择(
|
||
[{"judge_ref": "actual-receipt-placeholder", "choice": choice}], left_is_original=True
|
||
)
|
||
assert result["outcome"] == {"left": "original", "right": "candidate"}.get(choice, choice)
|
||
assert not result["execution_verified"]
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w25-250005",
|
||
environment="离线纯合同",
|
||
given="固定合成样本、实验身份和两侧文本",
|
||
when="匿名分配、逐维码点引文校验或选择汇总",
|
||
then=["拒绝重复/失衡/伪引文/身份泄露；保留原文、平局、都差、未知和分歧，执行认证保持false"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_分歧都差和缺执行不被多数投票掩盖__250005():
|
||
rows = [
|
||
{"judge_ref": f"receipt-{i}", "choice": c} for i, c in enumerate(("left", "left", "right"))
|
||
]
|
||
result = 汇总独立选择(rows, left_is_original=True)
|
||
assert result["outcome"] == "disagreement" and len(result["decisions"]) == 3
|
||
assert 汇总独立选择([], left_is_original=True)["outcome"] == "not_executed"
|
||
with pytest.raises(评测错误):
|
||
汇总独立选择([rows[0], rows[0]], left_is_original=False)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w25-250006",
|
||
environment="离线纯合同",
|
||
given="固定合成样本、实验身份和两侧文本",
|
||
when="匿名分配、逐维码点引文校验或选择汇总",
|
||
then=["拒绝重复/失衡/伪引文/身份泄露；保留原文、平局、都差、未知和分歧，执行认证保持false"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_空文本侧与相同文本不能伪造胜负__250006():
|
||
m = 组装成对材料("", "保留的版本。", ("叙事",))
|
||
r = _成对报告(m, "right")
|
||
assert 核对成对判断(m, r, writer_model="writer", judge_model="judge")["choice"] == "right"
|
||
m = 组装成对材料("相同文本。", "相同文本。", ("叙事",))
|
||
with pytest.raises(比较未执行):
|
||
核对成对判断(m, _成对报告(m), writer_model="writer", judge_model="judge")
|
||
assert (
|
||
核对成对判断(m, _成对报告(m, "tie"), writer_model="writer", judge_model="judge")["choice"]
|
||
== "tie"
|
||
)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-8dcdb623a36c",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
given="独立固定样本、合成模型身份或原字引文；每个原用例仅使用自身参数",
|
||
when="调用该原例对应的公开预注册或成对判断核验",
|
||
then=[
|
||
"模型合同任意层均不接受位置、哈希、身份或回执；逐维引文只收side/quote，伪造字段为PAIRWISE_NOT_EXECUTED。"
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize(
|
||
"field", ["start", "end", "candidateSha256", "reviewerInvocationId", "modelReceiptSha256"]
|
||
)
|
||
def test_模型合同排除位置哈希身份回执且未知字段拒绝__8dcdb6(field):
|
||
from copy import deepcopy
|
||
|
||
from muse.效果评测.接口 import 成对输出合同
|
||
|
||
schema = 成对输出合同()
|
||
nodes = [schema, *schema.get("$defs", {}).values()]
|
||
forbidden = {
|
||
"start",
|
||
"end",
|
||
"startCodePoint",
|
||
"endCodePoint",
|
||
"candidateSha256",
|
||
"reviewerInvocationId",
|
||
"modelReceiptSha256",
|
||
}
|
||
assert all(n.get("additionalProperties") is False for n in nodes if n.get("type") == "object")
|
||
assert all(forbidden.isdisjoint(n.get("properties", {})) for n in nodes)
|
||
material = _成对输入()
|
||
assert all(set(material[side]) == {"text"} for side in ("left", "right"))
|
||
report = deepcopy(_成对报告(material))
|
||
report["dimensions"][0]["evidence"][0][field] = "伪造"
|
||
with pytest.raises(比较未执行):
|
||
核对成对判断(material, report, writer_model="writer", judge_model="judge")
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-fec1f69eba18",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
given="独立固定样本、合成模型身份或原字引文；每个原用例仅使用自身参数",
|
||
when="调用该原例对应的公开预注册或成对判断核验",
|
||
then=[
|
||
"模型原始引文不带位置；代码按实际原文派生Unicode码点，原始输出不修改；具体内部sourceRef表示替换为start/end。"
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_模型只给原文代码派生正确码点并保留原始报告__fec1f6():
|
||
from copy import deepcopy
|
||
|
||
material = 组装成对材料("😀\r\n林澈守住城门。e\u0301", "另一侧正文。", ("叙事",))
|
||
raw = _成对报告(material, "tie")
|
||
raw["dimensions"][0]["evidence"][0] = {"side": "left", "quote": "林澈"}
|
||
original = deepcopy(raw)
|
||
result = 核对成对判断(material, raw, writer_model="writer", judge_model="judge")
|
||
quote = result["dimensions"][0]["evidence"][0]
|
||
assert quote == {"side": "left", "quote": "林澈", "start": 3, "end": 5}
|
||
assert material["left"]["text"][quote["start"] : quote["end"]] == quote["quote"]
|
||
assert raw == original and "start" not in raw["dimensions"][0]["evidence"][0]
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-9d8097665fa7",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
given="独立固定样本、合成模型身份或原字引文；每个原用例仅使用自身参数",
|
||
when="调用该原例对应的公开预注册或成对判断核验",
|
||
then=["唯一原字引文定位为确定码点区间，切片必须与引文逐字相同。"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_唯一引文保留原字码点位置__9d8097():
|
||
material = 组装成对材料("林澈守住城门。", "其他侧正文", ("叙事",))
|
||
raw = _成对报告(material, "tie")
|
||
raw["dimensions"][0]["evidence"][0] = {"side": "left", "quote": "城门"}
|
||
quote = 核对成对判断(material, raw, writer_model="writer", judge_model="judge")["dimensions"][
|
||
0
|
||
]["evidence"][0]
|
||
assert (quote["start"], quote["end"]) == (4, 6)
|
||
assert material["left"]["text"][quote["start"] : quote["end"]] == "城门"
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-23aa8e8b249f",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
given="独立固定样本、合成模型身份或原字引文；每个原用例仅使用自身参数",
|
||
when="调用该原例对应的公开预注册或成对判断核验",
|
||
then=["重复原字引文绑定首次出现，不需要模型指认出现序号。"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_重复引文按首次原样出现位置定位__23aa8e():
|
||
material = 组装成对材料("城门连着城门。", "其他侧正文", ("叙事",))
|
||
raw = _成对报告(material, "tie")
|
||
raw["dimensions"][0]["evidence"][0] = {"side": "left", "quote": "城门"}
|
||
quote = 核对成对判断(material, raw, writer_model="writer", judge_model="judge")["dimensions"][
|
||
0
|
||
]["evidence"][0]
|
||
assert (quote["start"], quote["end"], quote["quote"]) == (0, 2, "城门")
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-86ec39ac775c",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
given="独立固定样本、合成模型身份或原字引文；每个原用例仅使用自身参数",
|
||
when="调用该原例对应的公开预注册或成对判断核验",
|
||
then=["原文不存在的引文拒绝，原旧脚本错误码由B10公共错误PAIRWISE_QUOTE_NOT_FOUND承接。"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_原文不存在的引文精确拒绝__86ec39():
|
||
material = _成对输入()
|
||
raw = _成对报告(material)
|
||
raw["dimensions"][0]["evidence"][0] = {"side": "left", "quote": "不在原文"}
|
||
with pytest.raises(比较未执行) as error:
|
||
核对成对判断(material, raw, writer_model="writer", judge_model="judge")
|
||
assert error.value.错误码 == "PAIRWISE_QUOTE_NOT_FOUND"
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-0163585ebf22",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_report_must_not_reveal_arm_or_judge_target_role_coverage__016358():
|
||
material = _语义材料()
|
||
arm_leak = _语义输出(material)
|
||
arm_leak["arm"] = "outline_only"
|
||
with pytest.raises(审校错误):
|
||
核对语义检测(material, arm_leak)
|
||
|
||
target_role_judgment = _语义输出(material)
|
||
target_role_judgment["findings"] = [
|
||
{
|
||
"finding_id": "role-coverage",
|
||
"category": "missing_target_role_card",
|
||
"severity": "low",
|
||
"candidate_quote": material["candidate"],
|
||
"evidence_refs": ["source:outline"],
|
||
"message": "越权判断",
|
||
}
|
||
]
|
||
with pytest.raises(审校错误):
|
||
核对语义检测(material, target_role_judgment)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-ac50e47e39ba",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize(
|
||
"category",
|
||
[
|
||
"target_new_character_card_missing",
|
||
"new_role_without_card",
|
||
"invented_detector_category",
|
||
["frozen_context_gap"],
|
||
],
|
||
)
|
||
def test_semantic_alias_and_unknown_categories_fail_closed__ac50e4(category):
|
||
material = _语义材料()
|
||
output = _语义输出(material)
|
||
output["findings"] = [
|
||
{
|
||
"finding_id": "unknown-category",
|
||
"category": category,
|
||
"severity": "low",
|
||
"candidate_quote": material["candidate"],
|
||
"evidence_refs": ["source:outline"],
|
||
"message": "测试",
|
||
}
|
||
]
|
||
with pytest.raises(审校错误):
|
||
核对语义检测(material, output)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-0f5099de0182",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_registered_categories_are_accepted_and_detect_skill_assigns_target_coverage_to_eval__0f5099(): # noqa: E501
|
||
material = _语义材料()
|
||
output = _语义输出(material)
|
||
output["findings"] = [
|
||
{
|
||
"finding_id": "entity-state",
|
||
"category": "fact",
|
||
"severity": "medium",
|
||
"candidate_quote": material["candidate"],
|
||
"evidence_refs": ["source:outline"],
|
||
"message": "与冻结状态不一致",
|
||
}
|
||
]
|
||
report = 核对语义检测(material, output)
|
||
assert report["status"] == "passed"
|
||
assert report["findings"][0]["category"] == "fact"
|
||
assert "coverageFindings" not in report and "missing_target_role_card" not in str(report)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-087fe08ba218",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_nested_findings_reject_arm_and_card_manifest_leakage__087fe0():
|
||
material = _语义材料()
|
||
base = _语义输出(material)
|
||
base["findings"] = [
|
||
{
|
||
"finding_id": "entity-state",
|
||
"category": "fact",
|
||
"severity": "low",
|
||
"candidate_quote": material["candidate"],
|
||
"evidence_refs": ["source:outline"],
|
||
"message": "冻结状态核对",
|
||
}
|
||
]
|
||
leaking_finding = deepcopy(base)
|
||
leaking_finding["findings"][0]["arm"] = "outline_plus_cards"
|
||
leaking_claim = deepcopy(base)
|
||
leaking_claim["claims"][0]["cardManifest"] = {"count": 1}
|
||
for output in (leaking_finding, leaking_claim):
|
||
with pytest.raises(审校错误):
|
||
核对语义检测(material, output)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-6a2ca9bfcd0a",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_experiment_has_no_tools__6a2ca9():
|
||
policy = _角色目录().冻结(
|
||
"blind_judge",
|
||
provider="configured",
|
||
model="deepseek-v4.1-flash",
|
||
阶段="执行",
|
||
)
|
||
assert policy.角色 == "judge" and policy.工具 == ()
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-566cefcd2cd9",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_production_preflight_read_tools__566cef():
|
||
tools = ("read_chapter_text", "search_entities")
|
||
policy = _角色目录().冻结(
|
||
"judge",
|
||
provider="configured",
|
||
model="deepseek-v4.1-flash",
|
||
工具=tools,
|
||
阶段="执行",
|
||
)
|
||
assert policy.工具 == tools
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-e12d849e05bf",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_task_spec_carries_no_identity__e12d84():
|
||
model_input = _文学材料()
|
||
forbidden = {
|
||
"arm",
|
||
"realArm",
|
||
"evidenceStrategy",
|
||
"rawPath",
|
||
"rawDirectory",
|
||
"writerContextPath",
|
||
"runId",
|
||
"sampleId",
|
||
"reviewerInvocationId",
|
||
"candidateSha256",
|
||
"blindInputSha256",
|
||
"authorizationSnapshotId",
|
||
}
|
||
|
||
def keys(value):
|
||
if isinstance(value, dict):
|
||
return set(value) | {key for item in value.values() for key in keys(item)}
|
||
if isinstance(value, list):
|
||
return {key for item in value for key in keys(item)}
|
||
return set()
|
||
|
||
assert forbidden.isdisjoint(keys(model_input))
|
||
assert set(model_input["left"]) == {"text"} and set(model_input["right"]) == {"text"}
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-2bd29465a077",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_mode_controls_tool_allowlist_in_spec__2bd294():
|
||
policy = _角色目录().冻结(
|
||
"blind_judge",
|
||
provider="configured",
|
||
model="deepseek-v4.1-flash",
|
||
工具=(),
|
||
阶段="执行",
|
||
)
|
||
request = 模型请求(
|
||
"judge-call",
|
||
policy.provider,
|
||
policy.model,
|
||
"只评审匿名候选。",
|
||
"合成盲评输入",
|
||
{"type": "object"},
|
||
1024,
|
||
30,
|
||
工具=(),
|
||
)
|
||
assert policy.工具 == () and request.工具 == () and request.历史 == ()
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-145f4b68af3d",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_dispatch_failure_fails_closed__145f4b():
|
||
request = 模型请求(
|
||
"judge-call",
|
||
"configured",
|
||
"claude-opus-4-8[1M]",
|
||
"只评审匿名候选。",
|
||
"合成盲评输入",
|
||
{"type": "object"},
|
||
1024,
|
||
30,
|
||
)
|
||
failed = 模型结果("failed", "", None, None, 失败码="synthetic-dispatch-failure")
|
||
with pytest.raises(模型协议错误) as error:
|
||
校验模型输出(request, failed)
|
||
assert error.value.错误码 == "MODEL_PROTOCOL_INVALID"
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-86fa000f251e",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_assembled_input_passes_official_validator__86fa00():
|
||
material = 组装文学材料(
|
||
"甲候选正文：茧撕开舱门。",
|
||
"乙候选正文：林深盯着深渊。",
|
||
{
|
||
"scenario": "battle",
|
||
"sources": [
|
||
{
|
||
"source_id": "outline",
|
||
"kind": "fine_outline",
|
||
"text": "硬约束甲；硬约束乙；拍一。",
|
||
},
|
||
{"source_id": "history", "kind": "historical_prose", "text": "林深盯着城门。"},
|
||
],
|
||
"assertions": [
|
||
{"statement_id": "fact-one", "text": "新事实"},
|
||
{"statement_id": "fact-two", "text": "林深仍在场"},
|
||
],
|
||
"constraints": [
|
||
{"statement_id": "hard-one", "text": "硬约束甲"},
|
||
{"statement_id": "hard-two", "text": "硬约束乙"},
|
||
],
|
||
},
|
||
)
|
||
checked = _核对文学(material, _文学输出(material))
|
||
assert sorted((material["left"]["text"], material["right"]["text"])) == sorted(
|
||
("甲候选正文：茧撕开舱门。", "乙候选正文：林深盯着深渊。")
|
||
)
|
||
assert len(checked["assertion_verdicts"]) == 4
|
||
assert len(checked["constraint_verdicts"]) == 4
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-968a24cafe30",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_every_dimension_requires_evidence__968a24():
|
||
material = _文学材料()
|
||
assert len(_核对文学(material, _文学输出(material))["candidate_scores"]) == 2
|
||
missing_evidence = _文学输出(material)
|
||
missing_evidence["candidate_scores"][0]["scores"][0]["evidence"] = []
|
||
with pytest.raises(比较未执行):
|
||
_核对文学(material, missing_evidence)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-73f1f4e5ae77",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_prose_dimensions_are_rejected__73f1f4():
|
||
material = _文学材料()
|
||
scores = _文学输出(material)
|
||
scores["candidate_scores"][0]["scores"].append(
|
||
{
|
||
"dimension": "style_fit",
|
||
"score": 5,
|
||
"rationale": "不应出现",
|
||
"evidence": [_文学引文(material, "left")],
|
||
"inferences": [],
|
||
}
|
||
)
|
||
with pytest.raises(比较未执行):
|
||
_核对文学(material, scores)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-bf000c7c899a",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_profile_and_score_range_are_checked__bf000c():
|
||
material = _文学材料()
|
||
report = _文学输出(material, 6)
|
||
assert all(
|
||
score["score"] == 6
|
||
for candidate in _核对文学(material, report)["candidate_scores"]
|
||
for score in candidate["scores"]
|
||
)
|
||
bad_profile = deepcopy(material)
|
||
bad_profile["rubric"]["version"] = "quality_gate"
|
||
with pytest.raises(比较未执行):
|
||
_核对文学(bad_profile, report)
|
||
with pytest.raises(比较未执行):
|
||
_核对文学(material, _文学输出(material, 10.5))
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-e75ca4811e14",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_large_reviewer_gap_warns__e75ca4():
|
||
first = _文学评审(1, score=4)
|
||
dimension = "style_consistency"
|
||
second = _文学评审(2, score=4, change=(dimension, 5))
|
||
result = 仲裁文学评分([first, second])
|
||
assert result["status"] == "needs_third_reviewer"
|
||
assert result["score_gaps"] == [["blind-1", dimension]]
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-a204c4f941c4",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_missing_dimension_fails_stability_closed__a204c4():
|
||
first, second = _文学评审(1), _文学评审(2)
|
||
second["judgment"]["candidate_scores"][0]["scores"].pop()
|
||
result = 仲裁文学评分([first, second])
|
||
assert result == {"status": "invalid_report", "execution_verified": False}
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-98cb753cc286",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_batch_report_requires_exact_candidates_and_distinct_judge_identity__98cb75():
|
||
first, second = _文学评审(1), _文学评审(2)
|
||
assert 仲裁文学评分([first, second])["status"] == "stable_report"
|
||
duplicate = deepcopy(second)
|
||
duplicate["judge_ref"] = first["judge_ref"]
|
||
assert 仲裁文学评分([first, duplicate])["status"] == "invalid_report"
|
||
wrong_candidate = deepcopy(second)
|
||
wrong_candidate["sides"]["left"] = "blind-3"
|
||
assert 仲裁文学评分([first, wrong_candidate])["status"] == "invalid_report"
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-8381e1e91e63",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_nested_evaluation_and_score_reject_arm_card_manifest_leakage__8381e1():
|
||
material = _文学材料()
|
||
arm_leak = _文学输出(material)
|
||
arm_leak["candidate_scores"][0]["arm"] = "outline_only"
|
||
card_leak = _文学输出(material)
|
||
card_leak["candidate_scores"][0]["scores"][0]["cardManifest"] = {"count": 1}
|
||
for output in (arm_leak, card_leak):
|
||
with pytest.raises(比较未执行):
|
||
_核对文学(material, output)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-8ada90c788e8",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_adapter_sends_anonymous_common_input_and_binds_report__8ada90():
|
||
material = 组装文学材料(
|
||
"林澈守住城门。",
|
||
"林澈掩住门缝。",
|
||
_文学材料()["basis"],
|
||
)
|
||
report = _核对文学(material, _文学输出(material))
|
||
assert set(material) == {"left", "right", "dimensions", "rubric", "basis"}
|
||
assert all(set(material[side]) == {"text"} for side in ("left", "right"))
|
||
assert tuple(material["dimensions"]) == (
|
||
"setting_entity_fidelity",
|
||
"fine_outline_fidelity",
|
||
"style_consistency",
|
||
"narrative_tension",
|
||
"prose_readability",
|
||
)
|
||
assert report["basis_hash"] == 固定哈希(material["basis"])
|
||
assert report["rubric_hash"] == 固定哈希(material["rubric"])
|
||
serialized = json.dumps(material, ensure_ascii=False)
|
||
for forbidden in (
|
||
"runId",
|
||
"sampleId",
|
||
"reviewerInvocationId",
|
||
"candidateSha256",
|
||
"blindInputSha256",
|
||
"authorizationSnapshotId",
|
||
):
|
||
assert forbidden not in serialized
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-c800224586bd",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize(
|
||
("field", "value"),
|
||
[("scenario", "romance"), ("rubricPolicyVersion", "writer-replay-rubric-v1")],
|
||
)
|
||
def test_unregistered_scenario_and_policy_drift_fail_before_model_call__c80022(field, value):
|
||
calls = []
|
||
if field == "scenario":
|
||
with pytest.raises(评测错误):
|
||
_文学材料(scenario=value)
|
||
else:
|
||
material = _文学材料()
|
||
material["rubric"]["version"] = value
|
||
with pytest.raises(比较未执行):
|
||
_核对文学(material, _文学输出(_文学材料()))
|
||
assert calls == []
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-43c633c527fb",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_judge_inference_refs_bind_to_current_dimension_or_stable_id__43c633():
|
||
material = _文学材料()
|
||
draft = _文学输出(material)
|
||
for candidate in draft["candidate_scores"]:
|
||
for score in candidate["scores"]:
|
||
score["inferences"] = [
|
||
{
|
||
"claim": "推断只使用当前维度已登记证据。",
|
||
"refs": [score["evidence"][0]["evidence_id"]],
|
||
}
|
||
]
|
||
assert _核对文学(material, draft)["candidate_scores"]
|
||
invalid = deepcopy(draft)
|
||
invalid["assertion_verdicts"][0]["inferences"] = [
|
||
{"claim": "字段名不是稳定证据身份。", "refs": ["assertionId"]}
|
||
]
|
||
with pytest.raises(比较未执行):
|
||
_核对文学(material, invalid)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-5db0211546d6",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize(
|
||
"version", ["blind-judge-report-v3", "blind-judge-report-v2", "blind-judge-report-v1"]
|
||
)
|
||
def test_old_v3_v2_and_v1_reports_fail_closed__5db021(version):
|
||
material = _文学材料()
|
||
report = _文学输出(material)
|
||
report["schemaVersion"] = version
|
||
with pytest.raises(比较未执行):
|
||
_核对文学(material, report)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-8537f235f675",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_input_rejects_real_arm_and_stale_candidate_hash__8537f2():
|
||
material = _文学材料()
|
||
leaked = deepcopy(material)
|
||
leaked["arm"] = "A"
|
||
stale = deepcopy(material)
|
||
stale["left"]["candidateSha256"] = "sha256:" + "f" * 64
|
||
for invalid in (leaked, stale):
|
||
with pytest.raises(比较未执行):
|
||
_核对文学(invalid, _文学输出(material))
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-d94345b3c760",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("changed", ["raw_source", "future", "nested_extra"])
|
||
def test_fine_outline_and_oracle_freeze_validation_remain_fail_closed__d94345(changed):
|
||
basis = deepcopy(_文学材料()["basis"])
|
||
if changed == "raw_source":
|
||
basis["sources"][0]["source_id"] = "/private/tmp/raw/outline.json"
|
||
elif changed == "future":
|
||
basis["sources"][1]["chapterEnd"] = 489
|
||
else:
|
||
basis["constraints"][0]["debug"] = True
|
||
with pytest.raises(评测错误):
|
||
组装文学材料(文学甲, 文学乙, basis)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-b2a1b2e61fa7",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_model_must_cover_full_verdict_cartesian_product__b2a1b2():
|
||
material = _文学材料()
|
||
draft = _文学输出(material)
|
||
draft["assertion_verdicts"].pop()
|
||
with pytest.raises(比较未执行) as error:
|
||
_核对文学(material, draft)
|
||
assert error.value.错误码 == "PAIRWISE_NOT_EXECUTED"
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-9a7e59fe1698",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_reviewer_identity_reuse_fails_before_call__9a7e59():
|
||
calls = []
|
||
first, reused = _文学评审(1), _文学评审(2)
|
||
reused["judge_ref"] = first["judge_ref"]
|
||
result = 仲裁文学评分([first, reused])
|
||
assert result["status"] == "invalid_report" and calls == []
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-176c97a3b2d9",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_reviewers_with_different_scenarios_fail_before_call__176c97():
|
||
calls = []
|
||
first, second = _文学评审(1), _文学评审(2)
|
||
second["scope_hash"] = 固定哈希("battle")
|
||
result = 仲裁文学评分([first, second])
|
||
assert result["status"] == "invalid_report" and calls == []
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-3ee55ac52985",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_score_instability_triggers_exactly_one_third_reviewer__3ee55a():
|
||
dimension = "setting_entity_fidelity"
|
||
first = _文学评审(1, score=8.0)
|
||
second = _文学评审(2, score=8.0, change=(dimension, 6.5))
|
||
pending = 仲裁文学评分([first, second])
|
||
assert pending["status"] == "needs_third_reviewer"
|
||
third = _文学评审(3, score=8.0)
|
||
result = 仲裁文学评分([first, second, third])
|
||
assert result["status"] == "adjudicated_report"
|
||
assert len({r["judge_ref"] for r in (first, second, third)}) == 3
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-27a56a41221e",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_ellipsis_in_free_text_not_killed_but_real_traversal_blocked__27a56a():
|
||
material = 组装成对材料("林澈说：等等...别走。", "林澈说：慢着...且听我说完。", ("叙事",))
|
||
report = _成对报告(material, "tie")
|
||
report["dimensions"][0]["evidence"] = [
|
||
{"side": "left", "quote": "等等..."},
|
||
{"side": "right", "quote": "慢着..."},
|
||
]
|
||
checked = 核对成对判断(material, report, writer_model="writer-model", judge_model="judge-model")
|
||
assert checked["dimensions"][0]["evidence"][0]["quote"] == "等等..."
|
||
basis = deepcopy(_文学材料()["basis"])
|
||
basis["sources"][0]["source_id"] = "a/../b"
|
||
with pytest.raises(评测错误):
|
||
组装文学材料(文学甲, 文学乙, basis)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-bb1f26f796ff",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_three_reviewers_pairwise_unstable_fail_closed_without_winner__bb1f26():
|
||
reviews = [
|
||
_文学评审(1, score=7.0),
|
||
_文学评审(2, score=8.0),
|
||
_文学评审(3, score=9.0),
|
||
]
|
||
result = 仲裁文学评分(reviews)
|
||
assert result["status"] == "invalid_unstable"
|
||
assert not result["execution_verified"] and len(reviews) == 3
|
||
assert "scores" not in result and "winner" not in result and "candidate_scores" not in result
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-d22b57090ab1",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_pair_must_be_independent_blind_and_reverse_ordered__d22b57():
|
||
first, second = _文学评审(1), _文学评审(2)
|
||
assert 仲裁文学评分([first, second])["status"] == "stable_report"
|
||
|
||
same_order = deepcopy(second)
|
||
same_order["sides"] = first["sides"]
|
||
assert 仲裁文学评分([first, same_order])["status"] == "invalid_report"
|
||
|
||
different_candidate = deepcopy(second)
|
||
different_candidate["sides"]["left"] = "blind-3"
|
||
assert 仲裁文学评分([first, different_candidate])["status"] == "invalid_report"
|
||
|
||
leaked = deepcopy(second)
|
||
leaked["arm"] = "C"
|
||
assert 仲裁文学评分([first, leaked])["status"] == "invalid_report"
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-125ee8fdcfd6",
|
||
environment="离线新版匿名、引文、语义与文学仲裁公开合同；不认证真实派发或外部模型",
|
||
when="匿名分配样本、校验逐维引文并处理评委分歧。",
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_deblind_uses_preregistered_mapping_not_position__125ee8():
|
||
first, second = _文学评审(1), _文学评审(2)
|
||
result = 仲裁文学评分([first, second])
|
||
assert result["status"] == "stable_report"
|
||
assert {score["candidate_id"] for score in result["scores"]} == {"blind-1", "blind-2"}
|
||
assert len(result["scores"]) == 10
|