9e6f1c4481
- src/muse 新版全模块（装配/共享/上下文/任务运行/作品规划/故事世界/正文写作/审校修订/知识方法/作者经验/效果评测/交付连载/资料研究/正式变更/元数据/接入/基础设施/编排）+ 测试树（单元/契约/集成/架构/迁移/端到端/夹具） - 129 项功能全部实现与自动验证（功能覆盖.json/矩阵），含 W31 补齐的规则与代价/节奏安排/伏笔与承诺 - 旧实现按处置清单退出（702 条中 324 删，保护合同与未迁移条目留存有据）；web/app.py 旧工作台退役，新工作台为唯一写入口 - 数据库/旧库迁移：真实旧库内容批次迁移链（端点守卫/PG作品正文映射/质量资产缺省投影） - 运行手册 docs/运行手册.md；W30 本机服务阶段一已运行（infra PG 为正式内容权威） - R2 执行证据与私有运行材料在 .agents.local/改造/R2-20260909/（不入库）
451 lines
19 KiB
Python
451 lines
19 KiB
Python
"""真实S02合成HTTP交付验证文学评分与条件第三；不证明文学质量。"""
|
||
|
||
import json
|
||
|
||
import pytest
|
||
import test_回放资料封存 as 资料测试
|
||
import test_评测执行与失败收敛 as 执行测试
|
||
|
||
from muse.效果评测.文学评分 import 维度
|
||
|
||
pytestmark = pytest.mark.数据库
|
||
执行环境 = 执行测试.执行环境
|
||
资料环境 = 资料测试.资料环境
|
||
生成环境 = 资料测试.生成环境
|
||
|
||
依据 = {
|
||
"scenario": "turning_point",
|
||
"sources": [
|
||
{"source_id": "outline", "kind": "fine_outline", "text": "角色守住城门。"},
|
||
{"source_id": "history", "kind": "historical_prose", "text": "先前角色静静守门。"},
|
||
],
|
||
"assertions": [{"statement_id": "guard", "text": "角色是守卫。"}],
|
||
"constraints": [{"statement_id": "stay", "text": "角色不能离开城门。"}],
|
||
}
|
||
|
||
|
||
def _输出(material, script):
|
||
def quote(side, kind="candidate", source=None):
|
||
return {
|
||
"evidence_id": "candidate" if source is None else "source",
|
||
"source_type": kind,
|
||
"source_id": side
|
||
if source is None
|
||
else source.get("source_id", source.get("statement_id")),
|
||
"quote": material[side]["text"] if source is None else source["text"],
|
||
}
|
||
|
||
cards = []
|
||
for side in ("left", "right"):
|
||
scores = []
|
||
for dim in material["dimensions"]:
|
||
evidence = [quote(side)]
|
||
if dim != "prose_readability":
|
||
kind = "historical_prose" if dim == "style_consistency" else "fine_outline"
|
||
source = next(s for s in material["basis"]["sources"] if s["kind"] == kind)
|
||
evidence.append(quote(side, kind, source))
|
||
scores.append(
|
||
{
|
||
"dimension": dim,
|
||
"score": {
|
||
"score_gap": 6.0,
|
||
"unstable": 4.0,
|
||
"calibration_gap": 7.0,
|
||
"calibration_low": 7.5,
|
||
"calibration_high": 8.5,
|
||
}.get(script, 8.0),
|
||
"rationale": "合成评分依据候选与共同背景。",
|
||
"evidence": evidence,
|
||
"inferences": [],
|
||
}
|
||
)
|
||
cards.append({"side": side, "scores": scores})
|
||
result = {
|
||
"choice": "tie",
|
||
"rationale": "合成样本维持平局。",
|
||
"candidate_scores": cards,
|
||
"preferences": [
|
||
{"dimension": d, "choice": "tie", "rationale": "两侧合成表现相近。"}
|
||
for d in material["dimensions"]
|
||
],
|
||
}
|
||
for field, kind, key in (
|
||
("assertion_verdicts", "oracle_assertion", "assertions"),
|
||
("constraint_verdicts", "preregistered_constraint", "constraints"),
|
||
):
|
||
result[field] = [
|
||
{
|
||
"side": side,
|
||
"statement_id": statement["statement_id"],
|
||
"verdict": "fail" if script == "verdict_gap" else "pass",
|
||
"rationale": "按共同命题核对合成文本。",
|
||
"inferences": [],
|
||
"evidence": [quote(side), quote(side, kind, statement)],
|
||
}
|
||
for side in ("left", "right")
|
||
for statement in material["basis"][key]
|
||
]
|
||
if script == "shuffle":
|
||
result["candidate_scores"].reverse()
|
||
for card in result["candidate_scores"]:
|
||
card["scores"].reverse()
|
||
for field in ("preferences", "assertion_verdicts", "constraint_verdicts"):
|
||
result[field].reverse()
|
||
if script == "literary_bad_quote":
|
||
result["candidate_scores"][0]["scores"][0]["evidence"][0]["quote"] = "不存在的原文"
|
||
if script == "literary_duplicate":
|
||
result["candidate_scores"][0]["scores"][0]["dimension"] = result["candidate_scores"][0][
|
||
"scores"
|
||
][1]["dimension"]
|
||
return result
|
||
|
||
|
||
参数 = {
|
||
"judges": 2,
|
||
"arbitrator": True,
|
||
"judge_output": _输出,
|
||
"judging_basis": 依据,
|
||
"dimensions": 维度,
|
||
"calls": 5,
|
||
}
|
||
|
||
|
||
def _读(env):
|
||
return env["app"].要求评测().读取执行工作面(env["actor"], env["exp"]["experiment_id"])
|
||
|
||
|
||
def _推进(env):
|
||
return env["app"].要求评测().推进实验(env["actor"], env["exp"]["experiment_id"])
|
||
|
||
|
||
def _双评(env, script="ok"):
|
||
start = 执行测试._启动执行(env)
|
||
assert len(start["units"]) == 5
|
||
assert sum(u["task_id"] is not None for u in start["units"]) == 2
|
||
执行测试._运行就绪(env)
|
||
_推进(env)
|
||
env["scripted"].extend(["ok", script])
|
||
执行测试._运行就绪(env)
|
||
assert len(env["received"]) == 4
|
||
|
||
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_稳定双评不建第三任务且共同依据不泄露答案__25d001(执行环境):
|
||
env = 执行环境
|
||
_双评(env)
|
||
_推进(env)
|
||
执行测试._运行就绪(env)
|
||
result = _读(env)
|
||
assert result["state"] == "completed"
|
||
unneeded = [u for u in result["units"] if u["state"] == "not_required"]
|
||
assert len(unneeded) == 1 and unneeded[0]["task_id"] is None
|
||
assert len(env["received"]) == 4
|
||
inputs = [json.loads(r["input"]) for r in env["received"]]
|
||
assert all("basis" not in r for r in inputs[:2])
|
||
assert inputs[2]["left"] == inputs[3]["right"]
|
||
assert inputs[2]["basis"] == inputs[3]["basis"] == 依据
|
||
assert all("ORACLE-SECRET" not in r["input"] for r in env["received"])
|
||
assert all(not r.get("tools") and not r.get("previous_response_id") for r in env["received"])
|
||
report = env["app"].要求评测().读取实验报告(env["actor"], env["exp"]["experiment_id"])
|
||
assert report["coverage"]["compared"] == 1
|
||
literary = report["samples"][0]["literary"]["control:treatment"]
|
||
assert literary["status"] == "stable_report" and literary["execution_verified"]
|
||
assert report["literary_quality"]["metrics"] is None
|
||
|
||
|
||
@pytest.mark.parametrize("script", ["score_gap", "verdict_gap"])
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_分歧只追加预注册第三评且保留原始分歧__25d002(执行环境, script):
|
||
env = 执行环境
|
||
_双评(env, script)
|
||
_推进(env)
|
||
ready = _读(env)
|
||
assert sum(u["task_id"] is not None for u in ready["units"]) == 5
|
||
执行测试._运行就绪(env)
|
||
_推进(env)
|
||
执行测试._运行就绪(env)
|
||
assert len(env["received"]) == 5
|
||
original, third = (json.loads(env["received"][i]["input"]) for i in (2, 4))
|
||
assert original == third # 第三评不读取前两份报告或模型身份。
|
||
report = env["app"].要求评测().读取实验报告(env["actor"], env["exp"]["experiment_id"])
|
||
literary = report["samples"][0]["literary"]["control:treatment"]
|
||
assert literary["status"] == "adjudicated_report" and literary["execution_verified"]
|
||
assert literary["score_gaps"] if script == "score_gap" else literary["verdict_gaps"]
|
||
assert report["cost"]["sent_calls"] == 5
|
||
assert report["literary_quality"]["calibration"] == "unverified"
|
||
|
||
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_初评费用不明时不启动第三也不删失败样本__25d003(执行环境):
|
||
env = 执行环境
|
||
执行测试._启动执行(env)
|
||
执行测试._运行就绪(env)
|
||
_推进(env)
|
||
env["scripted"].extend(["ok", "unknown_cost"])
|
||
执行测试._运行就绪(env, allow_failure=True)
|
||
_推进(env)
|
||
result = _读(env)
|
||
assert len(env["received"]) == 4 and result["state"] == "reconciling"
|
||
assert sum(u["task_id"] is not None for u in result["units"]) == 4
|
||
report = env["app"].要求评测().读取实验报告(env["actor"], env["exp"]["experiment_id"])
|
||
assert report["coverage"] == {"samples": 1, "generated": 1, "compared": 0}
|
||
assert report["cost"]["total_usd"] is None
|
||
|
||
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_实际乱序交付归一同时保留S02原始输出__25d004(执行环境):
|
||
from muse.正式变更.接口 import 固定哈希
|
||
|
||
env = 执行环境
|
||
_双评(env, "shuffle")
|
||
_推进(env)
|
||
work = _读(env)
|
||
reversed_output = next(
|
||
u
|
||
for u in work["units"]
|
||
if u["kind"] == "comparison"
|
||
and u["output"]
|
||
and u["output"]["candidate_scores"][0]["side"] == "right"
|
||
)
|
||
evidence = reversed_output["evidence"]
|
||
assert evidence["structured_output_hash"] == 固定哈希(reversed_output["output"])
|
||
normalized = evidence["judgment"]["literary"]
|
||
assert [c["side"] for c in normalized["candidate_scores"]] == ["left", "right"]
|
||
assert all(
|
||
tuple(s["dimension"] for s in c["scores"]) == 维度 for c in normalized["candidate_scores"]
|
||
)
|
||
assert tuple(r["dimension"] for r in evidence["judgment"]["report"]["dimensions"]) == 维度
|
||
report = env["app"].要求评测().读取实验报告(env["actor"], env["exp"]["experiment_id"])
|
||
assert report["samples"][0]["literary"]["control:treatment"]["status"] == "stable_report"
|
||
assert report["report_hash"] == 固定哈希(
|
||
{k: v for k, v in report.items() if k != "report_hash"}
|
||
)
|
||
|
||
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_第三仍无稳定对保留全部评审且不产生合格分数__25d005(执行环境):
|
||
env = 执行环境
|
||
_双评(env, "score_gap")
|
||
_推进(env)
|
||
env["scripted"].append("unstable")
|
||
执行测试._运行就绪(env)
|
||
report = env["app"].要求评测().读取实验报告(env["actor"], env["exp"]["experiment_id"])
|
||
sample = report["samples"][0]
|
||
assert sample["literary"]["control:treatment"]["status"] == "invalid_unstable"
|
||
assert sample["literary"]["control:treatment"]["scores"] == []
|
||
assert sample["outcome"] == "incomplete" and report["outcomes"]["tie"] == 0
|
||
assert len(sample["decisions"]) == 3
|
||
assert report["literary_quality"]["metrics"] is None
|
||
assert len(env["received"]) == 5
|
||
|
||
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_第三决议不可覆盖且局部重签不能伪造必要性__25d006(执行环境, monkeypatch):
|
||
import psycopg
|
||
|
||
from muse.效果评测.存储 import 评测存储
|
||
from muse.效果评测.接口 import 评测错误
|
||
|
||
env = 执行环境
|
||
_双评(env)
|
||
work = _推进(env)
|
||
uid = next(u["unit_id"] for u in work["units"] if u["state"] == "not_required")
|
||
with pytest.raises(psycopg.Error), env["pool"].连接() as conn:
|
||
conn.execute(
|
||
"UPDATE evaluation.muse_evaluation_condition SET decision=decision WHERE unit_id=%s",
|
||
(uid,),
|
||
)
|
||
original = 评测存储.读取条件
|
||
|
||
def forged(self, unit_id):
|
||
row = original(self, unit_id)
|
||
return {**row, "disposition": "required"} if row else row
|
||
|
||
monkeypatch.setattr(评测存储, "读取条件", forged)
|
||
with pytest.raises(评测错误, match="重算结果"):
|
||
_推进(env)
|
||
with pytest.raises(评测错误, match="重算结果"):
|
||
_读(env)
|
||
assert len(env["received"]) == 4
|
||
|
||
|
||
@pytest.mark.parametrize("执行环境", [{**参数, "calls": 4}], indirect=True)
|
||
def test_总预算未覆盖候补时拒绝启动且不留孤儿任务__25d007(执行环境):
|
||
from muse.效果评测.接口 import 评测错误
|
||
|
||
env = 执行环境
|
||
with pytest.raises(评测错误, match="不足"):
|
||
执行测试._启动执行(env)
|
||
assert _读(env)["state"] == "registered"
|
||
assert not env["received"]
|
||
with env["pool"].连接(只读=True) as conn:
|
||
assert conn.execute("SELECT count(*) FROM evaluation.muse_task").fetchone()[0] == 0
|
||
|
||
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_评分标准变化拒绝新发送但已完成报告仍按原版本回查__25d008(执行环境, monkeypatch):
|
||
import muse.效果评测.文学执行 as adapter
|
||
|
||
env = 执行环境
|
||
_双评(env, "score_gap")
|
||
original = adapter.读取评分标准
|
||
monkeypatch.setattr(
|
||
adapter, "读取评分标准", lambda scenario: {**original(scenario), "common_dimensions": []}
|
||
)
|
||
assert len([u for u in _读(env)["units"] if u["output"]]) == 4
|
||
_推进(env)
|
||
执行测试._运行就绪(env, allow_failure=True)
|
||
assert len(env["received"]) == 4
|
||
assert _读(env)["state"] == "partial_failure"
|
||
|
||
|
||
@pytest.mark.parametrize("执行环境", [{**参数, "calls": 10}], indirect=True)
|
||
def test_文学坏引文沿原S02调用链有界纠正后再决定第三__25d009(执行环境):
|
||
env = 执行环境
|
||
执行测试._启动执行(env)
|
||
执行测试._运行就绪(env)
|
||
_推进(env)
|
||
env["scripted"].extend(["literary_bad_quote", "ok", "ok"])
|
||
执行测试._运行就绪(env)
|
||
_推进(env)
|
||
work = _读(env)
|
||
assert work["state"] == "completed"
|
||
assert sum(len(u["rejections"]) for u in work["units"]) == 1
|
||
assert len(env["received"]) == 5
|
||
corrections = [json.loads(r["input"]) for r in env["received"] if '"correction"' in r["input"]]
|
||
assert len(corrections) == 1
|
||
assert "candidate_scores" in corrections[0]["correction"]["previousDraft"]
|
||
assert corrections[0]["basis"] == 依据
|
||
|
||
|
||
def 创建ABC实验(env, 资料环境, *, detector=False, calls=12, audit=None):
|
||
from dataclasses import replace
|
||
|
||
from muse.任务运行.接口 import 角色策略目录
|
||
from muse.效果评测.接口 import 回放数据集发布, 实验请求
|
||
|
||
env = dict(env)
|
||
_, _, author, publisher, source_req = 资料环境
|
||
raw = source_req.model_dump(mode="json")
|
||
if audit is not None:
|
||
raw["samples"][0]["audit"] = audit
|
||
raw["samples"][0]["judging"] = {k: 依据[k] for k in ("scenario", "assertions", "constraints")}
|
||
receipt = publisher.发布回放数据集(author, 回放数据集发布.model_validate(raw))
|
||
frozen = env["exp"]["conditions"]
|
||
request = 实验请求.model_validate(
|
||
{
|
||
"dataset_version_id": receipt["version_id"],
|
||
"dataset_hash": receipt["public_hash"],
|
||
"split": "holdout",
|
||
"generator_role": "writer",
|
||
"target": {
|
||
"kind": "code",
|
||
"target_ref": "B09.writer-replay",
|
||
"version": 角色策略目录.从发布包().资源发布身份,
|
||
"content_hash": 角色策略目录.从发布包().资源发布身份,
|
||
},
|
||
"generator": {k: frozen["generator"][k] for k in ("config_id", "version")},
|
||
"judges": [{k: p[k] for k in ("config_id", "version")} for p in frozen["judges"]],
|
||
"arbitrator": {k: frozen["arbitrator"][k] for k in ("config_id", "version")},
|
||
"comparison_profile": "writer_rubric",
|
||
"detector": (
|
||
{k: frozen["detector"][k] for k in ("config_id", "version")} if detector else None
|
||
),
|
||
"dimensions": list(维度),
|
||
"arms": ["A", "B", "C"],
|
||
"max_cost_usd": "12",
|
||
"max_calls_per_sample": calls,
|
||
}
|
||
)
|
||
env["actor"] = replace(env["actor"], 作者=author.作者)
|
||
svc = env["app"].要求评测()
|
||
env["exp"] = svc.创建实验(env["actor"], "literary-abc", request)
|
||
return {**env, "dataset_receipt": receipt, "request": request}
|
||
|
||
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_真实ABC封存提供共同文学依据且目标正文仍只留oracle__25d00a(执行环境, 资料环境):
|
||
env = 创建ABC实验(执行环境, 资料环境)
|
||
svc = env["app"].要求评测()
|
||
receipt = env["dataset_receipt"]
|
||
started = 执行测试._启动执行(env)
|
||
assert len(started["units"]) == 12
|
||
执行测试._运行就绪(env)
|
||
_推进(env)
|
||
执行测试._运行就绪(env)
|
||
_推进(env)
|
||
report = svc.读取实验报告(env["actor"], env["exp"]["experiment_id"])
|
||
assert report["state"] == "completed" and report["primary_pair"] == ["A", "C"]
|
||
assert len(env["received"]) == 9
|
||
assert set(report["samples"][0]["literary"]) == {"A:B", "A:C", "B:C"}
|
||
assert all(x["status"] == "stable_report" for x in report["samples"][0]["literary"].values())
|
||
inputs = [json.loads(r["input"]) for r in env["received"]]
|
||
common = inputs[3]["basis"]
|
||
assert all(r["basis"] == common for r in inputs[3:])
|
||
assert {s["kind"] for s in common["sources"]} == {"fine_outline", "historical_prose"}
|
||
assert any(s["text"] == "林深回到渡口，雨渐渐小了。" for s in common["sources"])
|
||
assert "ORACLE-TARGET-PRIVATE" not in json.dumps(env["received"], ensure_ascii=False)
|
||
public = svc.读取数据集(env["actor"], receipt["version_id"])
|
||
assert "judging_basis" not in json.dumps(public, ensure_ascii=False)
|
||
with env["pool"].连接(只读=True) as conn:
|
||
answer = conn.execute(
|
||
"SELECT answers FROM oracle.muse_dataset_answers WHERE version_id=%s",
|
||
(receipt["version_id"],),
|
||
).fetchone()[0][0]["answer"]
|
||
assert answer["target_text"].startswith("ORACLE-TARGET-PRIVATE")
|
||
assert answer["judging_basis"] == common
|
||
|
||
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_停止后的初评分歧不再创建第三任务__25d00b(执行环境):
|
||
from muse.效果评测.接口 import 评测错误
|
||
|
||
env = 执行环境
|
||
_双评(env, "score_gap")
|
||
env["app"].要求评测().取消实验(env["actor"], env["exp"]["experiment_id"], "stop-before-third")
|
||
with pytest.raises(评测错误, match="停止"):
|
||
_推进(env)
|
||
assert _读(env)["state"] == "stopped"
|
||
assert sum(u["task_id"] is not None for u in _读(env)["units"]) == 4
|
||
assert len(env["received"]) == 4
|
||
|
||
|
||
@pytest.mark.parametrize("执行环境", [{**参数, "calls": 10}], indirect=True)
|
||
def test_实际重复维度不能借引文纠正继续调用或形成报告__25d00c(执行环境):
|
||
env = 执行环境
|
||
执行测试._启动执行(env)
|
||
执行测试._运行就绪(env)
|
||
_推进(env)
|
||
env["scripted"].extend(["literary_duplicate", "ok"])
|
||
执行测试._运行就绪(env, allow_failure=True)
|
||
_推进(env)
|
||
work = _读(env)
|
||
assert work["state"] == "partial_failure" and len(env["received"]) == 4
|
||
rejected = [r for u in work["units"] for r in u["rejections"]]
|
||
assert len(rejected) == 1 and rejected[0]["failure_code"] == "PAIRWISE_NOT_EXECUTED"
|
||
report = env["app"].要求评测().读取实验报告(env["actor"], env["exp"]["experiment_id"])
|
||
assert report["samples"][0]["literary"]["control:treatment"]["status"] == "incomplete"
|
||
|
||
|
||
def test_新增可选文学字段不改变已有普通实验的命令身份__25d00e(执行环境):
|
||
from muse.正式变更.接口 import 固定哈希
|
||
|
||
env = 执行环境
|
||
# 旧客户端没有这两个可选字段；原存储哈希必须能被同一请求命中。
|
||
legacy_request = env["request"].model_dump(mode="json")
|
||
legacy_request.pop("comparison_profile")
|
||
legacy_request.pop("arbitrator")
|
||
legacy_request.pop("calibration_policy")
|
||
legacy_request.pop("evaluation_goal")
|
||
legacy_request.pop("calibration_use")
|
||
legacy_request.pop("effect_policy")
|
||
with env["pool"].连接(只读=True) as conn:
|
||
stored = conn.execute(
|
||
"SELECT request_hash FROM evaluation.muse_experiment WHERE experiment_id=%s",
|
||
(env["exp"]["experiment_id"],),
|
||
).fetchone()[0]
|
||
assert stored == 固定哈希(legacy_request)
|
||
repeated = env["app"].要求评测().创建实验(env["actor"], "runtime-fixture", env["request"])
|
||
assert repeated["experiment_id"] == env["exp"]["experiment_id"]
|
||
assert not env["received"]
|