cf0f4fb985
接续 88dd570，保存 W20–W24 已实现的共享接口、业务入口、迁移、工作台、测试与文档。 W20/W22/W23 保持 in_progress，W21/W24 保持 verified；此提交不宣称方法或规则正式启用、多轮返修、真实角色评测完成。 W25 新增实验、标定、逐调用交付与角色执行及其迁移/测试/索引留在实施工作树，原有私人和旧实现保留项不纳入。 验证：离线 571、前端 33 通过；PG 469 项通过、2 项浏览器未启用，2 项误带入的 W25 用例已移出本提交；最终任务与交付边界 37 项通过。make 检查、最终类型、84 项资源及 diff 检查通过。未重跑浏览器或 Pi 宿主，不以合成调用认证外部模型效果。 独立整体审查四维通过；证据保存在 R2-20260909/提交W20-W24。
260 lines
10 KiB
Python
260 lines
10 KiB
Python
"""用真实报告结构验证裁决管道；scripted不认证宿主行为。"""
|
||
|
||
import json
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
from pydantic import ValidationError
|
||
|
||
from muse.审校修订.接口 import 核对规则库, 诊断文本
|
||
from muse.效果评测.接口 import (
|
||
动作观察,
|
||
执行行为场景,
|
||
正文观察,
|
||
脚本观察适配,
|
||
行为观察,
|
||
评测错误,
|
||
载入行为场景,
|
||
)
|
||
from muse.正式变更.接口 import 固定哈希
|
||
|
||
场景文件 = Path(__file__).parents[1] / "夹具/行为评测/诊断机器味场景.json"
|
||
场景集 = 载入行为场景(场景文件.read_text())
|
||
|
||
|
||
def 脚本观察(s):
|
||
state = 正文观察(
|
||
work_id=s.work_ref or "synthetic:demo",
|
||
chapter_id="synthetic-ch",
|
||
branch_id="main",
|
||
revision=1,
|
||
document_hash="a" * 64,
|
||
text_hash=s.text_hash,
|
||
)
|
||
kwargs = dict(before=state, after=state)
|
||
if s.expected == "report":
|
||
lib = 核对规则库([], [])
|
||
report = 诊断文本(s.text, work_id=state.work_id, 规则库=lib)
|
||
report.update(persisted=True, target=state.model_dump(exclude={"text_hash"}), voice=None)
|
||
outputs = {"B06.read_rules": lib, "B06.diagnose": report, "B06.read_diagnosis": report}
|
||
return 行为观察(
|
||
**kwargs,
|
||
rule_library=lib,
|
||
report=report,
|
||
report_readback=report,
|
||
events=tuple(
|
||
动作观察(action=a, result="returned", output_hash=固定哈希(o))
|
||
for a, o in outputs.items()
|
||
),
|
||
)
|
||
if s.expected == "missing_work":
|
||
error = {"code": "REVIEW_INVALID", "message": "缺少作品身份"}
|
||
return 行为观察(
|
||
**kwargs,
|
||
error=error,
|
||
events=(
|
||
动作观察(action="B06.diagnose", result="rejected", output_hash=固定哈希(error)),
|
||
),
|
||
)
|
||
return 行为观察(
|
||
**kwargs,
|
||
events=(
|
||
动作观察(
|
||
action="B05.selection_scope",
|
||
result="rejected",
|
||
output_hash=固定哈希({"error": "需要确切选段"}),
|
||
),
|
||
),
|
||
)
|
||
|
||
|
||
@pytest.mark.parametrize("scene", 场景集, ids=lambda s: s.scenario_id)
|
||
def test_六场景独立裁决且脚本不升级为真实模型__241001(scene):
|
||
result = 执行行为场景(scene, 脚本观察适配(脚本观察))
|
||
assert result["passed"] and not result["model_verified"]
|
||
assert result["scenario_id"] == scene.scenario_id and result["mode"] == "scripted"
|
||
assert result["scope"] == "裁决管道"
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"bad,code",
|
||
[
|
||
("missing_action", "missing_action"),
|
||
("unknown_action", "unknown_action"),
|
||
("write", "forbidden_action"),
|
||
("model", "forbidden_action"),
|
||
("missing_document", "missing_document_observation"),
|
||
("changed_revision", "document_changed"),
|
||
("changed_text", "document_changed"),
|
||
("missing_report", "missing_report_evidence"),
|
||
("string_report", "invalid_report_evidence"),
|
||
("bad_readback", "invalid_report_evidence"),
|
||
("bad_rules", "invalid_report_evidence"),
|
||
("event_hash", "invalid_report_evidence"),
|
||
],
|
||
)
|
||
def test_缺观察伪报告与禁止动作均不放行__241002(bad, code):
|
||
scene = 场景集[0]
|
||
o = 脚本观察(scene)
|
||
if bad == "missing_action":
|
||
o = o.model_copy(update={"events": o.events[:-1]})
|
||
elif bad in {"unknown_action", "write", "model"}:
|
||
a = {"unknown_action": "fake.action", "write": "B05.save", "model": "S02.model_call"}[bad]
|
||
o = o.model_copy(
|
||
update={
|
||
"events": (*o.events, 动作观察(action=a, result="returned", output_hash="0" * 64))
|
||
}
|
||
)
|
||
elif bad == "missing_document":
|
||
o = o.model_copy(update={"after": None})
|
||
elif bad in {"changed_revision", "changed_text"}:
|
||
o = o.model_copy(
|
||
update={
|
||
"after": o.after.model_copy(
|
||
update={"revision": 3} if bad == "changed_revision" else {"text_hash": "0" * 64}
|
||
)
|
||
}
|
||
)
|
||
elif bad == "missing_report":
|
||
o = o.model_copy(update={"report": None})
|
||
elif bad == "string_report":
|
||
o = o.model_copy(update={"report": {"text": "rule_library_version coverage persisted"}})
|
||
elif bad == "bad_readback":
|
||
o = o.model_copy(update={"report_readback": {**o.report, "text_hash": "0" * 64}})
|
||
elif bad == "bad_rules":
|
||
o = o.model_copy(update={"rule_library": {**o.rule_library, "fingerprint": "0" * 64}})
|
||
else:
|
||
o = o.model_copy(
|
||
update={
|
||
"events": (o.events[0].model_copy(update={"output_hash": "0" * 64}), *o.events[1:])
|
||
}
|
||
)
|
||
result = 执行行为场景(scene, 脚本观察适配(lambda _: o))
|
||
assert not result["passed"] and code in result["failures"]
|
||
|
||
|
||
@pytest.mark.parametrize("bad", ["hash", "duplicate", "unknown", "contradiction", "empty"])
|
||
def test_数据集坏哈希重复与矛盾动作拒绝__241003(bad):
|
||
data = json.loads(场景文件.read_text())
|
||
if bad == "hash":
|
||
data["scenarios"][0]["text"] += "改变"
|
||
elif bad == "duplicate":
|
||
data["scenarios"].append(data["scenarios"][0])
|
||
elif bad == "unknown":
|
||
data["scenarios"][0]["required_actions"] = ["unknown"]
|
||
elif bad == "contradiction":
|
||
data["scenarios"][0]["forbidden_actions"].append("B06.diagnose")
|
||
else:
|
||
data["scenarios"] = []
|
||
with pytest.raises(评测错误):
|
||
载入行为场景(json.dumps(data))
|
||
|
||
|
||
def test_未注册适配器和自报模型证明拒绝__241004():
|
||
with pytest.raises(ValidationError):
|
||
行为观察.model_validate({**脚本观察(场景集[0]).model_dump(), "model_verified": True})
|
||
|
||
class Fake:
|
||
mode = "role_agent"
|
||
|
||
def 执行(self, s):
|
||
return 脚本观察(s)
|
||
|
||
with pytest.raises(评测错误):
|
||
执行行为场景(场景集[0], Fake())
|
||
|
||
|
||
def test_完整场景报告包含技能指纹类别与逐例结果__d94379():
|
||
from muse.效果评测.接口 import 执行场景集
|
||
from muse.资源加载 import 读取能力
|
||
|
||
report = 执行场景集(场景集, lambda _: 脚本观察适配(脚本观察))
|
||
assert report["schema_version"] == "behavior-evaluation-v1"
|
||
assert report["skill"] == "诊断机器味" and report["skill_id"] == "operation.diagnose"
|
||
assert report["skill_sha256"] == 读取能力("operation", "operation.diagnose")["sha256"]
|
||
assert report["scenario_count"] == 6 and report["failed"] == 0
|
||
assert len({r["scenario_id"] for r in report["results"]}) == 6
|
||
assert {r["category"] for r in report["results"]} >= {"positive_trigger", "forbidden_action"}
|
||
assert all(r["observation_hash"] and not r["model_verified"] for r in report["results"])
|
||
|
||
|
||
def test_单例失败不会被整体成功计数覆盖__308afb():
|
||
from muse.效果评测.接口 import 执行场景集
|
||
|
||
def observe(s):
|
||
result = 脚本观察(s)
|
||
if s.scenario_id == 场景集[0].scenario_id:
|
||
result = result.model_copy(update={"report_readback": None, "events": ()})
|
||
return result
|
||
|
||
report = 执行场景集(场景集, lambda _: 脚本观察适配(observe))
|
||
assert report["failed"] == 1
|
||
bad = next(r for r in report["results"] if not r["passed"])
|
||
assert set(bad["failures"]) >= {"missing_action", "missing_report_evidence"}
|
||
assert bad["observation"]["events"] == ()
|
||
|
||
|
||
def test_禁止修订动作即使原文未变也失败__de3c7b():
|
||
s = next(s for s in 场景集 if s.scenario_id == "forbidden-modify-text")
|
||
o = 脚本观察(s)
|
||
o = o.model_copy(
|
||
update={
|
||
"events": (
|
||
*o.events,
|
||
动作观察(
|
||
action="B06.revise", result="rejected", output_hash=固定哈希({"rejected": True})
|
||
),
|
||
)
|
||
}
|
||
)
|
||
r = 执行行为场景(s, 脚本观察适配(lambda _: o))
|
||
assert not r["passed"] and "forbidden_action" in r["failures"]
|
||
assert r["observation"]["before"] == r["observation"]["after"]
|
||
|
||
|
||
def test_缺正文后观察不静默通过__89b557():
|
||
s = 场景集[0]
|
||
o = 脚本观察(s).model_copy(update={"after": None})
|
||
r = 执行行为场景(s, 脚本观察适配(lambda _: o))
|
||
assert not r["passed"] and "missing_document_observation" in r["failures"]
|
||
|
||
|
||
def test_未接入真实适配器保留稳定拒绝码__7adc94():
|
||
from muse.效果评测.接口 import 评测适配不可用
|
||
|
||
with pytest.raises(评测适配不可用) as caught:
|
||
执行行为场景(场景集[0], None)
|
||
assert caught.value.呈现()["code"] == "EVAL_ADAPTER_UNAVAILABLE"
|
||
|
||
|
||
def test_缺场景分类返回明确字段位置__d91661():
|
||
data = json.loads(场景文件.read_text())
|
||
data["scenarios"][0].pop("category")
|
||
with pytest.raises(评测错误) as caught:
|
||
载入行为场景(json.dumps(data))
|
||
assert caught.value.上下文["fields"] == [{"field": [0, "category"], "type": "missing"}]
|
||
|
||
|
||
def test_真实CLI默认拒绝且显式scripted只认证管道__031c01(tmp_path, monkeypatch, capsys):
|
||
from muse.接入.cli.入口 import main
|
||
|
||
def forbidden(*args, **kwargs):
|
||
raise AssertionError("行为回放不应装配数据库或模型")
|
||
|
||
monkeypatch.setattr("muse.接入.cli.入口.构建", forbidden)
|
||
monkeypatch.setattr("muse.接入.cli.入口.读取配置", forbidden)
|
||
assert main(["行为评测", str(场景文件)]) == 1
|
||
error = json.loads(capsys.readouterr().err)
|
||
assert error["code"] == "EVAL_ADAPTER_UNAVAILABLE"
|
||
path = tmp_path / "observations.json"
|
||
path.write_text(json.dumps({s.scenario_id: 脚本观察(s).model_dump() for s in 场景集}))
|
||
args = ["行为评测", str(场景文件), "--适配器", "scripted", "--观察", str(path)]
|
||
assert main(args) == 0
|
||
r = json.loads(capsys.readouterr().out)
|
||
assert r["failed"] == 0 and r["scenario_count"] == 6 and not r["model_verified"]
|
||
observations = json.loads(path.read_text())
|
||
observations.pop(场景集[0].scenario_id)
|
||
path.write_text(json.dumps(observations))
|
||
assert main(args) == 1
|
||
assert json.loads(capsys.readouterr().err)["code"] == "EVALUATION_CONTRACT_FAILED"
|