d909d1bd1b
实现侧： - 上下文：任务范围拆分为 范围校验/范围授权；索引按可发现口径重建、索引新鲜度改对称差；依赖校验统一快照漂移说明。 - 知识方法：方法与材料读取口径统一；超限方法材料按可选省略，核对路径不再二次计费；删除无合同的读时重算。 - 任务运行：新增 context.usage/tool.denied 事件类型；连接池常驻并在装配生命周期内开关；调用结算与核对分列。 - 效果评测/审校修订/交付连载/作者经验/作品规划：凭据冻结、标定消费、导出补证、事实引文核对等收尾修复。 - 资源加载：能力正文不再夹带索引用的导航注记（该注记此前进入角色与技能的模型提示）。 - 元数据：受保护骨架与代码保护属性对齐；字段校验与内置结构口径同步。 - 基础设施：环境预检进入装配生命周期；数据库连接运行期字段不参与相等比较；索引指纹归一化 jsonb 浮点。 - 删除被替代实现：7 份旧提示词模板与空壳 资料来源 读取器。 用例侧： - 用例身份与导航元信息迁移；夹具补生命周期、同库暴露与模板封存； - 本轮定向修复：方法材料省略、事实引文、迁移回执、额度与暂停用例、慢用例超时预算等。
350 lines
15 KiB
Python
350 lines
15 KiB
Python
"""用真实报告结构验证裁决管道；scripted不认证宿主行为。"""
|
||
|
||
import json
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
from pydantic import ValidationError
|
||
|
||
from muse.审校修订.接口 import 核对规则库, 诊断文本
|
||
from muse.效果评测.接口 import (
|
||
动作观察,
|
||
执行行为场景,
|
||
正文观察,
|
||
脚本观察适配,
|
||
行为观察,
|
||
评测错误,
|
||
载入行为场景,
|
||
)
|
||
from muse.正式变更.接口 import 固定哈希
|
||
|
||
场景文件 = Path(__file__).parents[1] / "夹具/行为评测/诊断机器味场景.json"
|
||
场景集 = 载入行为场景(场景文件.read_text())
|
||
|
||
|
||
def 脚本观察(s):
|
||
state = 正文观察(
|
||
work_id=s.work_ref or "synthetic:demo",
|
||
chapter_id="synthetic-ch",
|
||
branch_id="main",
|
||
revision=1,
|
||
document_hash="a" * 64,
|
||
text_hash=s.text_hash,
|
||
)
|
||
kwargs = dict(before=state, after=state)
|
||
if s.expected == "report":
|
||
lib = 核对规则库([], [])
|
||
report = 诊断文本(s.text, work_id=state.work_id, 规则库=lib)
|
||
report.update(persisted=True, target=state.model_dump(exclude={"text_hash"}), voice=None)
|
||
outputs = {"B06.read_rules": lib, "B06.diagnose": report, "B06.read_diagnosis": report}
|
||
return 行为观察(
|
||
**kwargs,
|
||
rule_library=lib,
|
||
report=report,
|
||
report_readback=report,
|
||
events=tuple(
|
||
动作观察(action=a, result="returned", output_hash=固定哈希(o))
|
||
for a, o in outputs.items()
|
||
),
|
||
)
|
||
if s.expected == "missing_work":
|
||
error = {"code": "REVIEW_INVALID", "message": "缺少作品身份"}
|
||
return 行为观察(
|
||
**kwargs,
|
||
error=error,
|
||
events=(
|
||
动作观察(action="B06.diagnose", result="rejected", output_hash=固定哈希(error)),
|
||
),
|
||
)
|
||
return 行为观察(
|
||
**kwargs,
|
||
events=(
|
||
动作观察(
|
||
action="B05.selection_scope",
|
||
result="rejected",
|
||
output_hash=固定哈希({"error": "需要确切选段"}),
|
||
),
|
||
),
|
||
)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w24-241001",
|
||
environment="离线裁决与合成观察",
|
||
given="固定六场景及逐例原始观察",
|
||
when="显式脚本或真实隔离服务执行，注入缺失、禁止动作及伪报告",
|
||
then=["场景逐个独立裁决，报告重算和读回、来源前后哈希、失败与模式均可追溯"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("scene", 场景集, ids=lambda s: s.scenario_id)
|
||
def test_六场景独立裁决且脚本不升级为真实模型__241001(scene):
|
||
result = 执行行为场景(scene, 脚本观察适配(脚本观察))
|
||
assert result["passed"] and not result["model_verified"]
|
||
assert result["scenario_id"] == scene.scenario_id and result["mode"] == "scripted"
|
||
assert result["scope"] == "裁决管道"
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w24-241002",
|
||
environment="离线裁决与合成观察",
|
||
given="固定六场景及逐例原始观察",
|
||
when="显式脚本或真实隔离服务执行，注入缺失、禁止动作及伪报告",
|
||
then=["场景逐个独立裁决，报告重算和读回、来源前后哈希、失败与模式均可追溯"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize(
|
||
"bad,code",
|
||
[
|
||
("missing_action", "missing_action"),
|
||
("unknown_action", "unknown_action"),
|
||
("write", "forbidden_action"),
|
||
("model", "forbidden_action"),
|
||
("missing_document", "missing_document_observation"),
|
||
("changed_revision", "document_changed"),
|
||
("changed_text", "document_changed"),
|
||
("missing_report", "missing_report_evidence"),
|
||
("string_report", "invalid_report_evidence"),
|
||
("bad_readback", "invalid_report_evidence"),
|
||
("bad_rules", "invalid_report_evidence"),
|
||
("event_hash", "invalid_report_evidence"),
|
||
],
|
||
)
|
||
def test_缺观察伪报告与禁止动作均不放行__241002(bad, code):
|
||
scene = 场景集[0]
|
||
o = 脚本观察(scene)
|
||
if bad == "missing_action":
|
||
o = o.model_copy(update={"events": o.events[:-1]})
|
||
elif bad in {"unknown_action", "write", "model"}:
|
||
a = {"unknown_action": "fake.action", "write": "B05.save", "model": "S02.model_call"}[bad]
|
||
o = o.model_copy(
|
||
update={
|
||
"events": (*o.events, 动作观察(action=a, result="returned", output_hash="0" * 64))
|
||
}
|
||
)
|
||
elif bad == "missing_document":
|
||
o = o.model_copy(update={"after": None})
|
||
elif bad in {"changed_revision", "changed_text"}:
|
||
o = o.model_copy(
|
||
update={
|
||
"after": o.after.model_copy(
|
||
update={"revision": 3} if bad == "changed_revision" else {"text_hash": "0" * 64}
|
||
)
|
||
}
|
||
)
|
||
elif bad == "missing_report":
|
||
o = o.model_copy(update={"report": None})
|
||
elif bad == "string_report":
|
||
o = o.model_copy(update={"report": {"text": "rule_library_version coverage persisted"}})
|
||
elif bad == "bad_readback":
|
||
o = o.model_copy(update={"report_readback": {**o.report, "text_hash": "0" * 64}})
|
||
elif bad == "bad_rules":
|
||
o = o.model_copy(update={"rule_library": {**o.rule_library, "fingerprint": "0" * 64}})
|
||
else:
|
||
o = o.model_copy(
|
||
update={
|
||
"events": (o.events[0].model_copy(update={"output_hash": "0" * 64}), *o.events[1:])
|
||
}
|
||
)
|
||
result = 执行行为场景(scene, 脚本观察适配(lambda _: o))
|
||
assert not result["passed"] and code in result["failures"]
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w24-241003",
|
||
environment="离线裁决与合成观察",
|
||
given="固定六场景及逐例原始观察",
|
||
when="显式脚本或真实隔离服务执行，注入缺失、禁止动作及伪报告",
|
||
then=["场景逐个独立裁决，报告重算和读回、来源前后哈希、失败与模式均可追溯"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("bad", ["hash", "duplicate", "unknown", "contradiction", "empty"])
|
||
def test_数据集坏哈希重复与矛盾动作拒绝__241003(bad):
|
||
data = json.loads(场景文件.read_text())
|
||
if bad == "hash":
|
||
data["scenarios"][0]["text"] += "改变"
|
||
elif bad == "duplicate":
|
||
data["scenarios"].append(data["scenarios"][0])
|
||
elif bad == "unknown":
|
||
data["scenarios"][0]["required_actions"] = ["unknown"]
|
||
elif bad == "contradiction":
|
||
data["scenarios"][0]["forbidden_actions"].append("B06.diagnose")
|
||
else:
|
||
data["scenarios"] = []
|
||
with pytest.raises(评测错误):
|
||
载入行为场景(json.dumps(data))
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w24-241004",
|
||
environment="离线裁决与合成观察",
|
||
given="固定六场景及逐例原始观察",
|
||
when="显式脚本或真实隔离服务执行，注入缺失、禁止动作及伪报告",
|
||
then=["场景逐个独立裁决，报告重算和读回、来源前后哈希、失败与模式均可追溯"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
def test_未注册适配器和自报模型证明拒绝__241004():
|
||
with pytest.raises(ValidationError):
|
||
行为观察.model_validate({**脚本观察(场景集[0]).model_dump(), "model_verified": True})
|
||
|
||
class Fake:
|
||
mode = "role_agent"
|
||
|
||
def 执行(self, s):
|
||
return 脚本观察(s)
|
||
|
||
with pytest.raises(评测错误):
|
||
执行行为场景(场景集[0], Fake())
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-d94379a8765f",
|
||
environment="离线；scripted和资源指纹不证明真实模型",
|
||
when="运行新版公共场景裁决、实际CLI或安装包能力查询",
|
||
then=[
|
||
"完整六场景聚合、类别、逐例观察与安装包技能指纹均核对；schema改为behavior-evaluation-v1，scripted不当真实模型。"
|
||
],
|
||
contract="docs/系统架构/新版设计/验证设计/诊断行为场景.md",
|
||
)
|
||
def test_完整场景报告包含技能指纹类别与逐例结果__d94379():
|
||
from muse.效果评测.接口 import 执行场景集
|
||
from muse.资源加载 import 读取能力
|
||
|
||
report = 执行场景集(场景集, lambda _: 脚本观察适配(脚本观察))
|
||
assert report["schema_version"] == "behavior-evaluation-v1"
|
||
assert report["skill"] == "诊断机器味" and report["skill_id"] == "operation.diagnose"
|
||
assert report["skill_sha256"] == 读取能力("operation", "operation.diagnose")["sha256"]
|
||
assert report["scenario_count"] == 6 and report["failed"] == 0
|
||
assert len({r["scenario_id"] for r in report["results"]}) == 6
|
||
assert {r["category"] for r in report["results"]} >= {"positive_trigger", "forbidden_action"}
|
||
assert all(r["observation_hash"] and not r["model_verified"] for r in report["results"])
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-308afb5eebb0",
|
||
environment="离线；scripted和资源指纹不证明真实模型",
|
||
when="运行新版公共场景裁决、实际CLI或安装包能力查询",
|
||
then=[
|
||
"一个场景缺动作及报告读回使整体failed=1；脚本退出码和字段名搜索替换为实际业务结果与结构化证据。"
|
||
],
|
||
contract="docs/系统架构/新版设计/验证设计/诊断行为场景.md",
|
||
)
|
||
def test_单例失败不会被整体成功计数覆盖__308afb():
|
||
from muse.效果评测.接口 import 执行场景集
|
||
|
||
def observe(s):
|
||
result = 脚本观察(s)
|
||
if s.scenario_id == 场景集[0].scenario_id:
|
||
result = result.model_copy(update={"report_readback": None, "events": ()})
|
||
return result
|
||
|
||
report = 执行场景集(场景集, lambda _: 脚本观察适配(observe))
|
||
assert report["failed"] == 1
|
||
bad = next(r for r in report["results"] if not r["passed"])
|
||
assert set(bad["failures"]) >= {"missing_action", "missing_report_evidence"}
|
||
assert bad["observation"]["events"] == ()
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-de3c7bd1b416",
|
||
environment="离线；scripted和资源指纹不证明真实模型",
|
||
when="运行新版公共场景裁决、实际CLI或安装包能力查询",
|
||
then=["即使正文未改变，被拒绝的修订尝试也记录为禁止动作失败，不只检查最终文本。"],
|
||
contract="docs/系统架构/新版设计/验证设计/诊断行为场景.md",
|
||
)
|
||
def test_禁止修订动作即使原文未变也失败__de3c7b():
|
||
s = next(s for s in 场景集 if s.scenario_id == "forbidden-modify-text")
|
||
o = 脚本观察(s)
|
||
o = o.model_copy(
|
||
update={
|
||
"events": (
|
||
*o.events,
|
||
动作观察(
|
||
action="B06.revise", result="rejected", output_hash=固定哈希({"rejected": True})
|
||
),
|
||
)
|
||
}
|
||
)
|
||
r = 执行行为场景(s, 脚本观察适配(lambda _: o))
|
||
assert not r["passed"] and "forbidden_action" in r["failures"]
|
||
assert r["observation"]["before"] == r["observation"]["after"]
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-89b557064033",
|
||
environment="离线；scripted和资源指纹不证明真实模型",
|
||
when="运行新版公共场景裁决、实际CLI或安装包能力查询",
|
||
then=["缺调用后正文观察明确失败；不接受模型自报input_text_modified=false。"],
|
||
contract="docs/系统架构/新版设计/验证设计/诊断行为场景.md",
|
||
)
|
||
def test_缺正文后观察不静默通过__89b557():
|
||
s = 场景集[0]
|
||
o = 脚本观察(s).model_copy(update={"after": None})
|
||
r = 执行行为场景(s, 脚本观察适配(lambda _: o))
|
||
assert not r["passed"] and "missing_document_observation" in r["failures"]
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-7adc94c63220",
|
||
environment="离线；scripted和资源指纹不证明真实模型",
|
||
when="运行新版公共场景裁决、实际CLI或安装包能力查询",
|
||
then=["未注册适配器明确EVAL_ADAPTER_UNAVAILABLE，不回退到脚本化观察。"],
|
||
contract="docs/系统架构/新版设计/验证设计/诊断行为场景.md",
|
||
)
|
||
def test_未接入真实适配器保留稳定拒绝码__7adc94():
|
||
from muse.效果评测.接口 import 评测适配不可用
|
||
|
||
with pytest.raises(评测适配不可用) as caught:
|
||
执行行为场景(场景集[0], None)
|
||
assert caught.value.呈现()["code"] == "EVAL_ADAPTER_UNAVAILABLE"
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-d9166147b7f2",
|
||
environment="离线；scripted和资源指纹不证明真实模型",
|
||
when="运行新版公共场景裁决、实际CLI或安装包能力查询",
|
||
then=["缺category返回结构化字段位置和missing类型，不依赖旧异常类或脚本错误字符串。"],
|
||
contract="docs/系统架构/新版设计/验证设计/诊断行为场景.md",
|
||
)
|
||
def test_缺场景分类返回明确字段位置__d91661():
|
||
data = json.loads(场景文件.read_text())
|
||
data["scenarios"][0].pop("category")
|
||
with pytest.raises(评测错误) as caught:
|
||
载入行为场景(json.dumps(data))
|
||
assert caught.value.上下文["fields"] == [{"field": [0, "category"], "type": "missing"}]
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-031c01997996",
|
||
environment="离线；scripted和资源指纹不证明真实模型",
|
||
when="运行新版公共场景裁决、实际CLI或安装包能力查询",
|
||
then=[
|
||
(
|
||
"实际CLI main默认真实适配器拒绝，业务错误统一新版退出码1；显式scripted通过仅证明裁"
|
||
"决，缺场景观察拒绝且不装配数据库。"
|
||
)
|
||
],
|
||
contract="docs/系统架构/新版设计/验证设计/诊断行为场景.md",
|
||
)
|
||
def test_真实CLI默认拒绝且显式scripted只认证管道__031c01(tmp_path, monkeypatch, capsys):
|
||
from muse.接入.cli.入口 import main
|
||
|
||
def forbidden(*args, **kwargs):
|
||
raise AssertionError("行为回放不应装配数据库或模型")
|
||
|
||
monkeypatch.setattr("muse.接入.cli.入口.构建", forbidden)
|
||
monkeypatch.setattr("muse.接入.cli.入口.读取配置", forbidden)
|
||
assert main(["行为评测", str(场景文件)]) == 1
|
||
error = json.loads(capsys.readouterr().err)
|
||
assert error["code"] == "EVAL_ADAPTER_UNAVAILABLE"
|
||
path = tmp_path / "observations.json"
|
||
path.write_text(json.dumps({s.scenario_id: 脚本观察(s).model_dump() for s in 场景集}))
|
||
args = ["行为评测", str(场景文件), "--适配器", "scripted", "--观察", str(path)]
|
||
assert main(args) == 0
|
||
r = json.loads(capsys.readouterr().out)
|
||
assert r["failed"] == 0 and r["scenario_count"] == 6 and not r["model_verified"]
|
||
observations = json.loads(path.read_text())
|
||
observations.pop(场景集[0].scenario_id)
|
||
path.write_text(json.dumps(observations))
|
||
assert main(args) == 1
|
||
assert json.loads(capsys.readouterr().err)["code"] == "EVALUATION_CONTRACT_FAILED"
|