d909d1bd1b
实现侧： - 上下文：任务范围拆分为 范围校验/范围授权；索引按可发现口径重建、索引新鲜度改对称差；依赖校验统一快照漂移说明。 - 知识方法：方法与材料读取口径统一；超限方法材料按可选省略，核对路径不再二次计费；删除无合同的读时重算。 - 任务运行：新增 context.usage/tool.denied 事件类型；连接池常驻并在装配生命周期内开关；调用结算与核对分列。 - 效果评测/审校修订/交付连载/作者经验/作品规划：凭据冻结、标定消费、导出补证、事实引文核对等收尾修复。 - 资源加载：能力正文不再夹带索引用的导航注记（该注记此前进入角色与技能的模型提示）。 - 元数据：受保护骨架与代码保护属性对齐；字段校验与内置结构口径同步。 - 基础设施：环境预检进入装配生命周期；数据库连接运行期字段不参与相等比较；索引指纹归一化 jsonb 浮点。 - 删除被替代实现：7 份旧提示词模板与空壳 资料来源 读取器。 用例侧： - 用例身份与导航元信息迁移；夹具补生命周期、同库暴露与模板封存； - 本轮定向修复：方法材料省略、事实引文、迁移回执、额度与暂停用例、慢用例超时预算等。
459 lines
20 KiB
Python
459 lines
20 KiB
Python
"""从实际S02交付核对判据来源；不接受留存副本自报的运行身份。"""
|
||
|
||
import copy
|
||
from decimal import Decimal
|
||
|
||
import pytest
|
||
import test_回放资料封存 as 资料测试
|
||
import test_实际效果判据 as 效果测试
|
||
import test_文学评分执行与条件第三 as 文学测试
|
||
import test_评测执行与失败收敛 as 运行测试
|
||
import test_评测语义检测 as 检测测试
|
||
|
||
from muse.任务运行.接口 import 任务状态
|
||
from muse.共享.错误 import Muse错误
|
||
from muse.效果评测.接口 import 评测错误
|
||
from muse.正式变更.接口 import 固定哈希
|
||
|
||
pytestmark = pytest.mark.数据库
|
||
执行环境 = 运行测试.执行环境
|
||
生成环境 = 资料测试.生成环境
|
||
资料环境 = 资料测试.资料环境
|
||
参数 = {**检测测试.参数, "calls": 21, "budget_calls": 100, "max_steps": 150}
|
||
|
||
|
||
def _读(env):
|
||
return env["app"].要求评测().读取实验报告(env["actor"], env["exp"]["experiment_id"])
|
||
|
||
|
||
def _完成(env):
|
||
运行测试._启动执行(env)
|
||
运行测试._运行就绪(env)
|
||
env["app"].要求评测().推进实验(env["actor"], env["exp"]["experiment_id"])
|
||
运行测试._运行就绪(env)
|
||
return _读(env)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-3d9888c411dd",
|
||
environment="隔离PG、真实S02与合成HTTP；不认证外部模型质量",
|
||
given="本原用例的独立输入与实际故障注入，不复用其他用例终态",
|
||
when="执行实际任务或重算原交付的公开报告",
|
||
then=[
|
||
"empty、wrong_role、unbound三种副本分别拒绝",
|
||
"错误明确来自交付快照与原S02不符，不能形成有效报告",
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("mutation", ["empty", "wrong_role", "unbound"])
|
||
def test_留存副本缺回执错角色或输入不符均拒绝__25f501(执行环境, monkeypatch, mutation):
|
||
from muse.效果评测.存储 import 评测存储
|
||
|
||
env = 执行环境
|
||
_完成(env)
|
||
read = 评测存储.读取交付
|
||
|
||
def changed(self, uid):
|
||
row = copy.deepcopy(read(self, uid))
|
||
if row and self.读取单元(uid)["kind"] == "generation":
|
||
runtime = row["evidence"]["runtime"]
|
||
if mutation == "empty":
|
||
row["evidence"]["runtime"] = {}
|
||
elif mutation == "wrong_role":
|
||
runtime["processor_id"] = "eval.call.detector"
|
||
else:
|
||
runtime["input_metadata"]["user_input_hash"] = "f" * 64
|
||
return row
|
||
|
||
monkeypatch.setattr(评测存储, "读取交付", changed)
|
||
with pytest.raises(评测错误, match="交付快照"):
|
||
_读(env)
|
||
assert len(env["received"]) == 3
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-59a7452d6e70",
|
||
environment="隔离PG、真实S02与合成HTTP；不认证外部模型质量",
|
||
given="本原用例的独立输入与实际故障注入，不复用其他用例终态",
|
||
when="执行实际任务或重算原交付的公开报告",
|
||
then=[
|
||
"命题身份改写并重签检测报告仍拒绝读回",
|
||
"不给出有效schema或效果结果；原候选与S02派生报告不允许错绑",
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_命题身份与原候选派生报告不能换绑__25f503(执行环境, monkeypatch):
|
||
from muse.效果评测.存储 import 评测存储
|
||
|
||
env = 执行环境
|
||
检测测试._执行(env)
|
||
read = 评测存储.读取交付
|
||
|
||
def changed(self, uid):
|
||
row = copy.deepcopy(read(self, uid))
|
||
if row and self.读取单元(uid)["kind"] == "detection":
|
||
result = row["evidence"]["detection"]
|
||
result["report"]["assertion_verdicts"][0]["statement_id"] = "forged-id"
|
||
result["report_hash"] = 固定哈希(result["report"])
|
||
return row
|
||
|
||
monkeypatch.setattr(评测存储, "读取交付", changed)
|
||
with pytest.raises(评测错误, match="检测报告"):
|
||
_读(env)
|
||
assert len(env["received"]) == 6
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-b02c228eb260",
|
||
environment="隔离PG、真实S02与合成HTTP；不认证外部模型质量",
|
||
given="本原用例的独立输入与实际故障注入，不复用其他用例终态",
|
||
when="执行实际任务或重算原交付的公开报告",
|
||
then=[
|
||
"合法未知检测完成并进入判据",
|
||
"真实任务completed，未知质量不是system_failure",
|
||
"目标硬约束pass=0、unknown=1，已知通过覆盖率0而非满覆盖",
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize(
|
||
"执行环境",
|
||
[{**参数, "detector_output": lambda material, _: 检测测试._输出(material, "unknown")}],
|
||
indirect=True,
|
||
)
|
||
def test_合法未知检测进入判据而覆盖不能记满__25f504(执行环境):
|
||
env = 效果测试._新实验(执行环境)
|
||
result = 效果测试._完成(env)
|
||
assert result["assessment"]["gate_a"] == {
|
||
"status": "insufficient_evidence",
|
||
"reasons": ["target_semantics_unresolved"],
|
||
}
|
||
report = _读(env)
|
||
assert report["state"] == "completed" and not report["runtime"]["states"].get("failed")
|
||
for sample in report["samples"]:
|
||
assert sample["detection_complete"]
|
||
counts = sample["detections"]["treatment"]["report"]["constraint_counts"]
|
||
assert counts == {"pass": 0, "fail": 0, "unknown": 1}
|
||
assert counts["pass"] / sum(counts.values()) == 0
|
||
|
||
|
||
def _恢复(env, unit, command):
|
||
return env["app"].任务运行.控制任务(
|
||
unit["task_id"], env["actor"].作者, 任务状态.已失败, "恢复", 命令ID=command
|
||
)
|
||
|
||
|
||
def _重试期(env, kind, script, attempts):
|
||
"""只制造本阶段每个臂的真实失败；逐臂原任务恢复，不重建实验或预算。"""
|
||
env["scripted"].extend([script] * 3)
|
||
运行测试._运行就绪(env, allow_failure=True)
|
||
failed = [u for u in 文学测试._读(env)["units"] if u["state"] == "failed"]
|
||
assert len(failed) == 3 and {u["kind"] for u in failed} == {kind}
|
||
for attempt in range(1, attempts):
|
||
if attempt < attempts - 1:
|
||
env["scripted"].extend([script] * 3)
|
||
for unit in failed:
|
||
_恢复(env, unit, f"retry-{kind}-{unit['unit_id']}-{attempt}")
|
||
运行测试._运行就绪(env, allow_failure=attempt < attempts - 1)
|
||
return {u["unit_id"] for u in failed}
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-de7ebdffa9d1",
|
||
environment="隔离PG、真实S02与合成HTTP；不认证外部模型质量",
|
||
given="本原用例的独立输入与实际故障注入，不复用其他用例终态",
|
||
when="执行实际任务或重算原交付的公开报告",
|
||
then=[
|
||
"真实ABC每臂写手与检测分别保留两次及三次尝试，最后实际成功确认绑定产物，历史失败不冒充当前系统失败"
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("attempts", [2, 3])
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_真实ABC写手与检测重试均绑定各臂最终交付__25f505(执行环境, 资料环境, attempts):
|
||
env = 文学测试.创建ABC实验(执行环境, 资料环境, detector=True, calls=45)
|
||
运行测试._启动执行(env)
|
||
writer_ids = _重试期(env, "generation", "bad_output", attempts)
|
||
文学测试._推进(env)
|
||
detector_ids = _重试期(env, "detection", "bad_output", attempts)
|
||
文学测试._推进(env)
|
||
work = 文学测试._读(env)
|
||
assert work["state"] == "completed"
|
||
for unit in work["units"]:
|
||
if unit["unit_id"] not in writer_ids | detector_ids:
|
||
continue
|
||
calls = unit["cost"]["calls"]
|
||
assert len(calls) == attempts and not unit["cost"]["has_unknown"]
|
||
current = unit["evidence"]["runtime"]
|
||
call_id = current["response_metadata"]["call_id"]
|
||
successful = next(c for c in calls if c["call_id"] == call_id)
|
||
assert current["attempt_id"] == successful["attempt_id"]
|
||
with env["pool"].连接(只读=True) as conn:
|
||
saved = env["app"].任务运行.列出已保存结构化交付于(
|
||
conn, env["actor"].作者, unit["task_id"]
|
||
)
|
||
assert [(r["call_id"], r["attempt_id"]) for r in saved] == [
|
||
(call_id, current["attempt_id"])
|
||
]
|
||
assert len({c["attempt_id"] for c in calls}) == attempts
|
||
report = _读(env)
|
||
assert set(report["samples"][0]["detections"]) == {"A", "B", "C"}
|
||
assert not report["runtime"]["states"].get("failed")
|
||
assert len(env["received"]) == 6 * attempts + 6
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-207459c9a289",
|
||
environment="隔离PG、真实S02与合成HTTP；不认证外部模型质量",
|
||
given="本原用例的独立输入与实际故障注入，不复用其他用例终态",
|
||
when="执行实际任务或重算原交付的公开报告",
|
||
then=[
|
||
"已知费用API终态失败可在原额度内显式恢复；前次调用与费用保留，最终成功后不记当前系统失败"
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_已知费用API失败后成功可用且失败回合不抹除__25f506(执行环境):
|
||
env = 执行环境
|
||
运行测试._启动执行(env)
|
||
运行测试._运行就绪(env)
|
||
文学测试._推进(env)
|
||
env["scripted"].append("api_error")
|
||
运行测试._运行就绪(env, allow_failure=True)
|
||
failed = next(u for u in 文学测试._读(env)["units"] if u["state"] == "failed")
|
||
assert failed["kind"] == "detection"
|
||
old_call = failed["cost"]["calls"][0]
|
||
# 本夹具按已登记的固定每调用费率计价；终态失败也不推定免费。
|
||
assert old_call["state"] == "settled" and Decimal(old_call["actual_amount"]) == Decimal(".125")
|
||
_恢复(env, failed, "known-api-retry")
|
||
运行测试._运行就绪(env)
|
||
文学测试._推进(env)
|
||
report = _读(env)
|
||
assert report["state"] == "completed" and not report["runtime"]["states"].get("failed")
|
||
unit = next(u for u in report["samples"][0]["units"] if u["unit_id"] == failed["unit_id"])
|
||
assert len(unit["calls"]) == 2 and old_call in unit["calls"]
|
||
final = next(
|
||
d for d in report["samples"][0]["detections"].values() if d["unit_id"] == unit["unit_id"]
|
||
)
|
||
assert final["call_id"] != old_call["call_id"]
|
||
assert len(env["received"]) == 7
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-d56b6e45a28d",
|
||
environment="隔离PG、真实S02与合成HTTP；不认证外部模型质量",
|
||
given="本原用例的独立输入与实际故障注入，不复用其他用例终态",
|
||
when="执行实际任务或重算原交付的公开报告",
|
||
then=[
|
||
"检测最后API回合失败时A阶段failed",
|
||
"首要system_failure保留，检测不完整，不借历史或样本不足放行",
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_API失败若为最后回合仍是系统失败__25f507(执行环境):
|
||
env = 效果测试._新实验(执行环境, 1)
|
||
运行测试._启动执行(env)
|
||
运行测试._运行就绪(env)
|
||
文学测试._推进(env)
|
||
env["scripted"].append("api_error")
|
||
运行测试._运行就绪(env, allow_failure=True)
|
||
result = env["app"].要求评测().读取效果判据(env["actor"], env["exp"]["experiment_id"])
|
||
assert result["assessment"]["gate_a"] == {"status": "failed", "reasons": ["system_failure"]}
|
||
assert not _读(env)["samples"][0]["detection_complete"]
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-02cfda2179fc",
|
||
environment="隔离PG、真实S02与合成HTTP；不认证外部模型质量",
|
||
given="本原用例的独立输入与实际故障注入，不复用其他用例终态",
|
||
when="执行实际任务或重算原交付的公开报告",
|
||
then=[
|
||
"冻结三次写手额度耗尽后恢复仍失败且无产物",
|
||
"第四次不外发、不增加回执或重置预算；不允许外部列表伪造额外尝试",
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("执行环境", [{"calls": 9}], indirect=True)
|
||
def test_三次写手额度耗尽恢复也不发送第四次__25f508(执行环境):
|
||
env = 执行环境
|
||
env["scripted"].append("bad_output")
|
||
运行测试._启动执行(env)
|
||
运行测试._运行就绪(env, allow_failure=True)
|
||
failed = next(u for u in 文学测试._读(env)["units"] if u["state"] == "failed")
|
||
for attempt in (2, 3):
|
||
env["scripted"].append("bad_output")
|
||
_恢复(env, failed, f"quota-retry-{attempt}")
|
||
运行测试._运行就绪(env, allow_failure=True)
|
||
before = len(env["received"])
|
||
_恢复(env, failed, "quota-retry-4")
|
||
运行测试._运行就绪(env, allow_failure=True)
|
||
work = 文学测试._读(env)
|
||
final = next(u for u in work["units"] if u["unit_id"] == failed["unit_id"])
|
||
assert final["state"] == "failed" and final["output"] is None
|
||
assert len(final["cost"]["calls"]) == 3 and len(env["received"]) == before == 4
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-7ca68e954f35",
|
||
environment="隔离PG、真实S02与合成HTTP；不认证外部模型质量",
|
||
given="本原用例的独立输入与实际故障注入，不复用其他用例终态",
|
||
when="执行实际任务或重算原交付的公开报告",
|
||
then=[
|
||
"篡改为999后原output_hash仍不变",
|
||
"读取被S02原结构化输出核验拒绝",
|
||
"不给出有效报告或评分",
|
||
"错误指出输入输出与真实调用不一致",
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_不重签原哈希的越界评分不得进入判据__25f509(执行环境, monkeypatch):
|
||
from muse.效果评测.存储 import 评测存储
|
||
|
||
env = 执行环境
|
||
检测测试._执行(env)
|
||
read = 评测存储.读取交付
|
||
changed_hashes = []
|
||
|
||
def changed(self, uid):
|
||
row = copy.deepcopy(read(self, uid))
|
||
if row and self.读取单元(uid)["kind"] == "comparison":
|
||
original_hash = row["output_hash"]
|
||
row["output"]["candidate_scores"][0]["scores"][0]["score"] = 999
|
||
changed_hashes.append((original_hash, row["output_hash"], 固定哈希(row["output"])))
|
||
return row
|
||
|
||
monkeypatch.setattr(评测存储, "读取交付", changed)
|
||
with pytest.raises(Muse错误, match="真实调用不一致"):
|
||
_读(env)
|
||
assert changed_hashes and all(a == b and b != c for a, b, c in changed_hashes)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-23458d2ee9ad",
|
||
environment="隔离PG、真实S02与合成HTTP；不认证外部模型质量",
|
||
given="本原用例的独立输入与实际故障注入，不复用其他用例终态",
|
||
when="执行实际任务或重算原交付的公开报告",
|
||
then=["文学报告重签仍不能换掉原调用引用", "不给出有效报告", "交付快照与原S02不符被明确拒绝"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_文学报告重签不能换掉原评委回合__25f50a(执行环境, monkeypatch):
|
||
from muse.效果评测.存储 import 评测存储
|
||
|
||
env = 执行环境
|
||
检测测试._执行(env)
|
||
read = 评测存储.读取交付
|
||
|
||
def changed(self, uid):
|
||
row = copy.deepcopy(read(self, uid))
|
||
if row and self.读取单元(uid)["kind"] == "comparison":
|
||
row["evidence"]["runtime"]["response_metadata"]["call_id"] = "forged-panel-call"
|
||
row["evidence"]["judgment"]["report_hash"] = 固定哈希(
|
||
row["evidence"]["judgment"]["report"]
|
||
)
|
||
return row
|
||
|
||
monkeypatch.setattr(评测存储, "读取交付", changed)
|
||
with pytest.raises(评测错误, match="交付快照"):
|
||
_读(env)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-a3346ca66033",
|
||
environment="隔离PG、真实S02与合成HTTP；不认证外部模型质量",
|
||
given="本原用例的独立输入与实际故障注入，不复用其他用例终态",
|
||
when="执行实际任务或重算原交付的公开报告",
|
||
then=[
|
||
"将报告评分策略替换为人物对话后拒绝",
|
||
"不给出有效报告",
|
||
"重签派生文学哈希仍与实际文本及原模型输出不符",
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_报告不能替换样本预注册场景标准__25f50b(执行环境, monkeypatch):
|
||
from muse.效果评测.存储 import 评测存储
|
||
from muse.效果评测.文学评分 import 读取评分标准
|
||
|
||
env = 执行环境
|
||
检测测试._执行(env)
|
||
read = 评测存储.读取交付
|
||
|
||
def changed(self, uid):
|
||
row = copy.deepcopy(read(self, uid))
|
||
if row and self.读取单元(uid)["kind"] == "comparison":
|
||
judgment = row["evidence"]["judgment"]
|
||
judgment["literary"]["rubric_hash"] = 固定哈希(读取评分标准("character_dialogue"))
|
||
judgment["literary_hash"] = 固定哈希(judgment["literary"])
|
||
return row
|
||
|
||
monkeypatch.setattr(评测存储, "读取交付", changed)
|
||
with pytest.raises(评测错误, match="实际文本及原始模型输出"):
|
||
_读(env)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"TC-b90d3d771b46",
|
||
environment="隔离PG、真实S02与合成HTTP；不认证外部模型质量",
|
||
given="本原用例的独立输入与实际故障注入，不复用其他用例终态",
|
||
when="执行实际任务或重算原交付的公开报告",
|
||
then=[
|
||
"合法9.5分及重新计算本地输出哈希仍拒绝",
|
||
"不给出有效报告或恢复评分",
|
||
"S02真实结构化输出不一致，局部重签不能替代原模型回合",
|
||
],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("执行环境", [参数], indirect=True)
|
||
def test_合法评分重签仍不能替代原结构化输出__25f50c(执行环境, monkeypatch):
|
||
from muse.效果评测.存储 import 评测存储
|
||
|
||
env = 执行环境
|
||
检测测试._执行(env)
|
||
read = 评测存储.读取交付
|
||
|
||
def changed(self, uid):
|
||
row = copy.deepcopy(read(self, uid))
|
||
if row and self.读取单元(uid)["kind"] == "comparison":
|
||
row["output"]["candidate_scores"][0]["scores"][0]["score"] = 9.5
|
||
row["output_hash"] = 固定哈希(row["output"])
|
||
row["evidence"]["structured_output_hash"] = row["output_hash"]
|
||
return row
|
||
|
||
monkeypatch.setattr(评测存储, "读取交付", changed)
|
||
with pytest.raises(Muse错误, match="真实调用不一致"):
|
||
_读(env)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w25-25f502",
|
||
environment="隔离PG与合成HTTP",
|
||
given="完成同一实际生成与比较；只篡改留存副本的调用引用或输出哈希",
|
||
when="经公开报告入口反查原S02",
|
||
then=["两类字段都拒绝，未篡改原S02也不能让伪引用或伪哈希进入报告"],
|
||
contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md",
|
||
)
|
||
@pytest.mark.parametrize("field", ["call_id", "structured_output_hash"])
|
||
def test_留存副本的调用与输出哈希不能污染报告__25f502(执行环境, monkeypatch, field):
|
||
from muse.效果评测.存储 import 评测存储
|
||
|
||
env = 执行环境
|
||
_完成(env)
|
||
read = 评测存储.读取交付
|
||
|
||
def changed(self, uid):
|
||
row = copy.deepcopy(read(self, uid))
|
||
if row and self.读取单元(uid)["kind"] == "comparison":
|
||
if field == "call_id":
|
||
row["evidence"]["runtime"]["response_metadata"]["call_id"] = "forged-call"
|
||
else:
|
||
row["evidence"]["structured_output_hash"] = "f" * 64
|
||
return row
|
||
|
||
monkeypatch.setattr(评测存储, "读取交付", changed)
|
||
with pytest.raises(评测错误, match="交付快照"):
|
||
_读(env)
|
||
assert len(env["received"]) == 3
|