"""标定采用真实隔离S02回合及合成标注,不认证外部模型的文学质量。""" import copy import json from dataclasses import replace import psycopg import pytest import test_文学评分执行与条件第三 as 文学测试 import test_评测执行与失败收敛 as 运行测试 from muse.共享.调用身份 import 用途 from muse.效果评测.接口 import 评测服务, 评测错误, 金标准发布 from muse.正式变更.接口 import 固定哈希 pytestmark = pytest.mark.数据库 执行环境 = 运行测试.执行环境 策略 = { "minimum_samples": 1, "minimum_source_groups": 1, "max_mae": 0.5, "max_absolute_error": 1.5, "minimum_verdict_agreement": 1.0, } 参数 = {**文学测试.参数, "calibration_policy": 策略} def _生成(env): 运行测试._启动执行(env) 运行测试._运行就绪(env) def _请求(env): work = 文学测试._读(env) return { "experiment_id": env["exp"]["experiment_id"], "approval_ref": "synthetic-human-label", "annotations": [ { "unit_id": u["unit_id"], "output_hash": u["evidence"]["structured_output_hash"], "scores": {d: 8.0 for d in 文学测试.维度}, "assertions": {"guard": "pass"}, "constraints": {"stay": "pass"}, } for u in work["units"] if u["kind"] == "generation" ], } def _发布(env, raw=None): return 评测服务(env["pools"][用途.维护]).发布标定金标准( replace(env["actor"], 用途=用途.维护), 金标准发布.model_validate(raw or _请求(env)) ) def _评分(env): 文学测试._推进(env) 运行测试._运行就绪(env) 文学测试._推进(env) def _读(env): return env["app"].要求评测().读取标定(env["actor"], env["exp"]["experiment_id"]) def _封存(env): return env["app"].要求评测().封存标定(env["actor"], env["exp"]["experiment_id"]) @pytest.mark.case_id( "NC-w25-25e001", environment="隔离PG与合成HTTP", given="预注册策略、独立样本和本例异常输入", when="通过公开标定接口或确定性数值合同执行", then=["实际候选、金标准、S02派发和标定凭据一致;模型不见标签;合成验证模式保留"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_金标准先于评委冻结且不进入模型输入__25e001(执行环境): env = 执行环境 _生成(env) waiting = 文学测试._推进(env) assert waiting["state"] == "awaiting_gold" and len(env["received"]) == 2 assert _读(env)["result"]["reasons"] == ["gold_missing"] published = _发布(env) assert not published["duplicate"] and _发布(env)["duplicate"] _评分(env) result = _封存(env) assert result["result"]["status"] == "passed" assert result["result"]["experiment_id"] == env["exp"]["experiment_id"] assert result["result"]["conditions_hash"] == env["exp"]["conditions_hash"] assert result["result"]["metrics"]["n"] == 20 assert result["result"]["metrics"]["mae"] == 0 assert result["result"]["metrics"]["max_abs"] == 0 assert result["result"]["metrics"]["verdict_count"] == 8 assert result["result"]["validation_modes"] == ["offline_contract"] assert result["receipt_hash"] == 固定哈希(result["result"]) assert _封存(env) == _读(env) == result assert len(env["received"]) == 4 for data in env["received"]: assert "annotations" not in data["input"] and "synthetic-human-label" not in data["input"] assert published["gold_hash"] not in data["input"] work = 文学测试._读(env) assert all( u["evidence"]["runtime"]["dispatch_metadata"]["calibration_gold_hash"] == published["gold_hash"] for u in work["units"] if u["kind"] == "comparison" and u["evidence"] ) report = env["app"].要求评测().读取实验报告(env["actor"], env["exp"]["experiment_id"]) assert report["calibration"] == result assert report["report_hash"] == 固定哈希( {k: v for k, v in report.items() if k != "report_hash"} ) assert report["literary_quality"]["metrics"] is None @pytest.mark.case_id( "NC-w25-25e002", environment="隔离PG与合成HTTP", given="预注册策略、独立样本和本例异常输入", when="通过公开标定接口或确定性数值合同执行", then=["均差、最坏单项、事实约束不一致分别失败,失败凭据仍可追溯"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize( "variant,reason", [ ("mae", "score_error_exceeded"), ("max", "score_error_exceeded"), ("verdict", "verdict_agreement_low"), ], ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_均差最坏单项和判定分歧分别保留失败凭据__25e002(执行环境, variant, reason): env = 执行环境 _生成(env) raw = _请求(env) if variant == "mae": for row in raw["annotations"]: row["scores"] = {d: 7.0 for d in 文学测试.维度} elif variant == "max": raw["annotations"][0]["scores"][文学测试.维度[0]] = 6.5 else: raw["annotations"][0]["assertions"]["guard"] = "fail" _发布(env, raw) _评分(env) result = _封存(env) assert result["result"]["status"] == "failed" assert reason in result["result"]["reasons"] assert result["receipt_id"] and _读(env) == result if variant == "max": assert result["result"]["metrics"]["mae"] < 0.5 assert result["result"]["metrics"]["max_abs"] == 1.5 @pytest.mark.case_id( "NC-w25-25e003", environment="隔离PG与合成HTTP", given="预注册策略、独立样本和本例异常输入", when="通过公开标定接口或确定性数值合同执行", then=["金标准严格覆盖全部实际候选、哈希、五维和命题"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("variant", ["missing", "duplicate", "text", "dimension", "assertion"]) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_金标准必须绑定全部实际文本和完整字段__25e003(执行环境, variant): env = 执行环境 _生成(env) raw = _请求(env) if variant == "missing": raw["annotations"].pop() elif variant == "duplicate": raw["annotations"].append(copy.deepcopy(raw["annotations"][0])) elif variant == "text": raw["annotations"][0]["output_hash"] = "0" * 64 elif variant == "dimension": raw["annotations"][0]["scores"].pop(文学测试.维度[0]) else: raw["annotations"][0]["assertions"] = {} with pytest.raises(评测错误): _发布(env, raw) assert _读(env)["result"]["reasons"] == ["gold_missing"] assert 文学测试._推进(env)["state"] == "awaiting_gold" assert len(env["received"]) == 2 @pytest.mark.case_id( "NC-w25-25e004", environment="隔离PG与合成HTTP", given="预注册策略、独立样本和本例异常输入", when="通过公开标定接口或确定性数值合同执行", then=["评测不能写oracle,生产不能读标定;历史标签与凭据不可改写"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_标注与凭据不可覆盖且数据库用途不越权__25e004(执行环境): env = 执行环境 _生成(env) raw = _请求(env) with pytest.raises(评测错误): env["app"].要求评测().发布标定金标准(env["actor"], 金标准发布.model_validate(raw)) for statement in ( "INSERT INTO oracle.muse_calibration_gold SELECT * FROM oracle.muse_calibration_gold", "UPDATE oracle.muse_calibration_gold SET payload=payload", ): with pytest.raises(psycopg.Error), env["pool"].连接() as conn: conn.execute(statement) _发布(env, raw) _评分(env) _封存(env) raw["annotations"][0]["scores"][文学测试.维度[0]] = 7.0 with pytest.raises(评测错误, match="不可替换"): _发布(env, raw) for table in ("oracle.muse_calibration_gold", "evaluation.muse_calibration_receipt"): with pytest.raises(psycopg.Error), env["pools"][用途.生产].连接(只读=True) as conn: conn.execute("SELECT * FROM " + table) with pytest.raises(psycopg.Error), env["pools"][用途.维护].连接() as conn: conn.execute("UPDATE " + table + " SET payload=payload") @pytest.mark.case_id( "NC-w25-25e005", environment="隔离PG与合成HTTP", given="预注册策略、独立样本和本例异常输入", when="通过公开标定接口或确定性数值合同执行", then=["重签本地金标准或汇总不能替换S02原派发及实际观察"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("variant", ["gold", "receipt"]) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_本地重签不能替代原派发或原标定观察__25e005(执行环境, monkeypatch, variant): from muse.效果评测.存储 import 评测存储 env = 执行环境 _生成(env) _发布(env) _评分(env) _封存(env) name = "读取金标准" if variant == "gold" else "读取标定凭据" original = getattr(评测存储, name) def changed(self, eid): row = copy.deepcopy(original(self, eid)) if row: if variant == "gold": row["payload"]["request"]["annotations"][0]["scores"][文学测试.维度[0]] = 7.0 else: row["payload"]["metrics"]["mae"] = 0.1 row["payload_hash"] = 固定哈希(row["payload"]) return row monkeypatch.setattr(评测存储, name, changed) with pytest.raises(评测错误): _读(env) assert len(env["received"]) == 4 @pytest.mark.case_id( "NC-w25-25e006", environment="隔离PG与合成HTTP", given="预注册策略、独立样本和本例异常输入", when="通过公开标定接口或确定性数值合同执行", then=["停止不抹已封存记录,但不签发新凭据;标签原样重放不写新记录"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) @pytest.mark.parametrize("sealed", [False, True]) def test_停止保留已封存标定但不允许新的凭据__25e006(执行环境, sealed): env = 执行环境 _生成(env) _发布(env) _评分(env) before = _封存(env) if sealed else _读(env) env["app"].要求评测().取消实验( env["actor"], env["exp"]["experiment_id"], "stop-after-calibration" ) after = _读(env) assert after["stopped"] and after["result"] == before["result"] assert after["receipt_hash"] == before["receipt_hash"] assert _发布(env)["duplicate"] if not sealed: with pytest.raises(评测错误, match="停止"): _封存(env) @pytest.mark.case_id( "NC-w25-25e007", environment="隔离PG与合成HTTP", given="预注册策略、独立样本和本例异常输入", when="通过公开标定接口或确定性数值合同执行", then=["来源不足保持未完成与空指标,不生成标定凭据"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize( "执行环境", [{**参数, "calibration_policy": {**策略, "minimum_source_groups": 2}}], indirect=True, ) def test_来源不足仍保留在分母且不签标定凭据__25e007(执行环境): env = 执行环境 _生成(env) _发布(env) _评分(env) result = _读(env) assert result["result"]["status"] == "incomplete" assert "source_groups_insufficient" in result["result"]["reasons"] assert result["result"]["metrics"] is None with pytest.raises(评测错误, match="未完成"): _封存(env) @pytest.mark.case_id( "NC-w25-25e00b", environment="隔离PG与合成HTTP", given="预注册策略、独立样本和本例异常输入", when="通过公开标定接口或确定性数值合同执行", then=["实际CLI发布、HTTP保存和CLI只读报告一致,权限拒绝且不追加模型调用"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_CLI维护标注与HTTP封存读回沿同一标定__25e00b(执行环境, tmp_path): import subprocess import sys import test_生产评测权限隔离 as 数据测试 from fastapi.testclient import TestClient from muse.接入.http.应用 import 创建应用 from muse.配置 import 读取配置 env = 执行环境 _生成(env) configs = {p: 数据测试._配置文件(pool, tmp_path) for p, pool in env["pools"].items()} source = tmp_path / "gold.json" source.write_text(json.dumps(_请求(env), ensure_ascii=False)) def run(purpose, action, target): return subprocess.run( [ sys.executable, "-I", "-m", "muse", "评测", str(configs[purpose]), action, str(target), ], cwd=tmp_path, capture_output=True, text=True, timeout=30, ) denied = run(用途.评测, "发布金标准", source) assert denied.returncode == 1 and "MUSE_PURPOSE_VIOLATION" in denied.stderr written = run(用途.维护, "发布金标准", source) assert written.returncode == 0, written.stderr assert not json.loads(written.stdout)["duplicate"] _评分(env) http = 创建应用(读取配置(configs[用途.评测])) endpoint = "/api/v1/evaluation/experiments/" + env["exp"]["experiment_id"] + "/calibration" with TestClient(http, headers={"origin": "http://testserver"}) as client: http.state.装配 = env["app"] assert ( client.post( "/api/v1/session", json={"password": "synthetic-evaluation-only"} ).status_code == 200 ) before = client.get(endpoint) assert before.status_code == 200 and before.json()["receipt_id"] is None denied = client.post("/api/v1/evaluation/calibration/gold", json=_请求(env)) assert denied.status_code >= 400 sealed = client.post(endpoint + "/seal", json={}) assert sealed.status_code == 200, sealed.text assert client.get(endpoint).json() == sealed.json() observed = run(用途.评测, "标定", env["exp"]["experiment_id"]) assert observed.returncode == 0, observed.stderr assert json.loads(observed.stdout) == sealed.json() replayed = run(用途.评测, "封存标定", env["exp"]["experiment_id"]) assert replayed.returncode == 0 and json.loads(replayed.stdout) == sealed.json() assert len(env["received"]) == 4 @pytest.mark.case_id( "NC-w25-25e00c", environment="隔离PG与合成HTTP", given="预注册策略、独立样本和本例异常输入", when="通过公开标定接口或确定性数值合同执行", then=["标定分割、策略指纹与幂等命令真实固定,非标定请求拒绝"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_显式标定分割与固定策略不能用批次名字或旧命令绕过__25e00c(执行环境): from uuid import UUID import test_生产评测权限隔离 as 数据测试 from muse.效果评测.接口 import 标定策略 env = 执行环境 assert env["exp"]["conditions"]["split"] == "calibration" assert len(env["exp"]["conditions_hash"]) == 64 assert env["app"].要求评测().读取实验(env["actor"], env["exp"]["experiment_id"]) == env["exp"] changed = env["request"].model_copy( update={"calibration_policy": 标定策略.model_validate({**策略, "max_mae": 0.4})} ) with pytest.raises(评测错误, match="不能更换"): env["app"].要求评测().创建实验(env["actor"], "runtime-fixture", changed) receipt = 评测服务(env["pools"][用途.维护]).发布数据集( replace(env["actor"], 用途=用途.维护), 数据测试._数据() ) wrong = env["request"].model_copy( update={ "dataset_version_id": UUID(receipt["version_id"]), "dataset_hash": receipt["public_hash"], "split": "holdout", } ) with pytest.raises(评测错误, match="calibration分割"): env["app"].要求评测().创建实验(env["actor"], "cal-name-cannot-authorize", wrong) assert not env["received"] @pytest.mark.case_id( "NC-w25-25e00d", environment="隔离PG与合成HTTP", given="预注册策略、独立样本和本例异常输入", when="通过公开标定接口或确定性数值合同执行", then=["并发原样标注只产生一份金标准,推进绑定相同原记录"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_并发封存相同金标准只产生一份且推进使用原记录__25e00d(执行环境): from concurrent.futures import ThreadPoolExecutor from threading import Barrier env = 执行环境 _生成(env) raw = _请求(env) barrier = Barrier(2) def publish(): barrier.wait(timeout=5) return _发布(env, raw) with ThreadPoolExecutor(max_workers=2) as workers: futures = [workers.submit(publish) for _ in range(2)] results = [f.result(timeout=10) for f in futures] assert sorted(r["duplicate"] for r in results) == [False, True] assert len({r["gold_hash"] for r in results}) == 1 _评分(env) assert _封存(env)["result"]["gold_hash"] == results[0]["gold_hash"] assert len(env["received"]) == 4 @pytest.mark.case_id( "NC-w25-25e00e", environment="隔离PG与合成HTTP", given="预注册策略、独立样本和本例异常输入", when="通过公开标定接口或确定性数值合同执行", then=["金标准事务失败全部回滚,原请求可恢复且没有提前评委"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_金标准事务失败不留下半份标签且可原请求恢复__25e00e(执行环境, monkeypatch): from muse.效果评测.存储 import 评测存储 env = 执行环境 _生成(env) raw = _请求(env) original = 评测存储.保存金标准 def failed(self, eid, payload): original(self, eid, payload) raise psycopg.Error("synthetic-after-insert") with monkeypatch.context() as patch: patch.setattr(评测存储, "保存金标准", failed) with pytest.raises(评测错误, match="存储操作失败"): _发布(env, raw) assert _读(env)["result"]["gold_hash"] is None assert 文学测试._推进(env)["state"] == "awaiting_gold" assert not _发布(env, raw)["duplicate"] _评分(env) assert _封存(env)["result"]["status"] == "passed" @pytest.mark.case_id( "TC-fa02755b06d5", environment="隔离PG、真实S02和合成HTTP", given="原观察的独立偏差或实际隔离实验及标注", when="调用确定性偏差计算或真实标定服务", then=["由显式calibration分割与预注册策略识别标定,不依据cal-文件名前缀放宽权限。"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_标定资格由实际分割和策略登记__fa0275(执行环境): env = 执行环境 actual = env["app"].要求评测().读取实验(env["actor"], env["exp"]["experiment_id"]) assert actual["conditions"]["split"] == "calibration" assert ( actual["conditions"]["calibration_policy"] == env["request"].calibration_policy.model_dump() ) _生成(env) assert 文学测试._推进(env)["state"] == "awaiting_gold" @pytest.mark.case_id( "TC-6534ec0b0c4b", environment="隔离PG、真实S02和合成HTTP", given="原观察的独立偏差或实际隔离实验及标注", when="调用确定性偏差计算或真实标定服务", then=["普通holdout分割不能使用标定策略,批次名字不能绕过真实条件。"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_非标定分割不因批次名字取得标定资格__6534ec(执行环境): from uuid import UUID import test_生产评测权限隔离 as 数据测试 env = 执行环境 data = 评测服务(env["pools"][用途.维护]).发布数据集( replace(env["actor"], 用途=用途.维护), 数据测试._数据() ) request = env["request"].model_copy( update={ "dataset_version_id": UUID(data["version_id"]), "dataset_hash": data["public_hash"], "split": "holdout", } ) with pytest.raises(评测错误, match="calibration分割"): env["app"].要求评测().创建实验(env["actor"], "cal-002", request) with env["pool"].连接(只读=True) as conn: assert ( conn.execute( "SELECT count(*) FROM evaluation.muse_experiment WHERE command_id='cal-002'" ).fetchone()[0] == 0 ) @pytest.mark.case_id( "TC-9a4006711b55", environment="隔离PG、真实S02和合成HTTP", given="原观察的独立偏差或实际隔离实验及标注", when="调用确定性偏差计算或真实标定服务", then=["真实实验冻结条件哈希为64位;包含实际配置、资源、评分标准及标定策略。"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_标定指纹涵盖实际标准与配置并用SHA256保存__9a4006(执行环境): env = 执行环境 with env["pool"].连接(只读=True) as conn: conditions, fingerprint = conn.execute( "SELECT conditions,conditions_hash FROM evaluation.muse_experiment " "WHERE experiment_id=%s", (env["exp"]["experiment_id"],), ).fetchone() assert len(fingerprint) == 64 and fingerprint == 固定哈希(conditions) assert conditions["generator"]["config_hash"] assert conditions["calibration_policy"] assert all( v["rubric_hash"] == 固定哈希(v["rubric"]) for v in conditions["literary_basis"].values() ) @pytest.mark.case_id( "TC-89dd605c4df6", environment="隔离PG、真实S02和合成HTTP", given="原观察的独立偏差或实际隔离实验及标注", when="调用确定性偏差计算或真实标定服务", then=["原实验读回条件及哈希不变;同命令改变策略被拒绝。"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_标定指纹重复读取稳定且调用方副本不改存储__89dd60(执行环境): env = 执行环境 svc = env["app"].要求评测() eid = env["exp"]["experiment_id"] before = svc.读取实验(env["actor"], eid) copy_of_result = svc.读取实验(env["actor"], eid) copy_of_result["conditions"]["calibration_policy"]["max_mae"] = 9.0 after = svc.读取实验(env["actor"], eid) assert after == before and after["conditions_hash"] == env["exp"]["conditions_hash"] @pytest.mark.case_id( "TC-93f97c5b83b9", environment="隔离PG、真实S02和合成HTTP", given="原观察的独立偏差或实际隔离实验及标注", when="调用确定性偏差计算或真实标定服务", then=["标定批次改为实验身份;封存凭据与实际原实验相同,读回幂等。"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_持久标定凭据读回相同实验批次__93f97c(执行环境): env = 执行环境 _生成(env) _发布(env) _评分(env) sealed = _封存(env) assert _读(env)["result"]["experiment_id"] == env["exp"]["experiment_id"] with env["pool"].连接(只读=True) as conn: batch = conn.execute( "SELECT experiment_id FROM evaluation.muse_calibration_receipt WHERE receipt_id=%s", (sealed["receipt_id"],), ).fetchone()[0] assert str(batch) == env["exp"]["experiment_id"] @pytest.mark.case_id( "TC-be8eeda69b0c", environment="隔离PG、真实S02和合成HTTP", given="原观察的独立偏差或实际隔离实验及标注", when="调用确定性偏差计算或真实标定服务", then=["凭据绑定原实验条件哈希及原金标准,实际S02派发证明没有事后换标注。"], contract="docs/系统架构/新版设计/模块设计/B10-效果评测.md", ) @pytest.mark.parametrize("执行环境", [参数], indirect=True) def test_标定凭据绑定封存时的原标准指纹__be8eed(执行环境): env = 执行环境 fingerprint = env["exp"]["conditions_hash"] _生成(env) _发布(env) _评分(env) sealed = _封存(env) with env["pool"].连接(只读=True) as conn: saved = conn.execute( "SELECT payload FROM evaluation.muse_calibration_receipt WHERE receipt_id=%s", (sealed["receipt_id"],), ).fetchone()[0] assert saved["conditions_hash"] == fingerprint assert saved["policy"] == env["exp"]["conditions"]["calibration_policy"]