9e6f1c4481
- src/muse 新版全模块（装配/共享/上下文/任务运行/作品规划/故事世界/正文写作/审校修订/知识方法/作者经验/效果评测/交付连载/资料研究/正式变更/元数据/接入/基础设施/编排）+ 测试树（单元/契约/集成/架构/迁移/端到端/夹具） - 129 项功能全部实现与自动验证（功能覆盖.json/矩阵），含 W31 补齐的规则与代价/节奏安排/伏笔与承诺 - 旧实现按处置清单退出（702 条中 324 删，保护合同与未迁移条目留存有据）；web/app.py 旧工作台退役，新工作台为唯一写入口 - 数据库/旧库迁移：真实旧库内容批次迁移链（端点守卫/PG作品正文映射/质量资产缺省投影） - 运行手册 docs/运行手册.md；W30 本机服务阶段一已运行（infra PG 为正式内容权威） - R2 执行证据与私有运行材料在 .agents.local/改造/R2-20260909/（不入库）
270 lines
9.7 KiB
Python
270 lines
9.7 KiB
Python
"""固定的逐例数学观察，不替代公开入口与真实S02证据的集成验证。"""
|
||
|
||
import pytest
|
||
|
||
from muse.效果评测.启用判据 import 判定效果, 读取效果标准
|
||
from muse.效果评测.文学评分 import 维度
|
||
from muse.效果评测.模型 import 评测错误
|
||
from muse.正式变更.接口 import 固定哈希
|
||
|
||
|
||
def _输入(count=10):
|
||
policy = 读取效果标准("writer-effect-v1")
|
||
conditions = {
|
||
"target": {
|
||
"kind": "code",
|
||
"target_ref": "fixture",
|
||
"version": "1",
|
||
"content_hash": "f" * 64,
|
||
},
|
||
"effect_policy": policy["version"],
|
||
"effect_policy_snapshot": policy,
|
||
"effect_policy_hash": 固定哈希(policy),
|
||
"evaluation_goal": "qualification",
|
||
"split": "holdout",
|
||
"literary_basis": {},
|
||
}
|
||
public, samples = [], []
|
||
for i in range(count):
|
||
sid = f"s{i}"
|
||
work = f"work-{i // 5}"
|
||
scenario = policy["scenarios"][i % 5]
|
||
conditions["literary_basis"][sid] = {"rubric": {"scenario": scenario}}
|
||
public.append(
|
||
{
|
||
"sample_id": sid,
|
||
"source_groups": [work],
|
||
"stratification": {
|
||
"work_ref": work,
|
||
"annotation_ref": "fixture",
|
||
"new_character_ratio": 0.0 if i % 5 == 0 else 0.75,
|
||
},
|
||
"input": {"original": sid, "context": {}},
|
||
}
|
||
)
|
||
literary = {}
|
||
for pair in ["A:B", "A:C", "B:C"]:
|
||
arms = pair.split(":")
|
||
literary[pair] = {
|
||
"status": "stable_report",
|
||
"execution_verified": True,
|
||
"candidates": {a: a for a in arms},
|
||
"scores": [
|
||
{
|
||
"candidate_id": a,
|
||
"dimension": dim,
|
||
"score": {"A": 7.5, "B": 2.0, "C": 8.0}[a],
|
||
}
|
||
for a in arms
|
||
for dim in 维度
|
||
],
|
||
}
|
||
samples.append(
|
||
{
|
||
"sample_id": sid,
|
||
"generation_complete": True,
|
||
"comparison_complete": True,
|
||
"detection_complete": True,
|
||
"detections": {"C": {"report": {"status": "passed"}}},
|
||
"literary": literary,
|
||
"comparisons": {"A:B": "A", "A:C": "C", "B:C": "C"},
|
||
}
|
||
)
|
||
dataset = {
|
||
"public_manifest": {"schema_version": "dataset-v1", "samples": public},
|
||
"public_hash": "dataset",
|
||
}
|
||
exp = {
|
||
"experiment_id": "effect-fixture",
|
||
"conditions": conditions,
|
||
"conditions_hash": 固定哈希(conditions),
|
||
"sample_ids": [s["sample_id"] for s in samples],
|
||
}
|
||
plan = {"primary_pair": ["A", "C"], "conditions_hash": exp["conditions_hash"]}
|
||
report = {
|
||
"experiment_id": exp["experiment_id"],
|
||
"conditions_hash": exp["conditions_hash"],
|
||
"dataset_hash": "dataset",
|
||
"plan_hash": 固定哈希(plan),
|
||
"samples": samples,
|
||
"runtime": {"states": {"completed": 100}},
|
||
"validation_modes": ["offline_contract"],
|
||
"calibration_use": {
|
||
"binding": {"version": "fixture"},
|
||
"validation_modes": ["offline_contract"],
|
||
},
|
||
"cost": {"over_budget": False, "has_unknown": False, "has_pending": False},
|
||
}
|
||
return exp, dataset, plan, report
|
||
|
||
|
||
def _下降(data, predicate, dimension):
|
||
exp, _, _, report = data
|
||
for sample in report["samples"]:
|
||
if predicate(sample, exp):
|
||
for row in sample["literary"]["A:C"]["scores"]:
|
||
if row["candidate_id"] == "C" and row["dimension"] == dimension:
|
||
row["score"] = 6.5
|
||
|
||
|
||
def test_主比较保真增益与B诊断分开__25f301():
|
||
result = 判定效果(*_输入())
|
||
assert result["gate_b"]["status"] == "passed"
|
||
assert result["metrics"]["average_deltas"]["setting_entity_fidelity"] == 0.5
|
||
assert result["metrics"]["diagnostic_deltas"]["A:B"]["setting_entity_fidelity"] == -5.5
|
||
assert result["activation_status"] == "not_evaluated"
|
||
|
||
|
||
@pytest.mark.parametrize("failure", ["system", "cost", "budget"])
|
||
def test_系统或费用失败先于样本不足__25f302(failure):
|
||
data = _输入(1)
|
||
if failure == "system":
|
||
data[3]["runtime"]["states"]["failed"] = 1
|
||
else:
|
||
data[3]["cost"]["has_unknown" if failure == "cost" else "over_budget"] = True
|
||
result = 判定效果(*data)
|
||
assert result["gate_a"]["status"] == result["gate_b"]["status"] == "failed"
|
||
|
||
|
||
def test_来源同组不能靠作品标签增加独立性__25f303():
|
||
data = _输入()
|
||
for row in data[1]["public_manifest"]["samples"]:
|
||
row["source_groups"].append("shared-history")
|
||
result = 判定效果(*data)
|
||
assert result["metrics"]["independent_sources"] == 1
|
||
assert "independent_sources_insufficient" in result["gate_b"]["reasons"]
|
||
|
||
|
||
@pytest.mark.parametrize("group", ["stratum", "scenario"])
|
||
def test_非空分层及场景退化不能由均值隐藏__25f304(group):
|
||
data = _输入()
|
||
_下降(
|
||
data,
|
||
lambda s, e: (
|
||
e["conditions"]["literary_basis"][s["sample_id"]]["rubric"]["scenario"] == "battle"
|
||
),
|
||
"style_consistency" if group == "stratum" else "narrative_tension",
|
||
)
|
||
result = 判定效果(*data)
|
||
assert result["gate_b"]["status"] == "failed"
|
||
assert (
|
||
"zero_style_regression" if group == "stratum" else "battle_scenario_regression"
|
||
) in result["gate_b"]["reasons"]
|
||
assert (
|
||
result["metrics"]["average_deltas"][
|
||
"style_consistency" if group == "stratum" else "narrative_tension"
|
||
]
|
||
== 0.2
|
||
)
|
||
if group == "scenario":
|
||
assert result["metrics"]["scenarios"]["battle"] == {
|
||
"sample_count": 2,
|
||
"stable_count": 2,
|
||
"average_deltas": {
|
||
"setting_entity_fidelity": 0.5,
|
||
"fine_outline_fidelity": 0.5,
|
||
"style_consistency": 0.5,
|
||
"narrative_tension": -1.0,
|
||
"prose_readability": 0.5,
|
||
},
|
||
}
|
||
assert (
|
||
result["metrics"]["scenarios"]["character_dialogue"]["average_deltas"][
|
||
"narrative_tension"
|
||
]
|
||
== 0.5
|
||
)
|
||
|
||
|
||
def test_没有增益与没有分数分别保留__25f305():
|
||
data = _输入()
|
||
for sample in data[3]["samples"]:
|
||
for score in sample["literary"]["A:C"]["scores"]:
|
||
score["score"] = 7.5
|
||
assert 判定效果(*data)["gate_b"]["status"] == "no_gain"
|
||
for sample in data[3]["samples"]:
|
||
sample["literary"]["A:C"]["status"] = "unstable"
|
||
result = 判定效果(*data)
|
||
assert result["gate_b"]["status"] == "insufficient_evidence"
|
||
assert result["metrics"]["average_deltas"]["setting_entity_fidelity"] is None
|
||
assert result["metrics"]["fidelity_positive_ratio"] is None
|
||
|
||
|
||
@pytest.mark.parametrize("value", [float("nan"), float("inf"), True])
|
||
def test_非有限分数与布尔不能被纳入效果__25f307(value):
|
||
data = _输入()
|
||
data[3]["samples"][0]["literary"]["A:C"]["scores"][0]["score"] = value
|
||
with pytest.raises(评测错误):
|
||
判定效果(*data)
|
||
|
||
|
||
def test_效果判断缺少固定样本不能缩小分母__25f308():
|
||
data = _输入()
|
||
data[3]["samples"].pop()
|
||
with pytest.raises(评测错误, match="全部固定"):
|
||
判定效果(*data)
|
||
|
||
|
||
def test_两作品完整五场景无选择偏差__25f30b():
|
||
result = 判定效果(*_输入())
|
||
assert not set(result["confounders"]) & {
|
||
"single_work",
|
||
"work_identity_unresolved",
|
||
"scenario_coverage_incomplete",
|
||
}
|
||
|
||
|
||
def test_五例重复场景仍须指出缺少的指定场景__25f30c():
|
||
data = _输入(5)
|
||
exp, _, plan, report = data
|
||
exp["conditions"]["literary_basis"]["s4"]["rubric"]["scenario"] = "battle"
|
||
exp["conditions_hash"] = 固定哈希(exp["conditions"])
|
||
plan["conditions_hash"] = report["conditions_hash"] = exp["conditions_hash"]
|
||
report["plan_hash"] = 固定哈希(plan)
|
||
result = 判定效果(*data)
|
||
assert result["gate_a"]["status"] == "insufficient_evidence"
|
||
assert "scenario_coverage_incomplete" in result["confounders"]
|
||
assert [
|
||
name for name, group in result["metrics"]["scenarios"].items() if group["sample_count"] == 0
|
||
] == ["returning_character"]
|
||
|
||
|
||
def test_空分层不改变旧数据集字节而声明须有来源__25f30d():
|
||
from pydantic import ValidationError
|
||
|
||
from muse.效果评测.接口 import 冻结数据集, 数据集发布
|
||
|
||
raw = {
|
||
"dataset_id": "legacy",
|
||
"revision": 1,
|
||
"samples": [
|
||
{
|
||
"sample_id": "s",
|
||
"source_ref": "synthetic:s",
|
||
"license_ref": "fixture",
|
||
"source_groups": ["work"],
|
||
"split": "holdout",
|
||
"input": {"instruction": "写一段。", "original": "原文", "context": {}},
|
||
"answer": {},
|
||
}
|
||
],
|
||
}
|
||
fixed = 冻结数据集(数据集发布.model_validate(raw))
|
||
public = {
|
||
"schema_version": "dataset-v1",
|
||
"dataset_id": "legacy",
|
||
"revision": 1,
|
||
"samples": [{k: v for k, v in raw["samples"][0].items() if k != "answer"}],
|
||
}
|
||
assert fixed["public_hash"] == 固定哈希(public)
|
||
raw["samples"][0]["stratification"] = {
|
||
"work_ref": "work",
|
||
"annotation_ref": "source",
|
||
"new_character_ratio": True,
|
||
}
|
||
with pytest.raises(ValidationError):
|
||
数据集发布.model_validate(raw)
|
||
raw["samples"][0]["stratification"] = {"work_ref": "work", "annotation_ref": " "}
|
||
with pytest.raises(评测错误, match="标注来源"):
|
||
冻结数据集(数据集发布.model_validate(raw))
|