9e6f1c4481
- src/muse 新版全模块（装配/共享/上下文/任务运行/作品规划/故事世界/正文写作/审校修订/知识方法/作者经验/效果评测/交付连载/资料研究/正式变更/元数据/接入/基础设施/编排）+ 测试树（单元/契约/集成/架构/迁移/端到端/夹具） - 129 项功能全部实现与自动验证（功能覆盖.json/矩阵），含 W31 补齐的规则与代价/节奏安排/伏笔与承诺 - 旧实现按处置清单退出（702 条中 324 删，保护合同与未迁移条目留存有据）；web/app.py 旧工作台退役，新工作台为唯一写入口 - 数据库/旧库迁移：真实旧库内容批次迁移链（端点守卫/PG作品正文映射/质量资产缺省投影） - 运行手册 docs/运行手册.md；W30 本机服务阶段一已运行（infra PG 为正式内容权威） - R2 执行证据与私有运行材料在 .agents.local/改造/R2-20260909/（不入库）
106 lines
4.2 KiB
Python
106 lines
4.2 KiB
Python
"""数据集的来源泄漏、版本和公开输入合同，不认证真实模型效果。"""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from muse.效果评测.接口 import 公开评测输入, 冻结数据集, 数据样本, 数据集发布, 评测错误
|
|
|
|
|
|
def _sample(id_="sample-1", split="holdout", group="book-1", text="我的公开合成正文。"):
|
|
return 数据样本(
|
|
sample_id=id_,
|
|
source_ref="synthetic:sample",
|
|
license_ref="synthetic:许可",
|
|
source_groups=(group,),
|
|
split=split,
|
|
input=公开评测输入(instruction="保持原意改写。", original=text),
|
|
answer={"private": "只属于oracle的金标准"},
|
|
)
|
|
|
|
|
|
def test_封存输入和答案分开且不受原请求容器修改影响__252001():
|
|
s = _sample()
|
|
request = 数据集发布(dataset_id="synthetic", revision=1, samples=(s,))
|
|
frozen = 冻结数据集(request)
|
|
assert "只属于oracle" not in json.dumps(frozen["public"], ensure_ascii=False)
|
|
assert frozen["answers"][0]["answer"] == s.answer
|
|
s.answer["private"] = "后来改变的值"
|
|
s.input.context["nested"] = {"change": 1}
|
|
assert frozen["answers"][0]["answer"]["private"] == "只属于oracle的金标准"
|
|
assert frozen["public"]["samples"][0]["input"]["context"] == {}
|
|
changed = 冻结数据集(request)
|
|
assert frozen["version_id"] == changed["version_id"]
|
|
assert frozen["public_hash"] != changed["public_hash"]
|
|
assert frozen["answer_hash"] != changed["answer_hash"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"bad",
|
|
[
|
|
"duplicate_id",
|
|
"same_group",
|
|
"same_text",
|
|
"instruction_variant",
|
|
"duplicate_group",
|
|
"blank_source",
|
|
"blank_license",
|
|
],
|
|
)
|
|
def test_样本重复和跨分割来源泄漏被拒绝__252002(bad):
|
|
a = _sample()
|
|
b = _sample("sample-2", "calibration", "book-2", "另一段不同正文。")
|
|
if bad == "duplicate_id":
|
|
b = b.model_copy(update={"sample_id": a.sample_id})
|
|
elif bad == "same_group":
|
|
b = b.model_copy(update={"source_groups": a.source_groups})
|
|
elif bad in {"same_text", "instruction_variant"}:
|
|
b = b.model_copy(
|
|
update={
|
|
"input": a.input.model_copy(
|
|
update={"instruction": "改变任务要求"} if bad == "instruction_variant" else {}
|
|
)
|
|
}
|
|
)
|
|
elif bad == "duplicate_group":
|
|
b = b.model_copy(update={"source_groups": ("book-2", "book-2")})
|
|
elif bad == "blank_source":
|
|
b = b.model_copy(update={"source_ref": " "})
|
|
else:
|
|
b = b.model_copy(update={"license_ref": " "})
|
|
with pytest.raises(评测错误):
|
|
冻结数据集(数据集发布(dataset_id="synthetic", revision=1, samples=(a, b)))
|
|
|
|
|
|
def test_只有指令的样本按实际任务内容识别而非都视为空文本__252003():
|
|
a = _sample(split="discovery", text="").model_copy(
|
|
update={"input": 公开评测输入(instruction="提出关于雨天的故事。")}
|
|
)
|
|
b = _sample("sample-2", "holdout", "book-2", text="").model_copy(
|
|
update={"input": 公开评测输入(instruction="提出关于沙漠的故事。")}
|
|
)
|
|
result = 冻结数据集(数据集发布(dataset_id="synthetic", revision=1, samples=(a, b)))
|
|
assert len(result["public"]["samples"]) == 2
|
|
with pytest.raises(评测错误):
|
|
冻结数据集(
|
|
数据集发布(
|
|
dataset_id="synthetic",
|
|
revision=1,
|
|
samples=(a, b.model_copy(update={"input": a.input})),
|
|
)
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("field", ["answer", "oracle", "blind_mapping", "arm", "model_id"])
|
|
def test_模型输入外壳不接受评测内部字段__252004(field):
|
|
with pytest.raises(ValidationError):
|
|
公开评测输入.model_validate({"instruction": "公开任务", field: {"secret": "不得合并"}})
|
|
|
|
|
|
def test_样本顺序不改变数据集身份及封存哈希__252005():
|
|
a, b = _sample(), _sample("sample-2", "calibration", "book-2", "另一段正文。")
|
|
one = 冻结数据集(数据集发布(dataset_id="synthetic", revision=1, samples=(a, b)))
|
|
two = 冻结数据集(数据集发布(dataset_id="synthetic", revision=1, samples=(b, a)))
|
|
assert one == two
|