87777975c7
actor/judge prompt eval 闭合: actor 3.0.5 正文强化+baseline(7轮); judge 3.0.14 正文强化+anyTerms同义词补全+baseline(judge-a 13轮/judge-b 33轮); registry/checker 版本对齐+顺序锚点 fail-closed 双保险 W-GOLD-LIVE live_prompt: play-loop 契约开口(request/3+provenance/3+ReferenceAssetRecord/1+Registry/1+迁移清单15条); cheap_verify 切v3+消费对账六闸(只消费active)+生成prompt注入接线(0 active不注入); Node runner playtest-v3.cdp.cjs 升/3+绊线恢复; 真模型回归验向后兼容 full_gate+三批基线: full_gate.py 集成runner串六子门+降级+22测; baseline_gates.py fresh25阈值+historical11预期表(2 needs_human交创始人定标)+shadow20框架+38测 注: registry cheap-system 登记 1.8.0→1.8.1 同步此前在途 cheap-system.md frontmatter 升版(修 pre-commit 版本漂移); cheap_verify/cheap_studio/validate/test_acceptance_v3/test_cheap_service_driver 为在途M含本会话叠加+此前W-AXIS在途(同一线无法hunk分离); 3红测试根因与raw审计未入本提交
464 lines
21 KiB
Python
464 lines
21 KiB
Python
"""三批基线闸门测试：fresh25 阈值边界 + historical11 固定预期表 + shadow20 达标断言（确定性，零模型）。
|
||
|
||
阈值严格按设计档 §5.3/§5.2/§5.4，本测试固化边界（19/25 拒 vs 20/25 过、品类 2/5 拒、成本 ¥1.5/¥15 临界、
|
||
假阳放行 0、needs_human 不自动 accept、tester_error=0、inconclusive≤1、固定六项零漂移、fresh25 硬前置）。
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
|
||
from baseline_gates import ( # noqa: E402
|
||
FRESH25_GENRES, build_shadow_plan, evaluate_fresh25_gate, evaluate_historical_gate,
|
||
evaluate_shadow20_gate, load_historical_expectations, run_shadow_batch,
|
||
)
|
||
|
||
|
||
# ────────────────────────── fresh25 ──────────────────────────
|
||
|
||
def _fresh_rows(accepted_per_genre=(4, 4, 4, 4, 4), *, first_pass=18, repair=0, rescued=0,
|
||
run_cost=1.0, chain_cost=5.0, drop=0, dup_gid=False):
|
||
"""按品类构造 25 局批结果行（gid/genre 与 hard_genre_batch 同 schema）。"""
|
||
rows = []
|
||
idx = 0
|
||
for gi, genre in enumerate(FRESH25_GENRES):
|
||
for rnd in range(1, 6):
|
||
rows.append({
|
||
"gid": f"{genre}-r{rnd}", "genre": genre, "round": rnd,
|
||
"accepted": rnd <= accepted_per_genre[gi],
|
||
"firstPassAccepted": idx < first_pass,
|
||
"repairAttempted": idx < repair,
|
||
"rescuedByRoll": 2 if idx < rescued else None,
|
||
"acceptedAfterRepair": False,
|
||
"outcome": "accept" if rnd <= accepted_per_genre[gi] else "reject",
|
||
"acceptanceCostRmb": run_cost, "parentChainCostRmb": chain_cost,
|
||
})
|
||
idx += 1
|
||
if drop:
|
||
rows = rows[:-drop]
|
||
if dup_gid:
|
||
rows[-1]["gid"] = rows[0]["gid"]
|
||
return rows
|
||
|
||
|
||
def test_fresh25_all_green_passes():
|
||
result = evaluate_fresh25_gate(_fresh_rows())
|
||
assert result["pass"] is True
|
||
assert result["metrics"]["accepted"] == 20
|
||
assert all(c["pass"] for c in result["checks"])
|
||
|
||
|
||
def test_fresh25_boundary_19_of_25_rejected():
|
||
"""accepted 19/25（4,4,4,4,3）→ 总数闸拒；品类 3/5 仍达标，只挂总数一项。"""
|
||
result = evaluate_fresh25_gate(_fresh_rows(accepted_per_genre=(4, 4, 4, 4, 3)))
|
||
assert result["pass"] is False
|
||
by_name = {c["name"]: c for c in result["checks"]}
|
||
assert by_name["acceptedTotal"]["pass"] is False
|
||
assert by_name["acceptedTotal"]["actual"] == "19/25"
|
||
assert by_name["perGenreMin"]["pass"] is True
|
||
|
||
|
||
def test_fresh25_boundary_20_of_25_passes():
|
||
result = evaluate_fresh25_gate(_fresh_rows(accepted_per_genre=(4, 4, 4, 4, 4)))
|
||
by_name = {c["name"]: c for c in result["checks"]}
|
||
assert by_name["acceptedTotal"]["pass"] is True
|
||
assert result["pass"] is True
|
||
|
||
|
||
def test_fresh25_genre_2_of_5_rejected_even_with_20_total():
|
||
"""总数 20/25 达标但某品类 2/5 → 品类闸拒（任一品类不低于 3/5）。"""
|
||
result = evaluate_fresh25_gate(_fresh_rows(accepted_per_genre=(5, 5, 5, 3, 2)))
|
||
assert result["pass"] is False
|
||
by_name = {c["name"]: c for c in result["checks"]}
|
||
assert by_name["acceptedTotal"]["pass"] is True # 总数 20 达标
|
||
assert by_name["perGenreMin"]["pass"] is False
|
||
assert "sim-business" in by_name["perGenreMin"]["detail"]
|
||
|
||
|
||
def test_fresh25_first_pass_boundary():
|
||
"""firstPassAccepted 18 过 / 17 拒。"""
|
||
assert evaluate_fresh25_gate(_fresh_rows(first_pass=18))["pass"] is True
|
||
result = evaluate_fresh25_gate(_fresh_rows(first_pass=17))
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["firstPassAccepted"]["actual"] == "17/25"
|
||
|
||
|
||
def test_fresh25_repair_rate_boundary():
|
||
"""writer repair 启动 5 过 / 6 拒。"""
|
||
assert evaluate_fresh25_gate(_fresh_rows(repair=5))["pass"] is True
|
||
assert evaluate_fresh25_gate(_fresh_rows(repair=6))["pass"] is False
|
||
|
||
|
||
def test_fresh25_rescued_boundary():
|
||
"""rescuedByRoll 5 过 / 6 拒。"""
|
||
assert evaluate_fresh25_gate(_fresh_rows(rescued=5))["pass"] is True
|
||
assert evaluate_fresh25_gate(_fresh_rows(rescued=6))["pass"] is False
|
||
|
||
|
||
def test_fresh25_run_cost_boundary():
|
||
"""单局验收成本 ¥1.5 过 / ¥1.5001 拒。"""
|
||
assert evaluate_fresh25_gate(_fresh_rows(run_cost=1.5))["pass"] is True
|
||
result = evaluate_fresh25_gate(_fresh_rows(run_cost=1.5001))
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["perRunCost"]["pass"] is False
|
||
|
||
|
||
def test_fresh25_chain_cost_boundary():
|
||
"""parentRun 全链 ¥15 过 / ¥15.0001 拒。"""
|
||
assert evaluate_fresh25_gate(_fresh_rows(chain_cost=15.0))["pass"] is True
|
||
assert evaluate_fresh25_gate(_fresh_rows(chain_cost=15.0001))["pass"] is False
|
||
|
||
|
||
def test_fresh25_missing_cost_fails_closed():
|
||
"""缺验收成本 = 无法证明达标 → fail-closed 拒。"""
|
||
rows = _fresh_rows()
|
||
rows[3]["acceptanceCostRmb"] = None
|
||
result = evaluate_fresh25_gate(rows)
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["perRunCost"]["pass"] is False
|
||
|
||
|
||
def test_fresh25_nan_cost_treated_as_missing():
|
||
rows = _fresh_rows()
|
||
rows[0]["parentChainCostRmb"] = float("nan")
|
||
assert evaluate_fresh25_gate(rows)["pass"] is False
|
||
|
||
|
||
def test_fresh25_incomplete_sample_rejected():
|
||
"""24 局（分母不完整）→ 样本闸拒，不能拿残缺基线冒充达标。"""
|
||
result = evaluate_fresh25_gate(_fresh_rows(drop=1))
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["sampleSize"]["pass"] is False
|
||
|
||
|
||
def test_fresh25_duplicate_gid_rejected():
|
||
"""同 gid 重跑洗数字 → 样本闸拒（distinct gid 纪律）。"""
|
||
result = evaluate_fresh25_gate(_fresh_rows(dup_gid=True))
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["sampleSize"]["pass"] is False
|
||
|
||
|
||
def test_fresh25_accepted_after_repair_is_observation_not_numerator():
|
||
"""acceptedAfterRepair 只作修复救回率观测，绝不当成功率分子。"""
|
||
result = evaluate_fresh25_gate(_fresh_rows())
|
||
assert result["metrics"]["acceptedAfterRepair_observation"] == 0
|
||
assert "acceptedAfterRepair" not in {c["name"] for c in result["checks"]} # 不作闸门分子
|
||
|
||
|
||
# ────────────────────────── historical11 ──────────────────────────
|
||
|
||
def _expected_matched_mapping():
|
||
"""与固定预期表逐局相符的重放结果（needs_human 局给 inconclusive，仍挂起待定标）。"""
|
||
return {
|
||
"narrative-r1": "accept", "narrative-r2": "accept", "narrative-r3": "accept",
|
||
"trpg-r1": "reject", "puzzle-r1": "inconclusive", "trpg-r2": "reject",
|
||
"heritage-r2": "inconclusive", "sim-business-r2": "reject",
|
||
"puzzle-r2": "reject",
|
||
"heritage-r1": "inconclusive", "sim-business-r1": "inconclusive",
|
||
}
|
||
|
||
|
||
def _rows_from(mapping):
|
||
return [{"gid": gid, "outcome": outcome} for gid, outcome in mapping.items()]
|
||
|
||
|
||
def test_expectations_fixture_is_eleven_well_formed_games():
|
||
data = load_historical_expectations()
|
||
rows = data["expectations"]
|
||
assert len(rows) == 11
|
||
assert {r["expected"] for r in rows} <= {"accept", "reject", "not_accept", "needs_human"}
|
||
categories = {r["category"]: sum(1 for x in rows if x["category"] == r["category"]) for r in rows}
|
||
assert categories["positive_control"] == 3 # 3 narrative 正例
|
||
assert categories["known_false_positive"] == 5 # 5 旧假阳
|
||
assert categories["true_bug"] == 1 # puzzle-r2
|
||
assert categories["suspected_false_negative"] == 2 # 2 疑似假阴
|
||
|
||
|
||
def test_historical_matched_but_blocked_on_human_calibration():
|
||
"""逐局相符 + needs_human 未定标 → 闸门不自动 PASS，挂起 2 局交创始人（红线）。"""
|
||
result = evaluate_historical_gate(_rows_from(_expected_matched_mapping()))
|
||
assert result["pass"] is False
|
||
assert result["falsePositiveReleased"] == []
|
||
assert sorted(result["blockedOnHumanCalibration"]) == ["heritage-r1", "sim-business-r1"]
|
||
by_name = {c["name"]: c for c in result["checks"]}
|
||
assert by_name["falsePositiveRelease"]["pass"] is True
|
||
assert by_name["perGameExpectation"]["pass"] is True
|
||
assert by_name["humanCalibrationSettled"]["pass"] is False
|
||
|
||
|
||
def test_historical_known_false_positive_accept_is_release_fail():
|
||
"""旧假阳无新硬证被 accept = 确认假阳放行，闸门红线。"""
|
||
mapping = _expected_matched_mapping()
|
||
mapping["sim-business-r2"] = "accept"
|
||
result = evaluate_historical_gate(_rows_from(mapping))
|
||
assert result["pass"] is False
|
||
assert result["falsePositiveReleased"] == ["sim-business-r2"]
|
||
assert {c["name"]: c for c in result["checks"]}["falsePositiveRelease"]["pass"] is False
|
||
|
||
|
||
def test_historical_true_bug_accept_is_release_fail():
|
||
"""puzzle-r2 真 bug 被 accept = 假阳放行。"""
|
||
mapping = _expected_matched_mapping()
|
||
mapping["puzzle-r2"] = "accept"
|
||
result = evaluate_historical_gate(_rows_from(mapping))
|
||
assert "puzzle-r2" in result["falsePositiveReleased"]
|
||
|
||
|
||
def test_historical_true_bug_inconclusive_is_mismatch():
|
||
"""puzzle-r2 固定预期 reject，inconclusive 与预期不符（查验收器为何拿不到硬证）。"""
|
||
mapping = _expected_matched_mapping()
|
||
mapping["puzzle-r2"] = "inconclusive"
|
||
result = evaluate_historical_gate(_rows_from(mapping))
|
||
assert result["pass"] is False
|
||
by_name = {c["name"]: c for c in result["checks"]}
|
||
assert by_name["perGameExpectation"]["pass"] is False
|
||
assert "puzzle-r2" in by_name["perGameExpectation"]["detail"]
|
||
|
||
|
||
def test_historical_positive_control_must_still_accept():
|
||
"""narrative 正例应保留：v3 判 inconclusive = 验收器回退，逐局预期不符。"""
|
||
mapping = _expected_matched_mapping()
|
||
mapping["narrative-r1"] = "inconclusive"
|
||
result = evaluate_historical_gate(_rows_from(mapping))
|
||
assert result["pass"] is False
|
||
assert "narrative-r1" in {c["name"]: c for c in result["checks"]}["perGameExpectation"]["detail"]
|
||
|
||
|
||
def test_historical_needs_human_accept_is_pre_calibration_fail():
|
||
"""疑似假阴未定标即 accept → 定标前不得自动接受（红线）。"""
|
||
mapping = _expected_matched_mapping()
|
||
mapping["heritage-r1"] = "accept"
|
||
result = evaluate_historical_gate(_rows_from(mapping))
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["preCalibrationAccept"]["pass"] is False
|
||
|
||
|
||
def test_historical_tester_error_is_mismatch_not_evidence():
|
||
"""tester_error 不构成证据 → 逐局不符，需重跑。"""
|
||
mapping = _expected_matched_mapping()
|
||
mapping["trpg-r1"] = "tester_error"
|
||
result = evaluate_historical_gate(_rows_from(mapping))
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["perGameExpectation"]["pass"] is False
|
||
|
||
|
||
def test_historical_missing_game_is_coverage_fail():
|
||
mapping = _expected_matched_mapping()
|
||
del mapping["puzzle-r1"]
|
||
result = evaluate_historical_gate(_rows_from(mapping))
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["coverage"]["pass"] is False
|
||
|
||
|
||
def test_historical_fully_calibrated_can_auto_pass():
|
||
"""创始人定标后（needs_human 改为固定预期）+ 逐局相符 → 闸门可自动 PASS。"""
|
||
data = load_historical_expectations()
|
||
calibrated = json.loads(json.dumps(data))
|
||
for row in calibrated["expectations"]:
|
||
if row["expected"] == "needs_human":
|
||
row["expected"] = "accept" # 真人定标结论：其实是好游戏
|
||
row["category"] = "calibrated"
|
||
mapping = _expected_matched_mapping()
|
||
mapping["heritage-r1"] = "accept"
|
||
mapping["sim-business-r1"] = "accept"
|
||
result = evaluate_historical_gate(_rows_from(mapping), expectations=calibrated,
|
||
fresh25_gate={"pass": True})
|
||
assert result["pass"] is True
|
||
assert result["blockedOnHumanCalibration"] == []
|
||
|
||
|
||
def test_historical_chain_blocked_by_failed_fresh25():
|
||
"""链式校验：fresh25 未过 → warn + PASS 受阻（顺序建议，指标仍评估）。"""
|
||
data = load_historical_expectations()
|
||
calibrated = json.loads(json.dumps(data))
|
||
for row in calibrated["expectations"]:
|
||
if row["expected"] == "needs_human":
|
||
row["expected"] = "reject"
|
||
mapping = _expected_matched_mapping()
|
||
mapping["heritage-r1"] = "reject"
|
||
mapping["sim-business-r1"] = "reject"
|
||
result = evaluate_historical_gate(_rows_from(mapping), expectations=calibrated,
|
||
fresh25_gate={"pass": False})
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["chainFresh25"]["pass"] is False
|
||
assert any("fresh25" in w for w in result["warnings"])
|
||
|
||
|
||
# ────────────────────────── shadow20 ──────────────────────────
|
||
|
||
_IDENTITY = {"commitHash": "c0ffee00", "chromeVersion": "126.0.6478.183",
|
||
"actorModel": "MiniMax-M3", "judgeModel": "MiniMax-M3",
|
||
"promptVersion": "3.0.14", "configSnapshotHash": "cfg-snap-01"}
|
||
|
||
|
||
def _shadow_rows(n=20, **overrides_by_index):
|
||
rows = []
|
||
for i in range(n):
|
||
row = {"gid": f"shadow20-{i + 1:02d}", "outcome": "accept",
|
||
"proofComplete": True, "confirmedFalsePositive": False,
|
||
"problems": [], "contradictions": [], "proofMissing": False,
|
||
"rescued": False, "discrepancy": False, "humanReviewed": True,
|
||
"identity": dict(_IDENTITY)}
|
||
row.update(overrides_by_index.get(i) or {})
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
_F25_PASS = {"pass": True}
|
||
_F25_FAIL = {"pass": False}
|
||
|
||
|
||
def test_shadow20_all_green_passes_with_fresh25_prereq():
|
||
result = evaluate_shadow20_gate(_shadow_rows(), fresh25_gate=_F25_PASS)
|
||
assert result["pass"] is True
|
||
assert all(c["pass"] for c in result["checks"])
|
||
|
||
|
||
def test_shadow20_fresh25_prereq_hard_required():
|
||
"""fresh25 未提供 → 硬前置无法验证，PASS 阻断（§5.4 顺序硬要求）。"""
|
||
result = evaluate_shadow20_gate(_shadow_rows(), fresh25_gate=None)
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["prerequisiteFresh25"]["pass"] is False
|
||
result2 = evaluate_shadow20_gate(_shadow_rows(), fresh25_gate=_F25_FAIL)
|
||
assert result2["pass"] is False
|
||
assert {c["name"]: c for c in result2["checks"]}["prerequisiteFresh25"]["actual"] == "fresh25 未达标"
|
||
|
||
|
||
def test_shadow20_nineteen_samples_rejected():
|
||
result = evaluate_shadow20_gate(_shadow_rows(n=19), fresh25_gate=_F25_PASS)
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["sampleSize"]["pass"] is False
|
||
|
||
|
||
def test_shadow20_single_tester_error_rejected():
|
||
"""20 局口径 <5% → tester_error 必须为 0：1 个即拒。"""
|
||
rows = _shadow_rows()
|
||
rows[7]["outcome"] = "tester_error"
|
||
rows[7]["humanReviewed"] = True
|
||
result = evaluate_shadow20_gate(rows, fresh25_gate=_F25_PASS)
|
||
assert result["pass"] is False
|
||
check = {c["name"]: c for c in result["checks"]}["testerError"]
|
||
assert check["pass"] is False and check["actual"] == "1/20"
|
||
|
||
|
||
def test_shadow20_inconclusive_boundary_one_ok_two_rejected():
|
||
"""<10% → inconclusive 最多 1：1 个过、2 个拒。"""
|
||
rows = _shadow_rows()
|
||
rows[0]["outcome"] = "inconclusive"
|
||
assert evaluate_shadow20_gate(rows, fresh25_gate=_F25_PASS)["pass"] is True
|
||
rows[1]["outcome"] = "inconclusive"
|
||
result = evaluate_shadow20_gate(rows, fresh25_gate=_F25_PASS)
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["inconclusive"]["actual"] == "2/20"
|
||
|
||
|
||
def test_shadow20_confirmed_false_positive_rejected():
|
||
rows = _shadow_rows()
|
||
rows[3]["confirmedFalsePositive"] = True
|
||
result = evaluate_shadow20_gate(rows, fresh25_gate=_F25_PASS)
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["confirmedFalsePositives"]["detail"] == "shadow20-04"
|
||
|
||
|
||
def test_shadow20_accepted_proof_incomplete_rejected():
|
||
"""accepted proof 完整率必须 100%：一局不完整即拒。"""
|
||
rows = _shadow_rows()
|
||
rows[5]["proofComplete"] = False
|
||
result = evaluate_shadow20_gate(rows, fresh25_gate=_F25_PASS)
|
||
assert result["pass"] is False
|
||
assert "shadow20-06" in {c["name"]: c for c in result["checks"]}["proofCompleteOnAccepted"]["detail"]
|
||
|
||
|
||
def test_shadow20_accepted_with_conflict_rejected():
|
||
"""accepted 与 problems/缺证/矛盾共存 0：三种脏状态各测一遍。"""
|
||
for field, value in (("problems", ["结论与截图矛盾"]), ("contradictions", ["事件序倒序"]),
|
||
("proofMissing", True)):
|
||
rows = _shadow_rows()
|
||
rows[2][field] = value
|
||
result = evaluate_shadow20_gate(rows, fresh25_gate=_F25_PASS)
|
||
assert result["pass"] is False, f"{field} 共存未拦"
|
||
assert {c["name"]: c for c in result["checks"]}["acceptedWithoutConflict"]["pass"] is False
|
||
|
||
|
||
def test_shadow20_identity_drift_rejected():
|
||
"""固定六项漂移即拒（shadow 可比性前提）。"""
|
||
rows = _shadow_rows()
|
||
rows[9]["identity"] = {**_IDENTITY, "commitHash": "drifted1"}
|
||
result = evaluate_shadow20_gate(rows, fresh25_gate=_F25_PASS)
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["identityFixed"]["pass"] is False
|
||
|
||
|
||
def test_shadow20_identity_drift_against_plan_rejected():
|
||
plan = build_shadow_plan([f"prompt-{i}" for i in range(20)], commit_hash=_IDENTITY["commitHash"],
|
||
chrome_version=_IDENTITY["chromeVersion"], actor_model=_IDENTITY["actorModel"],
|
||
judge_model=_IDENTITY["judgeModel"], prompt_version=_IDENTITY["promptVersion"],
|
||
config_snapshot_hash=_IDENTITY["configSnapshotHash"])
|
||
rows = _shadow_rows()
|
||
rows[0]["identity"] = {**_IDENTITY, "promptVersion": "3.0.13"} # 与 plan 不一致
|
||
result = evaluate_shadow20_gate(rows, fresh25_gate=_F25_PASS, plan=plan)
|
||
assert result["pass"] is False
|
||
|
||
|
||
def test_shadow20_human_review_coverage_enforced():
|
||
"""分歧/reject/inconclusive/tester_error 必查 + 普通 accept 至少抽 5。"""
|
||
rows = _shadow_rows()
|
||
rows[4]["outcome"] = "reject"
|
||
rows[4]["humanReviewed"] = False # 必查局未复核
|
||
result = evaluate_shadow20_gate(rows, fresh25_gate=_F25_PASS)
|
||
assert result["pass"] is False
|
||
assert {c["name"]: c for c in result["checks"]}["humanReviewCoverage"]["pass"] is False
|
||
|
||
# accept 抽查不足 5：6 个 accept 只复核 1 个
|
||
rows2 = _shadow_rows()
|
||
for i in range(14):
|
||
rows2[i]["outcome"] = "reject" # 14 reject（全已复核）+ 6 accept
|
||
for i in range(14, 19):
|
||
rows2[i]["humanReviewed"] = False # 6 accept 中 5 个未复核 → 只 1 个 < 5
|
||
result2 = evaluate_shadow20_gate(rows2, fresh25_gate=_F25_PASS)
|
||
assert result2["pass"] is False
|
||
|
||
|
||
def test_shadow20_plan_builder_validates_fixed_identity():
|
||
prompts = [f"prompt-{i}" for i in range(20)]
|
||
plan = build_shadow_plan(prompts, commit_hash="c1", chrome_version="126",
|
||
actor_model="MiniMax-M3", judge_model="MiniMax-M3",
|
||
prompt_version="3.0.14", config_snapshot_hash="cfg1")
|
||
assert plan["acceptanceMode"] == "v3_shadow"
|
||
assert len(plan["prompts"]) == 20
|
||
with pytest.raises(ValueError, match="≥20"):
|
||
build_shadow_plan(prompts[:19], commit_hash="c1", chrome_version="126",
|
||
actor_model="m", judge_model="m", prompt_version="p", config_snapshot_hash="c")
|
||
with pytest.raises(ValueError, match="不得为空"):
|
||
build_shadow_plan(prompts, commit_hash="", chrome_version="126",
|
||
actor_model="m", judge_model="m", prompt_version="p", config_snapshot_hash="c")
|
||
|
||
|
||
def test_shadow20_runner_framework_stamps_identity_serially():
|
||
"""runner 框架：串行调用注入的真跑 callable，每局盖固定身份戳（罐头 run_one，零模型）。"""
|
||
plan = build_shadow_plan([f"p{i}" for i in range(20)], commit_hash="c1", chrome_version="126",
|
||
actor_model="MiniMax-M3", judge_model="MiniMax-M3",
|
||
prompt_version="3.0.14", config_snapshot_hash="cfg1")
|
||
calls = []
|
||
|
||
async def _fake_run_one(prompt, identity):
|
||
calls.append(prompt)
|
||
return {"outcome": "accept", "proofComplete": True, "humanReviewed": True}
|
||
|
||
rows = asyncio.run(run_shadow_batch(plan, run_one=_fake_run_one))
|
||
assert len(rows) == 20 and len(calls) == 20
|
||
assert all(row["identity"]["commitHash"] == "c1" for row in rows)
|
||
assert [row["gid"] for row in rows[:2]] == ["shadow20-01", "shadow20-02"]
|
||
# 框架产出的行直接可进闸门（全绿 + fresh25 前置 → PASS）
|
||
assert evaluate_shadow20_gate(rows, fresh25_gate=_F25_PASS, plan=plan)["pass"] is True
|
||
|
||
|
||
def test_shadow20_runner_rejects_malformed_plan():
|
||
with pytest.raises(ValueError):
|
||
asyncio.run(run_shadow_batch({"prompts": "not-a-list"}, run_one=lambda p, i: {}))
|