87777975c7
actor/judge prompt eval 闭合: actor 3.0.5 正文强化+baseline(7轮); judge 3.0.14 正文强化+anyTerms同义词补全+baseline(judge-a 13轮/judge-b 33轮); registry/checker 版本对齐+顺序锚点 fail-closed 双保险 W-GOLD-LIVE live_prompt: play-loop 契约开口(request/3+provenance/3+ReferenceAssetRecord/1+Registry/1+迁移清单15条); cheap_verify 切v3+消费对账六闸(只消费active)+生成prompt注入接线(0 active不注入); Node runner playtest-v3.cdp.cjs 升/3+绊线恢复; 真模型回归验向后兼容 full_gate+三批基线: full_gate.py 集成runner串六子门+降级+22测; baseline_gates.py fresh25阈值+historical11预期表(2 needs_human交创始人定标)+shadow20框架+38测 注: registry cheap-system 登记 1.8.0→1.8.1 同步此前在途 cheap-system.md frontmatter 升版(修 pre-commit 版本漂移); cheap_verify/cheap_studio/validate/test_acceptance_v3/test_cheap_service_driver 为在途M含本会话叠加+此前W-AXIS在途(同一线无法hunk分离); 3红测试根因与raw审计未入本提交
478 lines
28 KiB
Python
478 lines
28 KiB
Python
"""baseline_gates.py — 生成线验收 v3 三批基线闸门（W-AXIS 收口 R1）。
|
||
|
||
把设计档 §5.2/§5.3/§5.4 的三批基线达标阈值落成机器断言，输入批结果、输出 PASS/FAIL + 逐项明细：
|
||
|
||
· fresh25（§5.3 波 3 新基线）：五品类各五局共 25 distinct gid；accepted≥20/25 且每品类≥3/5、
|
||
firstPassAccepted≥18/25、writer repair 启动率≤5/25、rescuedByRoll≤5/25、单局验收成本≤¥1.5、
|
||
parentRun 全链≤¥15。分子只认 row.accepted（硬证完整 + 双 Judge 共识 + finalPostguard 全绿的最终权威），
|
||
acceptedAfterRepair 只作修复救回率观测，绝不作成功率分子；inconclusive/tester_error 单列不剔出分母。
|
||
· historical11（§5.2 固定预期表）：假阳放行 0 + 逐局预期相符 + needs_human 项定标前不得自动 accept
|
||
（reject/inconclusive 挂起等创始人定标，闸门不自动 PASS）。预期表 fixture =
|
||
contracts/play-loop/historical-11-expectations.json。
|
||
· shadow20（§5.4 生产分布 shadow）：proof 完整率 100% + 确认假阳 0 + accepted 与 problems/缺证/矛盾共存 0
|
||
+ tester_error=0（20 局口径 <5%）+ inconclusive≤1（<10%）+ 人工复核覆盖；固定六项（commit/Chrome/
|
||
Actor+Judge 模型/prompt 版本/配置快照/人工标签）漂移即 fail；fresh25 达标为硬前置（§5.4「fresh 25 达标后」）。
|
||
|
||
三批依赖顺序在代码里是可选链式校验：fresh25 结果可传入 historical/shadow20 闸门——historical 收到未过的
|
||
fresh25 时 warn 并阻断 PASS（顺序是执行建议，不阻断单跑的指标评估）；shadow20 按 §5.4 硬查前置。
|
||
本模块纯函数 + 罐头单测，零模型零 I/O（除读预期表 fixture）；真跑留 mini-desktop。
|
||
"""
|
||
|
||
import json
|
||
from pathlib import Path
|
||
|
||
# ────────────────────────── fresh25（设计档 §5.3）──────────────────────────
|
||
|
||
# 五个【有 per-genre 模板】的已覆盖品类，与 hard_genre_batch.COVERED_BRIEFS 同集（硬编码保持闸门自包含）。
|
||
FRESH25_GENRES = ("narrative", "trpg", "heritage", "puzzle", "sim-business")
|
||
|
||
# 阈值严格按设计档 §5.3，不放松。
|
||
FRESH25_THRESHOLDS = {
|
||
"totalGames": 25, # 五品类各五局共 25 distinct gid
|
||
"gamesPerGenre": 5,
|
||
"minAccepted": 20, # MVP 终态成功率按 accepted 计 ≥ 20/25
|
||
"minAcceptedPerGenre": 3, # 任一品类不低于 3/5
|
||
"minFirstPassAccepted": 18, # firstPassAccepted ≥ 18/25（最多二掷、尚未 repair 的终态）
|
||
"maxRepairAttempted": 5, # writer repair 启动率 ≤ 5/25
|
||
"maxRescuedByRoll": 5, # rescuedByRoll（二掷救回）≤ 5/25
|
||
"maxRunAcceptanceCostRmb": 1.5, # 单局验收成本 ≤ ¥1.5
|
||
"maxParentChainCostRmb": 15.0, # parentRun 全链总成本 ≤ ¥15 硬地板
|
||
}
|
||
|
||
_COST_EPS = 1e-9 # 金额比较容差（与 _final_postguard_v3 的 +1e-9 口径一致）
|
||
|
||
|
||
def evaluate_fresh25_gate(rows, *, thresholds=None) -> dict:
|
||
"""fresh25 达标断言：输入批结果行（hard_genre_batch 行 schema），输出 {pass, checks, metrics, warnings}。
|
||
|
||
row 消费字段：gid/genre/accepted/firstPassAccepted/repairAttempted/rescuedByRoll/
|
||
acceptanceCostRmb/parentChainCostRmb/outcome/acceptedAfterRepair。
|
||
"""
|
||
th = dict(FRESH25_THRESHOLDS)
|
||
if thresholds:
|
||
th.update(thresholds)
|
||
rows = [r for r in (rows or []) if isinstance(r, dict)]
|
||
checks = []
|
||
warnings = []
|
||
|
||
def check(name, ok, actual, required, detail=""):
|
||
checks.append({"name": name, "pass": bool(ok), "actual": actual, "required": required, "detail": detail})
|
||
|
||
# ── 样本完整性：25 distinct gid、恰好 5 品类、每品类 5 局（分母不完整 → 基线不成立，直接 fail）。
|
||
gids = [r.get("gid") for r in rows]
|
||
dup_gids = sorted({g for g in gids if gids.count(g) > 1 and g is not None})
|
||
by_genre = {}
|
||
for r in rows:
|
||
by_genre.setdefault(r.get("genre"), []).append(r)
|
||
genre_counts = {g: len(by_genre.get(g) or []) for g in FRESH25_GENRES}
|
||
extra_genres = sorted(g for g in by_genre if g not in FRESH25_GENRES)
|
||
size_ok = (len(rows) == th["totalGames"] and not dup_gids and not extra_genres
|
||
and all(c == th["gamesPerGenre"] for c in genre_counts.values()))
|
||
check("sampleSize", size_ok, f"{len(rows)} 局/品类分布 {genre_counts}",
|
||
f"{th['totalGames']} 局（每品类 {th['gamesPerGenre']}，gid 不重复）",
|
||
f"重复 gid={dup_gids}" if dup_gids else (f"非覆盖品类={extra_genres}" if extra_genres else ""))
|
||
|
||
# ── accepted 总数（分子 = 硬证完整 + 双 Judge 共识 + finalPostguard 全绿的最终权威）。
|
||
accepted = [r for r in rows if r.get("accepted") is True]
|
||
check("acceptedTotal", len(accepted) >= th["minAccepted"], f"{len(accepted)}/{len(rows)}",
|
||
f"≥ {th['minAccepted']}/{th['totalGames']}")
|
||
|
||
# ── 每品类 accepted ≥ 3/5。
|
||
per_genre_accepted = {g: sum(1 for r in by_genre.get(g) or [] if r.get("accepted") is True)
|
||
for g in FRESH25_GENRES}
|
||
failing_genres = sorted(g for g, c in per_genre_accepted.items() if c < th["minAcceptedPerGenre"])
|
||
check("perGenreMin", not failing_genres, per_genre_accepted,
|
||
f"每品类 ≥ {th['minAcceptedPerGenre']}/{th['gamesPerGenre']}",
|
||
f"未达标品类={failing_genres}" if failing_genres else "")
|
||
|
||
# ── firstPassAccepted ≥ 18/25（首轮直接成功 + 二掷内成功，repair 前的终态）。
|
||
first_pass = sum(1 for r in rows if r.get("firstPassAccepted") is True)
|
||
check("firstPassAccepted", first_pass >= th["minFirstPassAccepted"], f"{first_pass}/{len(rows)}",
|
||
f"≥ {th['minFirstPassAccepted']}/{th['totalGames']}")
|
||
|
||
# ── writer repair 启动率 ≤ 5/25。
|
||
repair_attempted = sum(1 for r in rows if r.get("repairAttempted") is True)
|
||
check("repairRate", repair_attempted <= th["maxRepairAttempted"], f"{repair_attempted}/{len(rows)}",
|
||
f"≤ {th['maxRepairAttempted']}/{th['totalGames']}")
|
||
|
||
# ── rescuedByRoll（二掷救回，取值 2）≤ 5/25；与 hard_genre_batch 主汇总口径一致。
|
||
rescued = sum(1 for r in rows if r.get("rescuedByRoll") == 2)
|
||
check("rescuedByRollRate", rescued <= th["maxRescuedByRoll"], f"{rescued}/{len(rows)}",
|
||
f"≤ {th['maxRescuedByRoll']}/{th['totalGames']}")
|
||
|
||
# ── 单局验收成本 ≤ ¥1.5（缺成本 = 无法证明达标 → fail-closed 该检查）。
|
||
run_costs = []
|
||
missing_run_cost = []
|
||
for r in rows:
|
||
cost = r.get("acceptanceCostRmb")
|
||
if isinstance(cost, (int, float)) and cost == cost: # 拒 NaN（cost != cost 即 NaN）
|
||
run_costs.append((r.get("gid"), float(cost)))
|
||
else:
|
||
missing_run_cost.append(r.get("gid"))
|
||
over_run = [(g, c) for g, c in run_costs if c > th["maxRunAcceptanceCostRmb"] + _COST_EPS]
|
||
check("perRunCost", rows != [] and not over_run and not missing_run_cost,
|
||
f"max=¥{max((c for _, c in run_costs), default=0.0):.5f}/缺 {len(missing_run_cost)} 局",
|
||
f"每局 ≤ ¥{th['maxRunAcceptanceCostRmb']}",
|
||
f"超限={over_run[:3]}" if over_run else (f"缺成本 gid={missing_run_cost[:3]}" if missing_run_cost else ""))
|
||
|
||
# ── parentRun 全链总成本 ≤ ¥15（repair 历史 + writer + 验收的不重不漏权威总成本）。
|
||
chain_costs = []
|
||
missing_chain_cost = []
|
||
for r in rows:
|
||
cost = r.get("parentChainCostRmb")
|
||
if isinstance(cost, (int, float)) and cost == cost:
|
||
chain_costs.append((r.get("gid"), float(cost)))
|
||
else:
|
||
missing_chain_cost.append(r.get("gid"))
|
||
over_chain = [(g, c) for g, c in chain_costs if c > th["maxParentChainCostRmb"] + _COST_EPS]
|
||
check("parentChainCost", rows != [] and not over_chain and not missing_chain_cost,
|
||
f"max=¥{max((c for _, c in chain_costs), default=0.0):.5f}/缺 {len(missing_chain_cost)} 局",
|
||
f"全链 ≤ ¥{th['maxParentChainCostRmb']}",
|
||
f"超限={over_chain[:3]}" if over_chain else
|
||
(f"缺成本 gid={missing_chain_cost[:3]}" if missing_chain_cost else ""))
|
||
|
||
# ── 观测项（不作闸门）：四态分布 + acceptedAfterRepair（只统计修复救回率，绝不当成功率分子）。
|
||
outcomes = {name: sum(1 for r in rows if r.get("outcome") == name)
|
||
for name in ("accept", "reject", "inconclusive", "tester_error")}
|
||
repaired = sum(1 for r in rows if r.get("acceptedAfterRepair") is True)
|
||
if outcomes["inconclusive"] or outcomes["tester_error"]:
|
||
warnings.append(f"inconclusive={outcomes['inconclusive']} tester_error={outcomes['tester_error']} "
|
||
f"单列观测，未并入游戏失败也未剔出分母")
|
||
metrics = {
|
||
"accepted": len(accepted), "firstPassAccepted": first_pass,
|
||
"repairAttempted": repair_attempted, "rescuedByRoll": rescued,
|
||
"perGenreAccepted": per_genre_accepted, "outcomes": outcomes,
|
||
"acceptedAfterRepair_observation": repaired, # 修复救回率观测，不是成功率分子
|
||
"maxRunAcceptanceCostRmb": max((c for _, c in run_costs), default=None),
|
||
"maxParentChainCostRmb": max((c for _, c in chain_costs), default=None),
|
||
}
|
||
return {"pass": all(c["pass"] for c in checks), "checks": checks,
|
||
"metrics": metrics, "warnings": warnings, "thresholds": th}
|
||
|
||
|
||
# ────────────────────────── historical11（设计档 §5.2）──────────────────────────
|
||
|
||
# 预期表 fixture：11 局固定预期（3 narrative 正例 / 5 旧假阳 / puzzle-r2 真 bug / 2 疑似假阴）。
|
||
_HISTORICAL_EXPECTATIONS_PATH = (Path(__file__).resolve().parents[1]
|
||
/ "contracts" / "play-loop" / "historical-11-expectations.json")
|
||
|
||
_EXPECTED_VALUES = ("accept", "reject", "not_accept", "needs_human")
|
||
|
||
|
||
def load_historical_expectations(path=None) -> dict:
|
||
"""读并校验 11 局固定预期表；结构非法直接抛 ValueError（闸门 fixture 必须机器可信）。"""
|
||
p = Path(path) if path else _HISTORICAL_EXPECTATIONS_PATH
|
||
data = json.loads(p.read_text(encoding="utf-8"))
|
||
rows = data.get("expectations")
|
||
if not isinstance(rows, list) or len(rows) != 11:
|
||
raise ValueError(f"historical 预期表必须恰有 11 局，实际 {len(rows) if isinstance(rows, list) else 'N/A'}")
|
||
gids = [r.get("gid") for r in rows if isinstance(r, dict)]
|
||
if len(set(gids)) != 11 or any(not isinstance(g, str) or not g for g in gids):
|
||
raise ValueError("historical 预期表 gid 必须 11 个非空且不重复")
|
||
for r in rows:
|
||
if r.get("expected") not in _EXPECTED_VALUES:
|
||
raise ValueError(f"historical 预期 {r.get('gid')} expected 非法:{r.get('expected')}")
|
||
return data
|
||
|
||
|
||
def evaluate_historical_gate(replay_rows, *, expectations=None, expectations_path=None,
|
||
fresh25_gate=None) -> dict:
|
||
"""historical replay 闸门：假阳放行 0 + 逐局预期相符 + needs_human 定标前不自动 accept。
|
||
|
||
replay_rows：[{gid, outcome}]，outcome ∈ accept/reject/inconclusive/tester_error（historical_replay 产物）。
|
||
fresh25_gate：可选链式校验——提供且未过 → warn 并阻断 PASS（顺序是执行建议，不阻断指标评估）；None → warn 单跑。
|
||
"""
|
||
data = expectations if isinstance(expectations, dict) else load_historical_expectations(expectations_path)
|
||
exp_by_gid = {r["gid"]: r for r in data["expectations"]}
|
||
rows = [r for r in (replay_rows or []) if isinstance(r, dict)]
|
||
replay_by_gid = {}
|
||
for r in rows:
|
||
gid = r.get("gid")
|
||
if gid in exp_by_gid and gid not in replay_by_gid:
|
||
replay_by_gid[gid] = r
|
||
|
||
warnings = []
|
||
per_game = []
|
||
false_positive_released = [] # critical 局被 accept（已知坏例放行，闸门红线 = 0）
|
||
mismatches = [] # 逐局预期不符
|
||
pre_calibration_accepts = [] # needs_human 局定标前被 accept（红线）
|
||
pending_human = [] # needs_human 局待定标（挂起，阻断自动 PASS，交创始人）
|
||
|
||
extra = sorted({r.get("gid") for r in rows if r.get("gid") not in exp_by_gid})
|
||
if extra:
|
||
warnings.append(f"重放含预期表外 gid（忽略不计）:{extra}")
|
||
|
||
for gid, exp in exp_by_gid.items():
|
||
expected = exp["expected"]
|
||
row = replay_by_gid.get(gid)
|
||
if row is None:
|
||
mismatches.append({"gid": gid, "expected": expected, "actual": None, "reason": "重放缺该局"})
|
||
per_game.append({"gid": gid, "expected": expected, "actual": None, "match": False})
|
||
continue
|
||
outcome = row.get("outcome")
|
||
match = False
|
||
reason = ""
|
||
if expected == "accept":
|
||
match = outcome == "accept"
|
||
if not match:
|
||
reason = "正例应保留（accept），实际未 accept——验收器可能回退"
|
||
elif expected == "not_accept":
|
||
if outcome == "accept":
|
||
false_positive_released.append(gid)
|
||
reason = "旧假阳无新硬证被 accept = 确认假阳放行"
|
||
elif outcome in ("reject", "inconclusive"):
|
||
match = True
|
||
else:
|
||
reason = f"tester_error 不构成证据（{outcome}），需重跑"
|
||
elif expected == "reject":
|
||
if outcome == "accept":
|
||
false_positive_released.append(gid)
|
||
reason = "真 bug 局被 accept = 假阳放行"
|
||
elif outcome == "reject":
|
||
match = True
|
||
else:
|
||
reason = f"固定预期 reject，实际 {outcome} 不符——查验收器为何拿不到硬证"
|
||
elif expected == "needs_human":
|
||
if outcome == "accept":
|
||
pre_calibration_accepts.append(gid)
|
||
reason = "疑似假阴未定标即 accept——定标前不得自动接受"
|
||
elif outcome in ("reject", "inconclusive", "tester_error"):
|
||
pending_human.append(gid)
|
||
reason = "挂起等真人真浏览器定标（不计自动 accept，也不自动 PASS）"
|
||
else:
|
||
reason = f"未知 outcome:{outcome}"
|
||
if not match and gid not in pending_human:
|
||
mismatches.append({"gid": gid, "expected": expected, "actual": outcome, "reason": reason})
|
||
per_game.append({"gid": gid, "expected": expected, "actual": outcome,
|
||
"match": match, "pendingHuman": gid in pending_human, "detail": reason})
|
||
|
||
# ── 链式校验（可选）：fresh25 未过 → warn + 阻断 PASS；未提供 → warn 单跑。
|
||
chain_blocked = False
|
||
if fresh25_gate is None:
|
||
warnings.append("未提供 fresh25 闸门结果：单跑模式（设计档建议顺序 fresh25 → historical11 → shadow20）")
|
||
elif not isinstance(fresh25_gate, dict) or fresh25_gate.get("pass") is not True:
|
||
chain_blocked = True
|
||
warnings.append("链式校验：fresh25 未达标 → historical 闸门 PASS 受阻（执行顺序建议，不阻断指标评估）")
|
||
|
||
checks = [
|
||
{"name": "coverage", "pass": len(replay_by_gid) == 11,
|
||
"actual": f"{len(replay_by_gid)}/11", "required": "11 局全部重放",
|
||
"detail": ",".join(m["gid"] for m in mismatches if m["actual"] is None)},
|
||
{"name": "falsePositiveRelease", "pass": not false_positive_released,
|
||
"actual": len(false_positive_released), "required": "0（已知坏例放行数为 0）",
|
||
"detail": ",".join(false_positive_released)},
|
||
{"name": "preCalibrationAccept", "pass": not pre_calibration_accepts,
|
||
"actual": len(pre_calibration_accepts), "required": "0（needs_human 定标前不得 accept）",
|
||
"detail": ",".join(pre_calibration_accepts)},
|
||
{"name": "perGameExpectation", "pass": not mismatches,
|
||
"actual": f"{11 - len(mismatches)}/11 相符", "required": "逐局固定预期相符",
|
||
"detail": ";".join(f"{m['gid']}:{m['reason']}" for m in mismatches if m["actual"] is not None)},
|
||
{"name": "humanCalibrationSettled", "pass": not pending_human,
|
||
"actual": f"{len(pending_human)} 局待定标", "required": "0（疑似假阴须先真人定标）",
|
||
"detail": ",".join(pending_human)},
|
||
{"name": "chainFresh25", "pass": not chain_blocked,
|
||
"actual": "受阻" if chain_blocked else "通过/单跑", "required": "fresh25 达标（提供时）"},
|
||
]
|
||
return {"pass": all(c["pass"] for c in checks), "checks": checks, "perGame": per_game,
|
||
"blockedOnHumanCalibration": pending_human,
|
||
"falsePositiveReleased": false_positive_released,
|
||
"warnings": warnings}
|
||
|
||
|
||
# ────────────────────────── shadow20（设计档 §5.4）──────────────────────────
|
||
|
||
SHADOW20_MIN_SAMPLES = 20 # 「连续不少于 20 个真实生产 prompt」
|
||
|
||
# 固定六项：代码 commit / Chrome 版本 / Actor 模型 / Judge 模型 / prompt 版本 / 配置快照（+ 人工标签占位）。
|
||
SHADOW20_IDENTITY_FIELDS = ("commitHash", "chromeVersion", "actorModel", "judgeModel",
|
||
"promptVersion", "configSnapshotHash")
|
||
|
||
# 20 局口径下 <5% → tester_error 必须 0；<10% → inconclusive 最多 1（设计档 §5.4 明文换算）。
|
||
SHADOW20_THRESHOLDS = {
|
||
"minSamples": SHADOW20_MIN_SAMPLES,
|
||
"maxConfirmedFalsePositives": 0,
|
||
"maxAcceptedWithProblems": 0,
|
||
"maxTesterError": 0,
|
||
"maxInconclusive": 1,
|
||
"minOrdinaryAcceptHumanReviewed": 5, # 普通 accept 至少抽 5（accept 不足 5 则全查）
|
||
}
|
||
|
||
|
||
def build_shadow_plan(prompts, *, commit_hash, chrome_version, actor_model, judge_model,
|
||
prompt_version, config_snapshot_hash, human_labels=None, note="") -> dict:
|
||
"""shadow20 跑批计划：固定六项 + ≥20 连续生产 prompt + 人工标签占位。任一固定项缺失/样本不足 → ValueError。"""
|
||
prompts = list(prompts or [])
|
||
if len(prompts) < SHADOW20_MIN_SAMPLES:
|
||
raise ValueError(f"shadow20 需连续 ≥{SHADOW20_MIN_SAMPLES} 个真实生产 prompt，实际 {len(prompts)}")
|
||
identity = {"commitHash": commit_hash, "chromeVersion": chrome_version,
|
||
"actorModel": actor_model, "judgeModel": judge_model,
|
||
"promptVersion": prompt_version, "configSnapshotHash": config_snapshot_hash}
|
||
empty = [k for k, v in identity.items() if not (isinstance(v, str) and v.strip())]
|
||
if empty:
|
||
raise ValueError(f"shadow20 固定六项不得为空:{empty}")
|
||
return {
|
||
"schemaVersion": "shadow20-plan/1",
|
||
"identity": identity,
|
||
"prompts": prompts,
|
||
# 人工标签占位：真跑后由人工逐局回填（分歧/reject/rescued/inconclusive/tester_error 必查，accept 抽 ≥5）。
|
||
"humanLabels": dict(human_labels or {}),
|
||
"note": str(note or ""),
|
||
"acceptanceMode": "v3_shadow", # shadow 新生产 run 只跑 v3、内测隔离、冻结自动发布（§3.10）
|
||
}
|
||
|
||
|
||
async def run_shadow_batch(plan, *, run_one) -> list:
|
||
"""shadow runner 框架：串行编排 ≥20 局独立 v3_shadow，每局盖固定身份戳。
|
||
|
||
run_one：注入的真跑 callable（async/sync 均可）—— (prompt, identity) → 单局结果行（mini-desktop 提供真实现，
|
||
经 cheap_verify.run_acceptance_v3(mode=v3_shadow) 产出行）；本地单测注入罐头函数，不烧真模型。
|
||
本框架只负责串行调度 + 身份固化 + 行规范化，不碰模型。
|
||
"""
|
||
if not isinstance(plan, dict) or not isinstance(plan.get("prompts"), list):
|
||
raise ValueError("shadow plan 非法")
|
||
identity = plan.get("identity") if isinstance(plan.get("identity"), dict) else {}
|
||
rows = []
|
||
for index, prompt in enumerate(plan["prompts"]):
|
||
result = run_one(prompt, dict(identity))
|
||
if hasattr(result, "__await__"):
|
||
result = await result
|
||
row = dict(result) if isinstance(result, dict) else {"outcome": "tester_error", "raw": result}
|
||
# 每局固化身份 + 序号，供闸门核对固定六项零漂移；人工标签占位随行带出待回填。
|
||
row.setdefault("gid", f"shadow20-{index + 1:02d}")
|
||
row["identity"] = dict(identity)
|
||
row["planIndex"] = index
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def _identity_drift(rows, plan) -> list:
|
||
"""核对所有行固定六项逐字一致，且与 plan 一致（漂移即 fail——shadow 的可比性前提）。"""
|
||
drifts = []
|
||
reference = None
|
||
if isinstance(plan, dict) and isinstance(plan.get("identity"), dict):
|
||
reference = {k: plan["identity"].get(k) for k in SHADOW20_IDENTITY_FIELDS}
|
||
for row in rows:
|
||
identity = row.get("identity") if isinstance(row.get("identity"), dict) else {}
|
||
values = {k: identity.get(k) for k in SHADOW20_IDENTITY_FIELDS}
|
||
if any(not (isinstance(v, str) and v) for v in values.values()):
|
||
drifts.append({"gid": row.get("gid"), "reason": "固定六项有空值"})
|
||
continue
|
||
if reference is None:
|
||
reference = values
|
||
elif values != reference:
|
||
diff = [k for k in SHADOW20_IDENTITY_FIELDS if values.get(k) != reference.get(k)]
|
||
drifts.append({"gid": row.get("gid"), "reason": f"身份漂移:{diff}"})
|
||
return drifts
|
||
|
||
|
||
def evaluate_shadow20_gate(rows, *, fresh25_gate=None, plan=None, thresholds=None) -> dict:
|
||
"""shadow20 达标断言（§5.4）：proof 完整率 100% + 确认假阳 0 + accepted 无 problems/缺证/矛盾共存
|
||
+ tester_error=0 + inconclusive≤1 + 人工复核覆盖 + 固定六项零漂移；fresh25 达标为硬前置。
|
||
|
||
row 消费字段：gid/outcome/proofComplete/confirmedFalsePositive/problems/contradictions/proofMissing/
|
||
rescued/discrepancy/humanReviewed/identity{六项}。
|
||
"""
|
||
th = dict(SHADOW20_THRESHOLDS)
|
||
if thresholds:
|
||
th.update(thresholds)
|
||
rows = [r for r in (rows or []) if isinstance(r, dict)]
|
||
warnings = []
|
||
checks = []
|
||
|
||
def check(name, ok, actual, required, detail=""):
|
||
checks.append({"name": name, "pass": bool(ok), "actual": actual, "required": required, "detail": detail})
|
||
|
||
# ── 硬前置（§5.4「fresh 25 达标后」）：未提供 → 前置无法验证，阻断 PASS（warn）；提供且未过 → 硬 fail。
|
||
if fresh25_gate is None:
|
||
warnings.append("未提供 fresh25 闸门结果：shadow20 硬前置无法验证，PASS 阻断（§5.4 顺序硬要求）")
|
||
prereq_ok, prereq_actual = False, "未验证"
|
||
elif isinstance(fresh25_gate, dict) and fresh25_gate.get("pass") is True:
|
||
prereq_ok, prereq_actual = True, "fresh25 达标"
|
||
else:
|
||
prereq_ok, prereq_actual = False, "fresh25 未达标"
|
||
check("prerequisiteFresh25", prereq_ok, prereq_actual, "fresh25 闸门通过（§5.4 硬前置）")
|
||
|
||
check("sampleSize", len(rows) >= th["minSamples"], f"{len(rows)} 局",
|
||
f"连续 ≥ {th['minSamples']} 个真实生产 prompt")
|
||
|
||
drifts = _identity_drift(rows, plan)
|
||
check("identityFixed", rows != [] and not drifts, f"{len(rows) - len(drifts)}/{len(rows)} 行身份一致",
|
||
"固定六项（commit/Chrome/Actor/Judge/prompt/配置）逐字一致",
|
||
";".join(f"{d['gid']}:{d['reason']}" for d in drifts[:3]))
|
||
|
||
accepted = [r for r in rows if r.get("outcome") == "accept"]
|
||
tester_errors = [r for r in rows if r.get("outcome") == "tester_error"]
|
||
inconclusives = [r for r in rows if r.get("outcome") == "inconclusive"]
|
||
|
||
# ── accepted 的 proof 完整率 100%（缺 proofComplete 字段 = 无法证明完整 → fail-closed）。
|
||
proof_incomplete = [r.get("gid") for r in accepted if r.get("proofComplete") is not True]
|
||
check("proofCompleteOnAccepted", not proof_incomplete,
|
||
f"{len(accepted) - len(proof_incomplete)}/{len(accepted)} 完整", "accepted proof 完整率 100%",
|
||
f"不完整={proof_incomplete[:3]}")
|
||
|
||
# ── accepted 与 problems/缺证/矛盾共存 0。
|
||
conflicted = [r.get("gid") for r in accepted
|
||
if r.get("problems") or r.get("contradictions") or r.get("proofMissing") is True]
|
||
check("acceptedWithoutConflict", not conflicted,
|
||
f"{len(accepted) - len(conflicted)}/{len(accepted)} 干净", "accepted 与 problems/缺证/矛盾共存 0",
|
||
f"共存={conflicted[:3]}")
|
||
|
||
confirmed_fp = [r.get("gid") for r in rows if r.get("confirmedFalsePositive") is True]
|
||
check("confirmedFalsePositives", not confirmed_fp, len(confirmed_fp), "确认假阳 0", ",".join(confirmed_fp))
|
||
|
||
check("testerError", len(tester_errors) <= th["maxTesterError"], f"{len(tester_errors)}/{len(rows)}",
|
||
f"≤ {th['maxTesterError']}（20 局口径 <5% → 0）", ",".join(r.get("gid", "?") for r in tester_errors[:3]))
|
||
|
||
check("inconclusive", len(inconclusives) <= th["maxInconclusive"], f"{len(inconclusives)}/{len(rows)}",
|
||
f"≤ {th['maxInconclusive']}（20 局口径 <10%）", ",".join(r.get("gid", "?") for r in inconclusives[:3]))
|
||
|
||
# ── 人工复核覆盖：全部 reject/rescued/inconclusive/tester_error/分歧必查 + 普通 accept 至少抽 5。
|
||
must_review = [r for r in rows
|
||
if r.get("outcome") in ("reject", "inconclusive", "tester_error")
|
||
or r.get("rescued") is True or r.get("discrepancy") is True]
|
||
unreviewed = [r.get("gid") for r in must_review if r.get("humanReviewed") is not True]
|
||
reviewed_accepts = sum(1 for r in accepted if r.get("humanReviewed") is True)
|
||
accept_quota = min(th["minOrdinaryAcceptHumanReviewed"], len(accepted))
|
||
review_ok = not unreviewed and reviewed_accepts >= accept_quota
|
||
check("humanReviewCoverage", review_ok,
|
||
f"必查 {len(must_review) - len(unreviewed)}/{len(must_review)}，accept 抽查 {reviewed_accepts}/{accept_quota}",
|
||
f"分歧/reject/rescued/inconclusive/tester_error 全查 + 普通 accept ≥ {accept_quota}",
|
||
f"未复核={unreviewed[:3]}")
|
||
|
||
outcomes = {"accept": len(accepted), "reject": sum(1 for r in rows if r.get("outcome") == "reject"),
|
||
"inconclusive": len(inconclusives), "tester_error": len(tester_errors)}
|
||
return {"pass": all(c["pass"] for c in checks), "checks": checks,
|
||
"metrics": {"outcomes": outcomes, "sampleSize": len(rows),
|
||
"proofCompleteOnAccepted": len(accepted) - len(proof_incomplete)},
|
||
"warnings": warnings, "thresholds": th}
|
||
|
||
|
||
# ────────────────────────── CLI（mini-desktop 真跑后对账用）──────────────────────────
|
||
|
||
def _cli() -> int:
|
||
import argparse
|
||
parser = argparse.ArgumentParser(description="三批基线闸门（fresh25 / historical / shadow20）")
|
||
parser.add_argument("gate", choices=("fresh25", "historical", "shadow20"))
|
||
parser.add_argument("rows_json", help="批结果行 JSON 数组文件")
|
||
parser.add_argument("--fresh25-result", help="shadow20/historical 链式校验：fresh25 闸门结果 JSON 文件")
|
||
parser.add_argument("--expectations", help="historical 预期表（默认 contracts/play-loop/historical-11-expectations.json）")
|
||
parser.add_argument("--plan", help="shadow20 计划 JSON（核对固定六项）")
|
||
args = parser.parse_args()
|
||
rows = json.loads(Path(args.rows_json).read_text(encoding="utf-8"))
|
||
fresh25_gate = None
|
||
if args.fresh25_result:
|
||
fresh25_gate = json.loads(Path(args.fresh25_result).read_text(encoding="utf-8"))
|
||
if args.gate == "fresh25":
|
||
report = evaluate_fresh25_gate(rows)
|
||
elif args.gate == "historical":
|
||
report = evaluate_historical_gate(rows, expectations_path=args.expectations, fresh25_gate=fresh25_gate)
|
||
else:
|
||
plan = json.loads(Path(args.plan).read_text(encoding="utf-8")) if args.plan else None
|
||
report = evaluate_shadow20_gate(rows, fresh25_gate=fresh25_gate, plan=plan)
|
||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||
return 0 if report.get("pass") else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(_cli())
|