cbfd4d871b
固化 Match-3 生产者、视觉、音频与双 Judge 证据闭包。 将《山海行纪》r1.1 绑定新的不可变 release，并以生产预检现场核验 bundle、Registry/2 和 25 项 Writer 快照。 同步地图1平衡锁值、跨游戏回归修复、验收契约与 SoT 证据。
3478 lines
170 KiB
Python
Executable File
3478 lines
170 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""
|
||
eval_gate.py — Prompt 四道闸 · 真模型闸（段 B，薄版重建）
|
||
|
||
这是什么：对一条改动的 live prompt，拿它的 Golden 金标集真跑一遍模型，过「四道闸」才放行。
|
||
历史判定实现（agent-loop v1 的 orchestrator/evalflow.py + judge.py）已随 spike 存档删除、只在 git；
|
||
本脚本按 registry 契约重建一个薄版四闸判定，不复活整套批跑编排——只做「一条 prompt × 它的金标集 →
|
||
四闸裁决」这一件事，触发方式改成 CI 手动 workflow_dispatch（见 .gitea/workflows/prompt-eval.yml）。
|
||
|
||
四道闸（对齐 prompt治理 SoT §4）：
|
||
闸1 Schema —— 模型对每条输入的产物必须能解析成 JSON 且符合该 prompt 声明的 output_schema。
|
||
闸2 成功率 —— 通用条目判对率 ≥0.8；Actor 至少 4/5，双 Judge 各自必须 6/6。
|
||
闸3 回归 diff —— 只与同 Prompt/fixture/model/seed 身份的基线逐条比关键裁决，翻转即拦。
|
||
闸4 成本/延迟 —— 与基线比单跑成本/延迟增量，超阈值即拦（变贵/变慢）。
|
||
另有前置闸0（version 升号）由段 A check_version_bump.py 管，本脚本不重复。
|
||
|
||
只焊 live 面（step 0 消费面对账，见 registry.yaml 头部三态总表）：
|
||
· 非 live（fossil / batch-relic）条目 → 显式 SKIP 豁免，绝不花真模型钱焊死面；
|
||
· live 但无金标集（当前 cheap-system / 8 条 tier2，eval 待建）→ fail-closed 报「金标缺失」，绝不假绿；
|
||
· live 且有金标集（safety.prompt-check、playtest.actor、playtest.judge-a/b）→ 真跑四闸。
|
||
LIVE_PROMPT_IDS 是段 B 的判定白名单，逐条依据见 registry.yaml 头部对账总表；启动自检它们都在 registry 内。
|
||
|
||
真模型：MiniMax-M3 经 new-api 网关 Anthropic messages 协议，thinking 与 text 分块审计。
|
||
凭据从环境变量读（NEWAPI_KEY 必需 / NEWAPI_BASE_URL 可选），绝不硬编码 key、不设长期 env——
|
||
key 权威源 = docs/内网凭据与端点.md。内网直连一律绕系统代理（urllib ProxyHandler({})）。
|
||
|
||
单跑预算 cap：硬编码 ≤¥5，累计估算成本超即 fail（防真模型闸自身失控烧钱）。
|
||
成本按 new-api 权威计费倍率折算（M3: model_ratio=0.15 / completion_ratio=4；qpu=500000；usd_rate=7.3，
|
||
实测自 /api/pricing + /api/status，2026-07-04）；台账同时记原始 token，权威成本以 new-api 对账为准。
|
||
|
||
台账：每次真跑 append 一行到 eval/<id>/runs/（只追加，不回改），记 token/成本/延迟/四闸裁决,可审计可复现。
|
||
|
||
用法：
|
||
NEWAPI_KEY=sk-xxx python3 contracts/prompts/eval_gate.py --id safety.prompt-check
|
||
NEWAPI_KEY=sk-xxx python3 contracts/prompts/eval_gate.py --changed --base HEAD~1 # 自动对改动的 live 条目跑
|
||
python3 contracts/prompts/eval_gate.py --id playtest.actor --validate-only # 只跑离线金标/图片/parser/版本门
|
||
# 附加：--update-baseline 建/更新基线快照；--ledger-dir <dir> 台账写别处（演示用，避免污染仓）
|
||
# exit 0 = 四闸全绿（或 SKIP 豁免）；exit 1 = 有闸未过 / 金标缺失 fail-closed；exit 2 = 模型跑不通（人工兜底）
|
||
|
||
图片消费依赖 Pillow 做完整 PNG 解码与 DPR 归一；playtest 输出和封存 hash 经 Node 薄桥直接调用生产 runtime 实现。
|
||
"""
|
||
|
||
import argparse
|
||
import base64
|
||
import binascii
|
||
import copy
|
||
import fcntl
|
||
import hashlib
|
||
import importlib.metadata
|
||
import io
|
||
import json
|
||
import math
|
||
import os
|
||
import platform
|
||
import re
|
||
import shutil
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
import urllib.parse
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
try:
|
||
from PIL import Image, UnidentifiedImageError
|
||
except ImportError: # pragma: no cover - CI 缺依赖时由图片门给出可操作错误
|
||
Image = None
|
||
UnidentifiedImageError = OSError
|
||
|
||
PROMPTS_DIR = Path(__file__).resolve().parent
|
||
REGISTRY_FILE = PROMPTS_DIR / "registry.yaml"
|
||
EVAL_DIR = PROMPTS_DIR / "eval"
|
||
PLAY_LOOP_CONTRACT_DIR = PROMPTS_DIR.parents[1] / "contracts/play-loop"
|
||
ACTOR_OUTPUT_SCHEMA_FILE = PLAY_LOOP_CONTRACT_DIR / "actor-selection.schema.json"
|
||
JUDGE_OUTPUT_SCHEMA_FILE = PLAY_LOOP_CONTRACT_DIR / "judge-result.schema.json"
|
||
PRODUCTION_PARSER_BRIDGE = PROMPTS_DIR / "production_parser_bridge.cjs"
|
||
ACTOR_GUIDE_RENDERER = (
|
||
PROMPTS_DIR.parents[1]
|
||
/ "game-runtime/games/_wg1-gen/_shared/actor_guide_renderer.py"
|
||
)
|
||
VISUAL_TARGET_RENDERER = (
|
||
PROMPTS_DIR.parents[1]
|
||
/ "game-runtime/games/_wg1-gen/_shared/visual_target_renderer.py"
|
||
)
|
||
VISUAL_TARGET_PYTHON = PROMPTS_DIR.parents[1] / "cheap-worker/.venv/bin/python"
|
||
PLAYTEST_JUDGE_IDS = {"playtest.judge-a", "playtest.judge-b"}
|
||
PLAYTEST_STABILITY_STRATEGY = "playtest-aggregate-v2"
|
||
|
||
# ── 段 B 判定白名单：仅这些 live 条目才跑真模型闸（逐条依据见 registry.yaml 头部消费面三态对账总表）──
|
||
# live = 现行生产生成路径运行时读它、渲染后发给 LLM。非白名单条目一律 SKIP 豁免（fossil / batch-relic）。
|
||
LIVE_PROMPT_IDS = {
|
||
"safety.prompt-check", # SafetyCheckClient GP9 合规门（有 eval 金标集 → 真跑）
|
||
"config.cheap-system", # cheap_roles.py 便宜档 A-model（无金标集 → fail-closed）
|
||
"tier2.design-system", # 以下 8 条 tier2：roles.py prompts.load 运行时读（无金标集 → fail-closed）
|
||
"tier2.design-systems-designer",
|
||
"tier2.design-economy-designer",
|
||
"tier2.design-level-designer",
|
||
"tier2.design-presentation-designer",
|
||
"tier2.design-leader-system",
|
||
"tier2.writer-system",
|
||
"tier2.player-system",
|
||
"playtest.actor", # playtest/3 Actor：只产 ActorSelection/1（有协议金标 → 真跑）
|
||
"playtest.judge-a", # 双 Judge A/B：同包、独立检查顺序（共用语义金标）
|
||
"playtest.judge-b",
|
||
}
|
||
|
||
# ── 硬约束参数 ────────────────────────────────────────────────────────────────
|
||
PER_RUN_BUDGET_CAP_RMB = 5.0 # 单跑预算 cap（硬编码，累计估算成本超即 fail）
|
||
MIN_SUCCESS_RATE = 0.8 # 闸2 成功率阈值（对齐 MVP「AI 生成成功率 ≥80%」）
|
||
MIN_STABLE_RUNS = 3 # 相同请求连续三轮达到金标稳定门后才允许建 baseline；下一轮复跑确认
|
||
MAX_COST_DELTA_RATIO = 0.5 # 闸4 成本增量上限（相对基线 +50%）；prompt 契约未声明时的缺省
|
||
SEED_STRATEGY = "audit-only-sha256(promptId\\0evalKey)-uint32be;anthropic-provider-no-seed"
|
||
PROVIDER_PROTOCOL = "anthropic-messages"
|
||
ANTHROPIC_VERSION = "2023-06-01"
|
||
ANTHROPIC_JUDGE_THINKING_BUDGET = 2000
|
||
ANTHROPIC_JUDGE_MAX_TOKENS = 10000
|
||
ANTHROPIC_ACTOR_THINKING_BUDGET = 4000
|
||
ANTHROPIC_ACTOR_MAX_TOKENS = 20000
|
||
MAX_LATENCY_DELTA_RATIO = 1.0 # 闸4 延迟增量上限（相对基线 +100%）；缺省
|
||
MODEL_CALL_TIMEOUT_S = 120 # Anthropic thinking 可超过 60 秒；与生产 Actor/Judge 默认超时对齐
|
||
MODEL_CALL_RETRIES = 2 # 单条最多重试次数（限流/超时 best-effort）
|
||
MAX_RAW_AUDIT_BYTES = 128 * 1024 # 单条模型原始响应审计上限；超限保存有界前缀并使本轮 fail-closed
|
||
MAX_AUDIT_REQUEST_BYTES = 24 * 1024 * 1024
|
||
MAX_AUDIT_SAMPLE_IMAGE_BYTES = 16 * 1024 * 1024
|
||
MAX_AUDIT_RUN_BYTES = 256 * 1024 * 1024
|
||
MAX_AUDIT_ERROR_BYTES = 16 * 1024
|
||
AUDIT_SCHEMA_VERSION = "PromptEvalAudit/2"
|
||
|
||
# 只保存排障和配额需要的响应头。Authorization、Cookie、代理凭据及未知头一律不进入审计包。
|
||
AUDIT_RESPONSE_HEADER_ALLOWLIST = {
|
||
"content-type", "content-length", "date", "request-id", "x-request-id",
|
||
"x-trace-id", "traceparent", "openai-processing-ms",
|
||
}
|
||
|
||
# M3 new-api 权威计费倍率（源：GET /api/pricing 的 MiniMax-M3 + /api/status，2026-07-04 实测）。
|
||
# quota = (prompt_tokens + completion_tokens × completion_ratio) × model_ratio（保守不计 cache 折扣）；
|
||
# rmb = quota / quota_per_unit × usd_rate。权威成本以 new-api quota 对账为准，此处为 cap/闸4 用的一致口径估算。
|
||
M3_MODEL_RATIO = 0.15
|
||
M3_COMPLETION_RATIO = 4.0
|
||
NEWAPI_QUOTA_PER_UNIT = 500000.0
|
||
NEWAPI_USD_RATE = 7.3
|
||
|
||
DEFAULT_MODEL = "MiniMax-M3"
|
||
DEFAULT_BASE_URL = "http://100.64.0.8:3000" # new-api host 根（NEWAPI_BASE_URL 覆盖）；Python 侧调用补 /v1
|
||
|
||
# eval 图片只接受仓内真实 PNG 截图；尺寸/体积下限直接堵住 1×1 占位图与空壳资产。
|
||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||
MIN_EVAL_IMAGE_EDGE = 32
|
||
MIN_EVAL_IMAGE_BYTES = 1024
|
||
MAX_EVAL_IMAGE_BYTES = 5 * 1024 * 1024
|
||
MAX_EVAL_IMAGE_PIXELS = 16 * 1024 * 1024
|
||
EVAL_FRAME_SIZE = (390, 844)
|
||
|
||
|
||
# ══════════════════════ registry / frontmatter 解析（零依赖，复用 check_registry 口径）══════════════════════
|
||
|
||
def _strip_inline_comment(value):
|
||
out, in_quote = [], None
|
||
for ch in value:
|
||
if in_quote:
|
||
if ch == in_quote:
|
||
in_quote = None
|
||
out.append(ch)
|
||
elif ch in ("'", '"'):
|
||
in_quote = ch
|
||
out.append(ch)
|
||
elif ch == '#':
|
||
break
|
||
else:
|
||
out.append(ch)
|
||
s = "".join(out).strip()
|
||
if len(s) >= 2 and s[0] == s[-1] and s[0] in ("'", '"'):
|
||
s = s[1:-1]
|
||
return s
|
||
|
||
|
||
def parse_registry(path):
|
||
"""解析 registry.yaml → {id: {file, version, eval, ...}}（最小 YAML 子集，无三方依赖）。"""
|
||
text = path.read_text(encoding="utf-8")
|
||
entries = {}
|
||
current_id = None
|
||
in_prompts = False
|
||
for line in text.splitlines():
|
||
stripped = line.strip()
|
||
if not stripped or stripped.startswith('#'):
|
||
continue
|
||
if re.match(r'^prompts\s*:\s*(#.*)?$', line):
|
||
in_prompts = True
|
||
continue
|
||
if not in_prompts:
|
||
continue
|
||
m = re.match(r'^\s*-\s+id\s*:\s*(.+)$', line)
|
||
if m:
|
||
current_id = _strip_inline_comment(m.group(1))
|
||
entries[current_id] = {"id": current_id}
|
||
continue
|
||
m = re.match(r'^\s+([A-Za-z_][A-Za-z0-9_]*)\s*:\s*(.*)$', line)
|
||
if m and current_id is not None:
|
||
entries[current_id][m.group(1)] = _strip_inline_comment(m.group(2))
|
||
continue
|
||
if re.match(r'^[A-Za-z_]', line):
|
||
in_prompts = False
|
||
current_id = None
|
||
return entries
|
||
|
||
|
||
def split_frontmatter(text):
|
||
"""返回 (frontmatter_text, body)；无合法 frontmatter → ("", 原文)。"""
|
||
lines = text.split("\n")
|
||
if not lines or lines[0].strip() != "---":
|
||
return "", text
|
||
for i in range(1, len(lines)):
|
||
if lines[i].strip() == "---":
|
||
fm = "\n".join(lines[1:i])
|
||
body = "\n".join(lines[i + 1:])
|
||
return fm, body
|
||
return "", text
|
||
|
||
|
||
def _frontmatter_scalar(frontmatter_text, field):
|
||
"""读取 frontmatter 顶层标量，供 registry 与 prompt 身份对账。"""
|
||
pattern = rf'^{re.escape(field)}\s*:\s*(.+)$'
|
||
for line in frontmatter_text.splitlines():
|
||
match = re.match(pattern, line)
|
||
if match:
|
||
return _strip_inline_comment(match.group(1))
|
||
return None
|
||
|
||
|
||
def load_prompt_contract(prompt_id, entry):
|
||
"""加载并校验 prompt 文件、frontmatter id/version 与 registry 的单一身份。"""
|
||
errors = []
|
||
rel_file = entry.get("file") if isinstance(entry, dict) else None
|
||
if not isinstance(rel_file, str) or not rel_file:
|
||
return None, [f"{prompt_id}: registry 缺 file"]
|
||
candidate = PROMPTS_DIR / rel_file
|
||
try:
|
||
resolved = candidate.resolve(strict=True)
|
||
resolved.relative_to(PROMPTS_DIR.resolve())
|
||
except (OSError, ValueError) as exc:
|
||
return None, [f"{prompt_id}: prompt 文件不在 contracts/prompts 内:{exc}"]
|
||
try:
|
||
prompt_text = resolved.read_text(encoding="utf-8")
|
||
except OSError as exc:
|
||
return None, [f"{prompt_id}: prompt 文件读取失败:{exc}"]
|
||
frontmatter, body = split_frontmatter(prompt_text)
|
||
if not frontmatter:
|
||
errors.append(f"{prompt_id}: prompt 缺合法 frontmatter")
|
||
prompt_frontmatter_id = _frontmatter_scalar(frontmatter, "id")
|
||
prompt_frontmatter_version = _frontmatter_scalar(frontmatter, "version")
|
||
registry_version = entry.get("version")
|
||
if prompt_frontmatter_id != prompt_id:
|
||
errors.append(
|
||
f"{prompt_id}: frontmatter id={prompt_frontmatter_id!r} 与 registry id 不一致"
|
||
)
|
||
if prompt_frontmatter_version != registry_version:
|
||
errors.append(
|
||
f"{prompt_id}: frontmatter version={prompt_frontmatter_version!r} "
|
||
f"与 registry version={registry_version!r} 不一致"
|
||
)
|
||
registry_eval = entry.get("eval")
|
||
expected_eval = "eval/playtest.judge/" if prompt_id in PLAYTEST_JUDGE_IDS \
|
||
else f"eval/{prompt_id}/"
|
||
if registry_eval and registry_eval != expected_eval:
|
||
errors.append(
|
||
f"{prompt_id}: registry eval={registry_eval!r} 未指向 canonical {expected_eval}"
|
||
)
|
||
if not body.strip():
|
||
errors.append(f"{prompt_id}: prompt 正文为空")
|
||
return {
|
||
"path": resolved,
|
||
"frontmatter": frontmatter,
|
||
"body": body,
|
||
"schema": parse_output_schema(frontmatter),
|
||
"version": registry_version,
|
||
}, errors
|
||
|
||
|
||
def parse_output_schema(frontmatter_text):
|
||
"""从 frontmatter 提取 output_schema 的 required 字段与各字段 type（JSON Schema 子集）。
|
||
|
||
只解析本项目 prompt 契约实际用到的形态（safety 为例）：
|
||
output_schema:
|
||
type: object
|
||
required: [safe, reason]
|
||
properties:
|
||
safe: { type: boolean }
|
||
reason: { type: string }
|
||
返回 {"required": ["safe","reason"], "types": {"safe":"boolean","reason":"string"}}；
|
||
无 output_schema 块 → None（闸1 退化为「仅要求是合法 JSON 对象」）。
|
||
更复杂的嵌套 schema 需扩展本解析（诚实边界，非静默放过）。
|
||
"""
|
||
lines = frontmatter_text.split("\n")
|
||
# 定位 output_schema: 块的缩进范围
|
||
start = None
|
||
base_indent = None
|
||
for i, line in enumerate(lines):
|
||
if re.match(r'^output_schema\s*:', line):
|
||
start = i
|
||
base_indent = len(line) - len(line.lstrip())
|
||
break
|
||
if start is None:
|
||
return None
|
||
block = []
|
||
for line in lines[start + 1:]:
|
||
if not line.strip():
|
||
block.append(line)
|
||
continue
|
||
indent = len(line) - len(line.lstrip())
|
||
if indent <= base_indent:
|
||
break # 回到同级/更外层，块结束
|
||
block.append(line)
|
||
block_text = "\n".join(block)
|
||
|
||
required = []
|
||
m = re.search(r'required\s*:\s*\[([^\]]*)\]', block_text)
|
||
if m:
|
||
required = [x.strip().strip('"\'') for x in m.group(1).split(",") if x.strip()]
|
||
|
||
types = {}
|
||
# properties 下形如 ` safe: { type: boolean }`
|
||
for line in block:
|
||
pm = re.match(r'^\s+([A-Za-z_][A-Za-z0-9_]*)\s*:\s*\{\s*type\s*:\s*([A-Za-z]+)\s*\}', line)
|
||
if pm:
|
||
types[pm.group(1)] = pm.group(2)
|
||
return {"required": required, "types": types}
|
||
|
||
|
||
# ══════════════════════ 金标集加载 ══════════════════════
|
||
|
||
def _read_jsonl_strict(path):
|
||
"""严格读取 JSONL；坏行、非对象、空/重复 evalKey 任一出现都返回错误。"""
|
||
rows = []
|
||
seen = set()
|
||
errors = []
|
||
try:
|
||
lines = path.read_text(encoding="utf-8").splitlines()
|
||
except OSError as exc:
|
||
return None, [f"{path}: 读取失败:{exc}"]
|
||
for line_no, raw_line in enumerate(lines, start=1):
|
||
line = raw_line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
row = json.loads(line)
|
||
except json.JSONDecodeError as exc:
|
||
errors.append(f"{path}:{line_no}: 非法 JSON:{exc.msg}")
|
||
continue
|
||
if not isinstance(row, dict):
|
||
errors.append(f"{path}:{line_no}: 每行必须是 JSON 对象")
|
||
continue
|
||
eval_key = row.get("evalKey")
|
||
if not isinstance(eval_key, str) or not eval_key.strip():
|
||
errors.append(f"{path}:{line_no}: 缺非空字符串 evalKey")
|
||
continue
|
||
if eval_key in seen:
|
||
errors.append(f"{path}:{line_no}: 重复 evalKey:{eval_key}")
|
||
continue
|
||
seen.add(eval_key)
|
||
rows.append(row)
|
||
if not rows and not errors:
|
||
errors.append(f"{path}: JSONL 为空")
|
||
return (None, errors) if errors else (rows, None)
|
||
|
||
|
||
def load_golden(eval_id, eval_root=EVAL_DIR):
|
||
"""加载 eval/<eval_id>/{inputs,labels}.jsonl，按 evalKey 关联成 [{evalKey, input, label}]。
|
||
|
||
返回 (samples, err)：err 非 None 表示金标缺失/不可用（上层 fail-closed）。
|
||
"""
|
||
# Judge A/B 规则和输出 schema 相同，共用同一组输入/行为标签；prompt 身份和 hash 仍各自独立。
|
||
golden_id = "playtest.judge" if eval_id in PLAYTEST_JUDGE_IDS else eval_id
|
||
d = Path(eval_root) / golden_id
|
||
inputs_f = d / "inputs.jsonl"
|
||
labels_f = d / "labels.jsonl"
|
||
if not inputs_f.exists() or not labels_f.exists():
|
||
return None, f"金标集缺失：{d} 下无 inputs.jsonl / labels.jsonl"
|
||
|
||
input_rows, input_errors = _read_jsonl_strict(inputs_f)
|
||
label_rows, label_errors = _read_jsonl_strict(labels_f)
|
||
errors = [*(input_errors or []), *(label_errors or [])]
|
||
if errors:
|
||
return None, "金标 JSONL 不可用：" + "；".join(errors)
|
||
|
||
inputs = {row["evalKey"]: row for row in input_rows}
|
||
labels = {row["evalKey"]: row for row in label_rows}
|
||
input_keys = set(inputs)
|
||
label_keys = set(labels)
|
||
if input_keys != label_keys:
|
||
only_inputs = sorted(input_keys - label_keys)
|
||
only_labels = sorted(label_keys - input_keys)
|
||
return None, (
|
||
f"inputs/labels evalKey 不完全一致:{d} "
|
||
f"仅 inputs={only_inputs} 仅 labels={only_labels}"
|
||
)
|
||
samples = [
|
||
{"evalKey": row["evalKey"], "input": row, "label": labels[row["evalKey"]]}
|
||
for row in input_rows
|
||
]
|
||
return samples, None
|
||
|
||
|
||
def _contains_mapping_key(value, target_key):
|
||
"""递归检查金标输入是否携带禁止持久化的内嵌字段。"""
|
||
if isinstance(value, dict):
|
||
return target_key in value or any(
|
||
_contains_mapping_key(item, target_key) for item in value.values()
|
||
)
|
||
if isinstance(value, list):
|
||
return any(_contains_mapping_key(item, target_key) for item in value)
|
||
return False
|
||
|
||
|
||
# ══════════════════════ 模型调用（M3 via new-api，thinking 关，绕代理）══════════════════════
|
||
|
||
def _no_proxy_opener():
|
||
"""构造禁用系统代理的 urllib opener（内网直连必须绕本机 fake-ip 代理，否则栽 502/连接关闭）。"""
|
||
return urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||
|
||
|
||
def _to_anthropic_content(content):
|
||
"""把生产同形的 OpenAI 多模态块无损投影为 Anthropic messages 内容块。"""
|
||
if isinstance(content, str):
|
||
return content
|
||
if not isinstance(content, list):
|
||
raise ValueError("Anthropic message content 必须是字符串或数组")
|
||
converted = []
|
||
for block in content:
|
||
if not isinstance(block, dict):
|
||
raise ValueError("Anthropic message block 必须是对象")
|
||
if block.get("type") == "text" and isinstance(block.get("text"), str):
|
||
converted.append({"type": "text", "text": block["text"]})
|
||
continue
|
||
image_url = (block.get("image_url") or {}).get("url") \
|
||
if block.get("type") == "image_url" and isinstance(block.get("image_url"), dict) else None
|
||
match = re.fullmatch(r"data:([^;,]+);base64,([A-Za-z0-9+/=]+)", image_url or "")
|
||
if not match:
|
||
raise ValueError("Anthropic 图片必须是合法 base64 data URL")
|
||
converted.append({
|
||
"type": "image",
|
||
"source": {"type": "base64", "media_type": match.group(1), "data": match.group(2)},
|
||
})
|
||
return converted
|
||
|
||
|
||
def load_provider_output_schema(prompt_id):
|
||
"""读取 playtest 角色唯一 canonical 输出 schema；缺失或身份漂移时 fail-closed。"""
|
||
if prompt_id == "playtest.actor":
|
||
schema_file = ACTOR_OUTPUT_SCHEMA_FILE
|
||
expected_id = "https://wanxiang.ai/contracts/play-loop/actor-selection.schema.json"
|
||
elif prompt_id in PLAYTEST_JUDGE_IDS:
|
||
schema_file = JUDGE_OUTPUT_SCHEMA_FILE
|
||
expected_id = "https://wanxiang.ai/contracts/play-loop/judge-result.schema.json"
|
||
else:
|
||
return None
|
||
try:
|
||
schema = json.loads(schema_file.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError) as exc:
|
||
raise ValueError(f"canonical output schema 不可读:{schema_file}:{exc}") from exc
|
||
if not isinstance(schema, dict) or schema.get("$id") != expected_id:
|
||
raise ValueError(f"canonical output schema 身份错误:{schema_file}")
|
||
return schema
|
||
|
||
|
||
def build_provider_request_payload(model, messages, max_tokens=None, seed=None, output_schema=None):
|
||
"""构造真正发送的 Anthropic 请求字节；M3 不支持 seed，seed 只进入审计身份。"""
|
||
del seed
|
||
systems = [row.get("content") for row in messages if row.get("role") == "system"]
|
||
if any(not isinstance(value, str) for value in systems):
|
||
raise ValueError("Anthropic system 必须是字符串")
|
||
provider_messages = [
|
||
{"role": row["role"], "content": _to_anthropic_content(row.get("content"))}
|
||
for row in messages if row.get("role") != "system"
|
||
]
|
||
effective_max_tokens = max(ANTHROPIC_JUDGE_MAX_TOKENS, int(max_tokens or 0))
|
||
thinking_budget = ANTHROPIC_ACTOR_THINKING_BUDGET \
|
||
if effective_max_tokens >= ANTHROPIC_ACTOR_MAX_TOKENS \
|
||
else ANTHROPIC_JUDGE_THINKING_BUDGET
|
||
body = {
|
||
"model": model,
|
||
"max_tokens": effective_max_tokens,
|
||
"thinking": {"type": "enabled", "budget_tokens": thinking_budget},
|
||
"system": "\n\n".join(systems),
|
||
"messages": provider_messages,
|
||
}
|
||
if output_schema is not None:
|
||
if not isinstance(output_schema, dict) or isinstance(output_schema, list):
|
||
raise ValueError("Anthropic output schema 必须是对象")
|
||
body["output_config"] = {
|
||
"format": {"type": "json_schema", "schema": output_schema},
|
||
}
|
||
return json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||
|
||
|
||
def _audit_headers(headers):
|
||
"""响应头采用显式白名单；未知头和所有凭据相关头默认丢弃。"""
|
||
result = {}
|
||
for key, value in (headers.items() if headers is not None else []):
|
||
lowered = str(key).lower().strip()
|
||
allow_rate_limit = lowered.startswith("x-ratelimit-limit-") \
|
||
or lowered.startswith("x-ratelimit-remaining-") \
|
||
or lowered.startswith("x-ratelimit-reset-")
|
||
if lowered in AUDIT_RESPONSE_HEADER_ALLOWLIST or allow_rate_limit:
|
||
result[lowered] = str(value)[:4096]
|
||
return result
|
||
|
||
|
||
def _sanitize_audit_error(value, secrets=()):
|
||
"""错误文本可能带 URL 或凭据；落盘前移除 query、userinfo 和已知 secret。"""
|
||
text = str(value or "")
|
||
for secret in secrets:
|
||
if secret:
|
||
text = text.replace(str(secret), "[REDACTED]")
|
||
text = re.sub(r'(?i)(authorization|proxy-authorization|cookie)\s*[:=]\s*\S+', r'\1=[REDACTED]', text)
|
||
text = re.sub(r'(https?://)([^/@\s]+@)', r'\1[REDACTED]@', text)
|
||
text = re.sub(r'(https?://[^?\s]+)\?\S+', r'\1?[REDACTED]', text)
|
||
encoded = text.encode("utf-8", errors="replace")[:MAX_AUDIT_ERROR_BYTES]
|
||
return encoded.decode("utf-8", errors="ignore")
|
||
|
||
|
||
def _provider_response_identity(response_body):
|
||
"""从完整 provider envelope 提取允许进入 manifest 的稳定身份，不复制模型正文。"""
|
||
if not isinstance(response_body, dict):
|
||
return {}
|
||
blocks = response_body.get("content") if isinstance(response_body.get("content"), list) else []
|
||
return {
|
||
"protocol": PROVIDER_PROTOCOL,
|
||
"id": response_body.get("id"),
|
||
"model": response_body.get("model"),
|
||
"stopReason": response_body.get("stop_reason"),
|
||
"blockTypes": [block.get("type") for block in blocks if isinstance(block, dict)],
|
||
}
|
||
|
||
|
||
def _utc_now():
|
||
"""统一生成带时区的 UTC 审计时点。"""
|
||
return datetime.now(timezone.utc).isoformat()
|
||
|
||
|
||
def call_model(base_url, api_key, model, messages, max_tokens=None, seed=None, output_schema=None):
|
||
"""调用 Anthropic messages 端点，并返回 thinking/text 分离的完整逐 attempt 审计。"""
|
||
url = base_url.rstrip("/") + "/v1/messages"
|
||
payload = build_provider_request_payload(
|
||
model, messages, max_tokens=max_tokens, seed=seed, output_schema=output_schema,
|
||
)
|
||
if len(payload) > MAX_AUDIT_REQUEST_BYTES:
|
||
return {
|
||
"content": None, "promptTokens": 0, "completionTokens": 0, "latencyMs": 0,
|
||
"error": f"request_oversize:{len(payload)}>{MAX_AUDIT_REQUEST_BYTES}",
|
||
"requestPayload": payload[:MAX_AUDIT_REQUEST_BYTES], "requestReplayable": False,
|
||
"attempts": [],
|
||
}
|
||
opener = _no_proxy_opener()
|
||
attempts = []
|
||
last_err = None
|
||
for attempt_no in range(1, MODEL_CALL_RETRIES + 2):
|
||
started_at = _utc_now()
|
||
t0 = time.time()
|
||
status = None
|
||
response_bytes = None
|
||
response_headers = {}
|
||
try:
|
||
request = urllib.request.Request(
|
||
url, data=payload,
|
||
headers={
|
||
"x-api-key": api_key,
|
||
"anthropic-version": ANTHROPIC_VERSION,
|
||
"Content-Type": "application/json",
|
||
},
|
||
method="POST",
|
||
)
|
||
with opener.open(request, timeout=MODEL_CALL_TIMEOUT_S) as response:
|
||
status = int(response.status)
|
||
response_headers = _audit_headers(response.headers)
|
||
response_bytes = response.read(MAX_RAW_AUDIT_BYTES + 1)
|
||
latency_ms = int((time.time() - t0) * 1000)
|
||
if len(response_bytes) > MAX_RAW_AUDIT_BYTES:
|
||
last_err = f"provider_response_oversize:{len(response_bytes)}>{MAX_RAW_AUDIT_BYTES}"
|
||
attempts.append({
|
||
"attempt": attempt_no, "httpStatus": status, "latencyMs": latency_ms,
|
||
"startedAt": started_at, "finishedAt": _utc_now(),
|
||
"headers": response_headers, "responseBytes": response_bytes[:MAX_RAW_AUDIT_BYTES],
|
||
"responseComplete": False, "error": last_err,
|
||
})
|
||
break
|
||
parsed_body = json.loads(response_bytes.decode("utf-8"))
|
||
blocks = parsed_body.get("content")
|
||
if not isinstance(blocks, list):
|
||
raise ValueError("Anthropic response.content 不是数组")
|
||
content = "".join(
|
||
block.get("text", "") for block in blocks
|
||
if isinstance(block, dict) and block.get("type") == "text"
|
||
)
|
||
if not content.strip():
|
||
latency_ms = int((time.time() - t0) * 1000)
|
||
last_err = (
|
||
f"anthropic_no_text_block:stop_reason={parsed_body.get('stop_reason')}"
|
||
)
|
||
attempts.append({
|
||
"attempt": attempt_no, "httpStatus": status, "latencyMs": latency_ms,
|
||
"startedAt": started_at, "finishedAt": _utc_now(),
|
||
"headers": response_headers, "responseBytes": response_bytes,
|
||
"responseComplete": True, "provider": _provider_response_identity(parsed_body),
|
||
"error": last_err,
|
||
})
|
||
# 已收到完整模型内容但没有可消费 text，属于语义/额度失败，禁止重采样挑答案。
|
||
break
|
||
usage = parsed_body.get("usage") or {}
|
||
attempts.append({
|
||
"attempt": attempt_no, "httpStatus": status, "latencyMs": latency_ms,
|
||
"startedAt": started_at, "finishedAt": _utc_now(),
|
||
"headers": response_headers, "responseBytes": response_bytes,
|
||
"responseComplete": True, "provider": _provider_response_identity(parsed_body),
|
||
"error": None,
|
||
})
|
||
return {
|
||
"content": content,
|
||
"promptTokens": int(usage.get("input_tokens") or 0)
|
||
+ int(usage.get("cache_read_input_tokens") or 0)
|
||
+ int(usage.get("cache_creation_input_tokens") or 0),
|
||
"completionTokens": int(usage.get("output_tokens") or 0),
|
||
"latencyMs": latency_ms,
|
||
"error": None,
|
||
"requestPayload": payload,
|
||
"requestReplayable": True,
|
||
"attempts": attempts,
|
||
}
|
||
except urllib.error.HTTPError as exc:
|
||
status = int(exc.code)
|
||
response_headers = _audit_headers(exc.headers)
|
||
try:
|
||
response_bytes = exc.read(MAX_RAW_AUDIT_BYTES + 1)
|
||
except OSError:
|
||
response_bytes = b""
|
||
latency_ms = int((time.time() - t0) * 1000)
|
||
complete = len(response_bytes) <= MAX_RAW_AUDIT_BYTES
|
||
last_err = _sanitize_audit_error(f"HTTPError:{status}:{exc.reason}", (api_key,))
|
||
attempts.append({
|
||
"attempt": attempt_no, "httpStatus": status, "latencyMs": latency_ms,
|
||
"startedAt": started_at, "finishedAt": _utc_now(),
|
||
"headers": response_headers,
|
||
"responseBytes": response_bytes[:MAX_RAW_AUDIT_BYTES],
|
||
"responseComplete": complete, "error": last_err,
|
||
})
|
||
except Exception as exc: # noqa: BLE001 —— 网络、超时和解析错误都必须留痕后 best-effort 重试
|
||
latency_ms = int((time.time() - t0) * 1000)
|
||
last_err = _sanitize_audit_error(f"{type(exc).__name__}: {exc}", (api_key,))
|
||
attempts.append({
|
||
"attempt": attempt_no, "httpStatus": status, "latencyMs": latency_ms,
|
||
"startedAt": started_at, "finishedAt": _utc_now(),
|
||
"headers": response_headers, "responseBytes": response_bytes,
|
||
# 超时/连接失败本来就没有 provider body；错误本身已完整记录，不算审计截断。
|
||
"responseComplete": True, "error": last_err,
|
||
})
|
||
if attempt_no <= MODEL_CALL_RETRIES:
|
||
attempts[-1]["backoffMs"] = attempt_no * 1000
|
||
time.sleep(1.0 * attempt_no)
|
||
return {
|
||
"content": None, "promptTokens": 0, "completionTokens": 0,
|
||
"latencyMs": attempts[-1]["latencyMs"] if attempts else 0,
|
||
"error": last_err, "requestPayload": payload, "requestReplayable": True,
|
||
"attempts": attempts,
|
||
}
|
||
|
||
|
||
def estimate_cost_rmb(prompt_tokens, completion_tokens):
|
||
"""按 new-api M3 权威倍率估算单次调用成本（¥）；口径见文件头计费说明。"""
|
||
quota = (prompt_tokens + completion_tokens * M3_COMPLETION_RATIO) * M3_MODEL_RATIO
|
||
return quota / NEWAPI_QUOTA_PER_UNIT * NEWAPI_USD_RATE
|
||
|
||
|
||
def eval_policy_seed(prompt_id, eval_key):
|
||
"""按 promptId + evalKey 固定 uint32 seed；prompt 升版仍沿用同一采样点，便于回归比较。"""
|
||
digest = hashlib.sha256(f"{prompt_id}\0{eval_key}".encode("utf-8")).digest()
|
||
return int.from_bytes(digest[:4], "big", signed=False)
|
||
|
||
|
||
def minimum_correct_required(prompt_id, sample_count):
|
||
"""返回专项闸2的最少正确数；双 Judge 必须全对，不能被通用 80% 门误放行。"""
|
||
if prompt_id in PLAYTEST_JUDGE_IDS:
|
||
return sample_count
|
||
return math.ceil(sample_count * MIN_SUCCESS_RATE)
|
||
|
||
|
||
# ══════════════════════ 生产消息、产物解析 + 四闸判定 ══════════════════════
|
||
|
||
def stable_json(value):
|
||
"""与 runner 的稳定 JSON 口径对齐：递归键排序、无多余空白、UTF-8 中文直写。"""
|
||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||
|
||
|
||
def _hash_named_blobs(domain, named_blobs):
|
||
"""按域标签、名字和长度封帧计算集合 hash，避免裸拼接产生边界歧义。"""
|
||
digest = hashlib.sha256()
|
||
domain_bytes = str(domain).encode("utf-8")
|
||
digest.update(len(domain_bytes).to_bytes(8, "big"))
|
||
digest.update(domain_bytes)
|
||
for name, data in sorted(named_blobs, key=lambda item: item[0].encode("utf-8")):
|
||
name_bytes = str(name).encode("utf-8")
|
||
blob = bytes(data)
|
||
digest.update(len(name_bytes).to_bytes(8, "big"))
|
||
digest.update(name_bytes)
|
||
digest.update(len(blob).to_bytes(8, "big"))
|
||
digest.update(blob)
|
||
return digest.hexdigest()
|
||
|
||
|
||
def _iter_image_fixture_refs(value):
|
||
"""只从已校验金标结构中提取 imagePath；模型输出不会进入文件路径。"""
|
||
if isinstance(value, dict):
|
||
for key, item in value.items():
|
||
if key == "imagePath" and isinstance(item, str) and item:
|
||
yield item
|
||
else:
|
||
yield from _iter_image_fixture_refs(item)
|
||
elif isinstance(value, list):
|
||
for item in value:
|
||
yield from _iter_image_fixture_refs(item)
|
||
|
||
|
||
def build_eval_snapshot(eval_id, contract, samples, eval_root=EVAL_DIR, provider_output_schema=None):
|
||
"""封存本轮 prompt、金标、图片与供应商结构化输出契约的确定性 hash。"""
|
||
golden_id = "playtest.judge" if eval_id in PLAYTEST_JUDGE_IDS else eval_id
|
||
eval_dir = Path(eval_root) / golden_id
|
||
inputs_bytes = (eval_dir / "inputs.jsonl").read_bytes()
|
||
labels_bytes = (eval_dir / "labels.jsonl").read_bytes()
|
||
eval_root_resolved = Path(eval_root).resolve()
|
||
image_blobs = []
|
||
image_refs = {
|
||
ref
|
||
for sample in samples
|
||
for ref in _iter_image_fixture_refs(sample.get("input"))
|
||
}
|
||
for ref in sorted(image_refs, key=lambda item: item.encode("utf-8")):
|
||
resolved = (Path(eval_root) / ref).resolve(strict=True)
|
||
resolved.relative_to(eval_root_resolved)
|
||
image_blobs.append((ref, resolved.read_bytes()))
|
||
inputs_hash = hashlib.sha256(inputs_bytes).hexdigest()
|
||
labels_hash = hashlib.sha256(labels_bytes).hexdigest()
|
||
images_hash = _hash_named_blobs("prompt-eval-images/1", image_blobs)
|
||
overall_hash = _hash_named_blobs("prompt-eval-fixtures/1", [
|
||
("inputs.jsonl", inputs_bytes),
|
||
("labels.jsonl", labels_bytes),
|
||
("image-fixtures.sha256", bytes.fromhex(images_hash)),
|
||
])
|
||
snapshot = {
|
||
"promptBodySha256": hashlib.sha256(contract["body"].encode("utf-8")).hexdigest(),
|
||
"fixtures": {
|
||
"inputsSha256": inputs_hash,
|
||
"labelsSha256": labels_hash,
|
||
"imageFixturesSha256": images_hash,
|
||
"overallSha256": overall_hash,
|
||
},
|
||
}
|
||
if provider_output_schema is not None:
|
||
snapshot["providerOutputSchemaSha256"] = hashlib.sha256(
|
||
stable_json(provider_output_schema).encode("utf-8")
|
||
).hexdigest()
|
||
return snapshot
|
||
|
||
|
||
def strip_prompt_input_placeholder(body):
|
||
"""复刻 runner 的 prompt 输入占位清理，避免 system 与 user 重复携带同一包。"""
|
||
without_block = re.sub(
|
||
r'\n*【本轮\s+(?:ActorView|JudgePackage)】\s*\r?\n\s*\{\{\s*input\.prompt\s*\}\}\s*$',
|
||
'',
|
||
str(body or ''),
|
||
)
|
||
cleaned = re.sub(r'\{\{\s*input\.prompt\s*\}\}', '', without_block).strip()
|
||
if not cleaned:
|
||
raise ValueError("移除 input.prompt 占位后 prompt 正文为空")
|
||
if re.search(r'\{\{\s*input\.prompt\s*\}\}', cleaned):
|
||
raise ValueError("prompt 仍残留 input.prompt 占位")
|
||
return cleaned
|
||
|
||
|
||
def _decode_image_data_url(value, label):
|
||
"""严格解码 PNG data URL；损坏的金标不能在花模型费用后才暴露。"""
|
||
if not isinstance(value, str) or not value.startswith("data:image/png;base64,"):
|
||
raise ValueError(f"{label} 必须是 PNG base64 data URL")
|
||
encoded = value.split(",", 1)[1]
|
||
try:
|
||
image_bytes = base64.b64decode(encoded, validate=True)
|
||
except (ValueError, binascii.Error) as exc:
|
||
raise ValueError(f"{label} 不是合法 base64 图片") from exc
|
||
if not image_bytes:
|
||
raise ValueError(f"{label} 图片为空")
|
||
return image_bytes
|
||
|
||
|
||
def _validate_png_bytes(image_bytes, label):
|
||
"""完整解码 PNG，校验格式、体积和像素上限，拒绝伪 chunk 与占位图。"""
|
||
if len(image_bytes) < MIN_EVAL_IMAGE_BYTES:
|
||
raise ValueError(f"{label} 图片体积过小（{len(image_bytes)}B），不得使用占位图")
|
||
if len(image_bytes) > MAX_EVAL_IMAGE_BYTES:
|
||
raise ValueError(f"{label} 图片超过 {MAX_EVAL_IMAGE_BYTES}B 上限")
|
||
if len(image_bytes) < 24 or image_bytes[:8] != PNG_SIGNATURE:
|
||
raise ValueError(f"{label} 不是合法 PNG")
|
||
if Image is None:
|
||
raise ValueError(f"{label} 无法完整解码：当前 Python 缺 Pillow")
|
||
try:
|
||
with Image.open(io.BytesIO(image_bytes)) as image:
|
||
if image.format != "PNG":
|
||
raise ValueError(f"{label} 实际格式不是 PNG")
|
||
width, height = image.size
|
||
if width * height > MAX_EVAL_IMAGE_PIXELS:
|
||
raise ValueError(f"{label} 像素数超过 {MAX_EVAL_IMAGE_PIXELS} 上限")
|
||
image.verify()
|
||
# verify 只检查容器；重新打开并 load，强制执行 IDAT 解压和像素解码。
|
||
with Image.open(io.BytesIO(image_bytes)) as image:
|
||
image.load()
|
||
except (OSError, SyntaxError, UnidentifiedImageError) as exc:
|
||
raise ValueError(f"{label} PNG 解码失败:{exc}") from exc
|
||
if width < MIN_EVAL_IMAGE_EDGE or height < MIN_EVAL_IMAGE_EDGE:
|
||
raise ValueError(
|
||
f"{label} 尺寸 {width}x{height} 低于 {MIN_EVAL_IMAGE_EDGE}px，不得使用占位图"
|
||
)
|
||
return width, height
|
||
|
||
|
||
def _normalize_eval_frame(image_bytes, label, target_size=EVAL_FRAME_SIZE):
|
||
"""把历史 DPR=2 真截图确定性缩放为生产 390x844@1 消费帧。"""
|
||
width, height = _validate_png_bytes(image_bytes, label)
|
||
if (width, height) == target_size:
|
||
return image_bytes
|
||
if Image is None:
|
||
raise ValueError(f"{label} 无法缩放：当前 Python 缺 Pillow")
|
||
if width % target_size[0] != 0 or height % target_size[1] != 0 \
|
||
or width // target_size[0] != height // target_size[1]:
|
||
raise ValueError(
|
||
f"{label} 尺寸 {width}x{height} 不能按整数 DPR 缩放为 {target_size[0]}x{target_size[1]}"
|
||
)
|
||
try:
|
||
with Image.open(io.BytesIO(image_bytes)) as source:
|
||
rendered = source.convert("RGBA" if source.mode == "RGBA" else "RGB").resize(
|
||
target_size, Image.Resampling.LANCZOS
|
||
)
|
||
output = io.BytesIO()
|
||
rendered.save(output, format="PNG", optimize=False, compress_level=6)
|
||
except (OSError, ValueError) as exc:
|
||
raise ValueError(f"{label} 缩放失败:{exc}") from exc
|
||
normalized = output.getvalue()
|
||
if _validate_png_bytes(normalized, f"{label} 生产尺寸帧") != target_size:
|
||
raise ValueError(f"{label} 缩放后尺寸不符合生产视口")
|
||
return normalized
|
||
|
||
|
||
def _resolve_eval_image(image_row, label, *, normalize_frame=True):
|
||
"""读取一份 eval 图片并在调用前转 data URL；路径必须受 EVAL_DIR 约束且不可与内嵌图并存。"""
|
||
if not isinstance(image_row, dict):
|
||
raise ValueError(f"{label} 图片描述必须是对象")
|
||
has_path = "imagePath" in image_row
|
||
has_data_url = "imageDataUrl" in image_row
|
||
if has_path == has_data_url:
|
||
raise ValueError(f"{label} 必须且只能提供 imagePath 或 imageDataUrl")
|
||
|
||
if has_path:
|
||
image_path = image_row.get("imagePath")
|
||
if not isinstance(image_path, str) or not image_path.strip():
|
||
raise ValueError(f"{label}.imagePath 必须是非空字符串")
|
||
relative = Path(image_path)
|
||
if relative.is_absolute():
|
||
raise ValueError(f"{label}.imagePath 不得使用绝对路径")
|
||
candidate = EVAL_DIR / relative
|
||
try:
|
||
resolved = candidate.resolve(strict=True)
|
||
resolved.relative_to(EVAL_DIR.resolve())
|
||
except (OSError, ValueError) as exc:
|
||
raise ValueError(f"{label}.imagePath 必须指向 eval 根内现存文件") from exc
|
||
if candidate.is_symlink() or not resolved.is_file() or resolved.suffix.lower() != ".png":
|
||
raise ValueError(f"{label}.imagePath 必须指向 eval 根内普通 PNG 文件")
|
||
try:
|
||
image_bytes = resolved.read_bytes()
|
||
except OSError as exc:
|
||
raise ValueError(f"{label}.imagePath 读取失败:{exc}") from exc
|
||
_validate_png_bytes(image_bytes, label)
|
||
if normalize_frame:
|
||
image_bytes = _normalize_eval_frame(image_bytes, label)
|
||
image_data_url = "data:image/png;base64," + base64.b64encode(image_bytes).decode("ascii")
|
||
return image_data_url, image_bytes
|
||
|
||
image_data_url = image_row.get("imageDataUrl")
|
||
image_bytes = _decode_image_data_url(image_data_url, f"{label}.imageDataUrl")
|
||
_validate_png_bytes(image_bytes, label)
|
||
if normalize_frame:
|
||
image_bytes = _normalize_eval_frame(image_bytes, label)
|
||
image_data_url = "data:image/png;base64," + base64.b64encode(image_bytes).decode("ascii")
|
||
return image_data_url, image_bytes
|
||
|
||
|
||
def _build_eval_visual_target_context(actor_view, clean_bytes):
|
||
"""调用生产 renderer 和 catalog projector，生成与当前 clean 帧 hash 绑定的 target guide。"""
|
||
if not VISUAL_TARGET_RENDERER.is_file():
|
||
raise ValueError(f"VisualTarget renderer 不存在:{VISUAL_TARGET_RENDERER}")
|
||
with tempfile.TemporaryDirectory(prefix="playtest-actor-targets-") as temp_dir:
|
||
root = Path(temp_dir)
|
||
source = root / "current.clean.png"
|
||
target_set_file = root / "current.targets.json"
|
||
guide_file = root / "current.target-guide.png"
|
||
manifest_file = root / "current.target-guide.manifest.json"
|
||
source.write_bytes(clean_bytes)
|
||
argv = [
|
||
str(VISUAL_TARGET_PYTHON if VISUAL_TARGET_PYTHON.is_file() else Path(sys.executable)),
|
||
str(VISUAL_TARGET_RENDERER),
|
||
"--source", str(source),
|
||
"--expected-source-hash", actor_view["currentFrameHash"],
|
||
"--source-frame-ref", actor_view["currentFrameRef"],
|
||
"--target-set-output", str(target_set_file),
|
||
"--target-set-ref", "actor/targets/current.targets.json",
|
||
"--guide-output", str(guide_file),
|
||
"--guide-ref", "actor/targets/current.target-guide.png",
|
||
"--guide-manifest-output", str(manifest_file),
|
||
]
|
||
proc = subprocess.run(argv, capture_output=True, text=True, timeout=20, check=False)
|
||
if proc.returncode != 0:
|
||
raise ValueError(
|
||
"VisualTarget renderer 失败:"
|
||
+ (proc.stderr.strip() or proc.stdout.strip() or str(proc.returncode))
|
||
)
|
||
try:
|
||
target_set = json.loads(target_set_file.read_text(encoding="utf-8"))
|
||
manifest = json.loads(manifest_file.read_text(encoding="utf-8"))
|
||
guide_bytes = guide_file.read_bytes()
|
||
except (OSError, json.JSONDecodeError) as exc:
|
||
raise ValueError(f"VisualTarget 产物不可读:{exc}") from exc
|
||
catalog = invoke_production_target_catalog(target_set)
|
||
if not catalog.get("ok"):
|
||
raise ValueError(f"生产 VisualTarget catalog 投影失败:{catalog}")
|
||
projected = catalog.get("catalog")
|
||
guide_hash = hashlib.sha256(guide_bytes).hexdigest()
|
||
if manifest.get("guideHash") != guide_hash \
|
||
or manifest.get("sourceFrameHash") != actor_view["currentFrameHash"] \
|
||
or manifest.get("targetSetHash") != projected.get("targetSetHash"):
|
||
raise ValueError("target guide 未与 source/TargetSet/guide 三方 hash 绑定")
|
||
_validate_png_bytes(guide_bytes, "Actor target guide")
|
||
return {
|
||
"catalog": projected,
|
||
"manifest": manifest,
|
||
"targetSet": target_set,
|
||
"cleanBytes": clean_bytes,
|
||
"guideBytes": guide_bytes,
|
||
"dataUrl": "data:image/png;base64," + base64.b64encode(guide_bytes).decode("ascii"),
|
||
}
|
||
|
||
|
||
def materialize_eval_actor_input(input_row):
|
||
"""由 eval clean 帧确定性补入生产 VisualTarget catalog，返回真正发送给模型的 ActorView/3。"""
|
||
actor_base = input_row.get("actorView")
|
||
if not isinstance(actor_base, dict):
|
||
raise ValueError("Actor 金标必须包含 actorView 对象")
|
||
expected_base_keys = _ACTOR_VIEW_KEYS - {"visualTargetCatalog"}
|
||
if set(actor_base) != expected_base_keys:
|
||
raise ValueError(
|
||
"Actor eval 基础投影字段漂移，"
|
||
f"缺={sorted(expected_base_keys - set(actor_base))} "
|
||
f"多={sorted(set(actor_base) - expected_base_keys)}"
|
||
)
|
||
_image_data_url, image_bytes = _resolve_eval_image(input_row, "Actor 当前帧")
|
||
frame_hash = actor_base.get("currentFrameHash")
|
||
if not isinstance(frame_hash, str) \
|
||
or hashlib.sha256(image_bytes).hexdigest() != frame_hash:
|
||
raise ValueError("Actor currentFrameHash 与图片原始字节不一致")
|
||
context = _build_eval_visual_target_context(actor_base, image_bytes)
|
||
expected_target_hash = input_row.get("expectedTargetSetHash")
|
||
if not isinstance(expected_target_hash, str) or not _SHA256_RE.fullmatch(expected_target_hash) \
|
||
or context["catalog"].get("targetSetHash") != expected_target_hash:
|
||
raise ValueError("Actor expectedTargetSetHash 与生产 renderer 输出漂移")
|
||
actor_view = copy.deepcopy(actor_base)
|
||
actor_view["visualTargetCatalog"] = context["catalog"]
|
||
errors = _validate_actor_view(actor_view)
|
||
if errors:
|
||
raise ValueError("ActorView 非生产投影:" + "；".join(errors))
|
||
return actor_view, context
|
||
|
||
|
||
def build_model_request_material(prompt_id, body, input_row):
|
||
"""构造真模型消息，并返回审计所需的实际图片与 TargetGuide 物化产物。"""
|
||
if prompt_id == "playtest.actor":
|
||
actor_view, context = materialize_eval_actor_input(input_row)
|
||
manifest = context["manifest"]
|
||
messages = [
|
||
{"role": "system", "content": strip_prompt_input_placeholder(body)},
|
||
{"role": "user", "content": [
|
||
{"type": "text", "text": stable_json(actor_view)},
|
||
{"type": "text", "text": (
|
||
f"targetGuideRef:{manifest['guideRef']} hash:{manifest['guideHash']} "
|
||
f"targetSetHash:{manifest['targetSetHash']}"
|
||
)},
|
||
{"type": "image_url", "image_url": {"url": context["dataUrl"]}},
|
||
]},
|
||
]
|
||
return messages, ANTHROPIC_ACTOR_MAX_TOKENS, {
|
||
"images": [{"role": "actor-target-guide", "bytes": context["guideBytes"]}],
|
||
"actor": {
|
||
"cleanFrame": context["cleanBytes"],
|
||
"targetSet": context["targetSet"],
|
||
"guide": context["guideBytes"],
|
||
"manifest": context["manifest"],
|
||
},
|
||
}
|
||
|
||
if prompt_id in PLAYTEST_JUDGE_IDS:
|
||
judge_package = input_row.get("judgePackage")
|
||
images = input_row.get("images")
|
||
if not isinstance(judge_package, dict) or not isinstance(images, list):
|
||
raise ValueError("Judge 金标必须包含 judgePackage 对象与 images 数组")
|
||
package_errors = _validate_judge_package(judge_package)
|
||
if package_errors:
|
||
raise ValueError("JudgePackage 非生产投影:" + "；".join(package_errors))
|
||
package_text = stable_json(judge_package)
|
||
sealed = invoke_production_sealed_json(judge_package, project="judge")
|
||
if not sealed.get("ok") or sealed.get("compact") != package_text \
|
||
or stable_json(sealed.get("projected")) != package_text:
|
||
raise ValueError(f"JudgePackage 生产封存序列化失败:{sealed}")
|
||
package_hash = sealed["hash"]
|
||
materialized_images = []
|
||
user_content = [{
|
||
"type": "text",
|
||
"text": f"JudgePackageHash:{package_hash}\n{package_text}",
|
||
}]
|
||
for index, image_row in enumerate(images):
|
||
if not isinstance(image_row, dict):
|
||
raise ValueError(f"Judge images[{index}] 必须是对象")
|
||
frame_ref = image_row.get("frameRef")
|
||
frame_hash = image_row.get("hash")
|
||
if not isinstance(frame_ref, str) or not frame_ref or not isinstance(frame_hash, str) \
|
||
or not re.fullmatch(r'[0-9a-f]{64}', frame_hash):
|
||
raise ValueError(f"Judge images[{index}] 的 frameRef/hash 非法")
|
||
image_data_url, image_bytes = _resolve_eval_image(
|
||
image_row, f"Judge images[{index}]"
|
||
)
|
||
if hashlib.sha256(image_bytes).hexdigest() != frame_hash:
|
||
raise ValueError(f"Judge images[{index}] 的 hash 与图片原始字节不一致")
|
||
materialized_images.append({
|
||
"role": "judge-frame", "frameRef": frame_ref, "bytes": image_bytes,
|
||
})
|
||
user_content.append({"type": "text", "text": f"frameRef:{frame_ref} hash:{frame_hash}"})
|
||
user_content.append({"type": "image_url", "image_url": {"url": image_data_url}})
|
||
return ([
|
||
{"role": "system", "content": strip_prompt_input_placeholder(body)},
|
||
{"role": "user", "content": user_content},
|
||
], ANTHROPIC_JUDGE_MAX_TOKENS, {
|
||
"images": materialized_images, "judgePackageHash": package_hash,
|
||
})
|
||
|
||
rendered = body.replace("{{input.prompt}}", str(input_row.get("prompt", "")))
|
||
return ([{"role": "user", "content": rendered}], None, {"images": []})
|
||
|
||
|
||
def build_model_messages(prompt_id, body, input_row):
|
||
"""兼容旧调用方的二元返回；真跑路径使用 build_model_request_material 获取完整审计材料。"""
|
||
messages, max_tokens, _material = build_model_request_material(prompt_id, body, input_row)
|
||
return messages, max_tokens
|
||
|
||
|
||
def parse_json_object_strict(content):
|
||
"""接受裸 JSON 或唯一 fenced JSON 块；任何前后解释、多个代码块仍 fail-closed。"""
|
||
if content is None:
|
||
return None
|
||
text = str(content).strip()
|
||
fenced = re.fullmatch(r"```(?:json)?\s*\n?(\{.*\})\s*```", text, flags=re.DOTALL | re.IGNORECASE)
|
||
if fenced:
|
||
text = fenced.group(1)
|
||
try:
|
||
obj = json.loads(text)
|
||
except json.JSONDecodeError:
|
||
return None
|
||
return obj if isinstance(obj, dict) else None
|
||
|
||
|
||
def invoke_production_parser(role, raw, candidates=None, evidence_context=None):
|
||
"""经 Node 薄桥调用生产 parser；桥或 parser 异常均 fail-closed。"""
|
||
request = {
|
||
"role": role,
|
||
"raw": "" if raw is None else str(raw),
|
||
"candidates": candidates or [],
|
||
}
|
||
if evidence_context is not None:
|
||
request["evidenceContext"] = evidence_context
|
||
try:
|
||
proc = subprocess.run(
|
||
["node", str(PRODUCTION_PARSER_BRIDGE)],
|
||
input=json.dumps(request, ensure_ascii=False),
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=10,
|
||
check=False,
|
||
)
|
||
except (OSError, subprocess.SubprocessError) as exc:
|
||
return {"ok": False, "code": "production_parser_unavailable", "message": str(exc)}
|
||
try:
|
||
result = json.loads(proc.stdout.strip())
|
||
except json.JSONDecodeError:
|
||
return {
|
||
"ok": False,
|
||
"code": "production_parser_invalid_response",
|
||
"message": proc.stderr.strip() or proc.stdout.strip() or f"exit={proc.returncode}",
|
||
}
|
||
if not isinstance(result, dict) or not isinstance(result.get("ok"), bool):
|
||
return {"ok": False, "code": "production_parser_invalid_response", "message": str(result)}
|
||
return result
|
||
|
||
|
||
def invoke_production_target_catalog(target_set):
|
||
"""经 Node 薄桥调用生产 VisualTarget catalog projector，避免 eval 复制可见字段口径。"""
|
||
request = {"role": "target-catalog", "targetSet": target_set}
|
||
try:
|
||
proc = subprocess.run(
|
||
["node", str(PRODUCTION_PARSER_BRIDGE)],
|
||
input=json.dumps(request, ensure_ascii=False),
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=10,
|
||
check=False,
|
||
)
|
||
except (OSError, subprocess.SubprocessError) as exc:
|
||
return {"ok": False, "code": "production_target_catalog_unavailable", "message": str(exc)}
|
||
try:
|
||
result = json.loads(proc.stdout.strip())
|
||
except json.JSONDecodeError:
|
||
return {
|
||
"ok": False,
|
||
"code": "production_target_catalog_invalid_response",
|
||
"message": proc.stderr.strip() or proc.stdout.strip() or f"exit={proc.returncode}",
|
||
}
|
||
return result if isinstance(result, dict) else {
|
||
"ok": False, "code": "production_target_catalog_invalid_response", "message": str(result),
|
||
}
|
||
|
||
|
||
def invoke_production_sealed_json(value, *, project=None):
|
||
"""调用运行时唯一封存序列化实现，返回生产 pretty bytes hash 与紧凑消息文本。"""
|
||
request = {"role": "sealed-json", "value": value}
|
||
if project is not None:
|
||
request["project"] = project
|
||
try:
|
||
proc = subprocess.run(
|
||
["node", str(PRODUCTION_PARSER_BRIDGE)],
|
||
input=json.dumps(request, ensure_ascii=False),
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=10,
|
||
check=False,
|
||
)
|
||
except (OSError, subprocess.SubprocessError) as exc:
|
||
return {"ok": False, "code": "production_sealer_unavailable", "message": str(exc)}
|
||
try:
|
||
result = json.loads(proc.stdout.strip())
|
||
except json.JSONDecodeError:
|
||
return {
|
||
"ok": False,
|
||
"code": "production_sealer_invalid_response",
|
||
"message": proc.stderr.strip() or proc.stdout.strip() or f"exit={proc.returncode}",
|
||
}
|
||
if not isinstance(result, dict) or result.get("ok") is not True \
|
||
or not isinstance(result.get("hash"), str) or not isinstance(result.get("compact"), str):
|
||
return {"ok": False, "code": "production_sealer_invalid_response", "message": str(result)}
|
||
return result
|
||
|
||
|
||
def _actor_guide_last_tap(actor_view, frame_ref, frame_hash):
|
||
"""从生产 ActorView 历史中还原该 post 帧对应的最近成功 tap 审计值。"""
|
||
for row in reversed(actor_view.get("history", [])):
|
||
post_ref = row.get("postFrameRef") if isinstance(row, dict) else None
|
||
normalized = row.get("normalized") if isinstance(row, dict) else None
|
||
if not isinstance(post_ref, dict) or not isinstance(normalized, dict):
|
||
continue
|
||
if post_ref.get("path") == frame_ref and row.get("postFrameHash") == frame_hash \
|
||
and row.get("valid") is True and normalized.get("type") == "tap":
|
||
return {
|
||
"actionId": row.get("actionId"),
|
||
"x": normalized.get("x"),
|
||
"y": normalized.get("y"),
|
||
}
|
||
return None
|
||
|
||
|
||
def build_eval_actor_guides(actor_view, frames):
|
||
"""调用生产唯一 renderer，为 eval clean 帧生成双 hash 绑定的 Actor guide。"""
|
||
if not ACTOR_GUIDE_RENDERER.is_file():
|
||
raise ValueError(f"Actor guide renderer 不存在:{ACTOR_GUIDE_RENDERER}")
|
||
guides = []
|
||
with tempfile.TemporaryDirectory(prefix="playtest-actor-guide-") as temp_dir:
|
||
root = Path(temp_dir)
|
||
for index, frame in enumerate(frames):
|
||
frame_ref = frame.get("ref") if isinstance(frame, dict) else None
|
||
frame_hash = frame.get("hash") if isinstance(frame, dict) else None
|
||
if not isinstance(frame_ref, str) or not isinstance(frame_hash, str):
|
||
raise ValueError(f"Actor clean frame[{index}] 缺 ref/hash")
|
||
clean_bytes = _decode_image_data_url(
|
||
frame.get("dataUrl"), f"Actor clean frame[{index}]"
|
||
)
|
||
if hashlib.sha256(clean_bytes).hexdigest() != frame_hash:
|
||
raise ValueError(f"Actor clean frame[{index}] 字节与 hash 不一致")
|
||
source_file = root / f"frame-{index}.clean.png"
|
||
guide_file = root / f"frame-{index}.guide.png"
|
||
source_file.write_bytes(clean_bytes)
|
||
last_tap = _actor_guide_last_tap(actor_view, frame_ref, frame_hash)
|
||
argv = [
|
||
sys.executable, str(ACTOR_GUIDE_RENDERER),
|
||
"--source", str(source_file), "--output", str(guide_file),
|
||
"--expected-source-hash", frame_hash,
|
||
"--guide-version", "ActorGuide/1",
|
||
]
|
||
if last_tap is not None:
|
||
argv.extend([
|
||
"--last-tap-x", str(last_tap["x"]),
|
||
"--last-tap-y", str(last_tap["y"]),
|
||
])
|
||
proc = subprocess.run(
|
||
argv, capture_output=True, text=True, timeout=20, check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise ValueError(
|
||
f"Actor guide[{index}] 生产 renderer 失败:"
|
||
f"{proc.stderr.strip() or proc.stdout.strip() or proc.returncode}"
|
||
)
|
||
try:
|
||
manifest = json.loads(proc.stdout.strip())
|
||
guide_bytes = guide_file.read_bytes()
|
||
except (OSError, json.JSONDecodeError) as exc:
|
||
raise ValueError(f"Actor guide[{index}] 清单或字节不可读:{exc}") from exc
|
||
guide_hash = hashlib.sha256(guide_bytes).hexdigest()
|
||
expected_tap = None if last_tap is None else {
|
||
"x": last_tap["x"], "y": last_tap["y"],
|
||
}
|
||
expected_manifest = {
|
||
"guideVersion": "ActorGuide/1",
|
||
"sourceHash": frame_hash,
|
||
"guideHash": guide_hash,
|
||
"width": EVAL_FRAME_SIZE[0],
|
||
"height": EVAL_FRAME_SIZE[1],
|
||
"lastTap": expected_tap,
|
||
}
|
||
if manifest != expected_manifest or guide_hash == frame_hash:
|
||
raise ValueError(f"Actor guide[{index}] 清单未与 source/guide 双 hash 绑定")
|
||
_validate_png_bytes(guide_bytes, f"Actor guide[{index}]")
|
||
guides.append({
|
||
"sourceFrameRef": frame_ref,
|
||
"sourceFrameHash": frame_hash,
|
||
"guideRef": f"actor/guides/eval-{index}.guide.png",
|
||
"guideHash": guide_hash,
|
||
"guideVersion": "ActorGuide/1",
|
||
"lastTap": last_tap,
|
||
"dataUrl": "data:image/png;base64," + base64.b64encode(guide_bytes).decode("ascii"),
|
||
})
|
||
return guides
|
||
|
||
|
||
def invoke_production_actor_context(actor_view, frames, guides):
|
||
"""经 Node 薄桥调用生产 Actor 多模态上下文组装器，避免 eval 自己猜历史帧顺序。"""
|
||
request = {
|
||
"role": "actor-context", "actorView": actor_view,
|
||
"frames": frames, "guides": guides,
|
||
}
|
||
try:
|
||
proc = subprocess.run(
|
||
["node", str(PRODUCTION_PARSER_BRIDGE)],
|
||
input=json.dumps(request, ensure_ascii=False),
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=20,
|
||
check=False,
|
||
)
|
||
except (OSError, subprocess.SubprocessError) as exc:
|
||
return {"ok": False, "code": "production_actor_context_unavailable", "message": str(exc)}
|
||
try:
|
||
result = json.loads(proc.stdout.strip())
|
||
except json.JSONDecodeError:
|
||
return {
|
||
"ok": False,
|
||
"code": "production_actor_context_invalid_response",
|
||
"message": proc.stderr.strip() or proc.stdout.strip() or f"exit={proc.returncode}",
|
||
}
|
||
if not isinstance(result, dict) or result.get("ok") is not True \
|
||
or not isinstance(result.get("userContent"), list) \
|
||
or not isinstance(result.get("manifest"), dict):
|
||
return {"ok": False, "code": "production_actor_context_invalid_response", "message": str(result)}
|
||
return result
|
||
|
||
|
||
def parse_model_output(prompt_id, content, input_row):
|
||
"""按生产消费面解析模型输出，返回统一的 ok/normalized/code 结构。"""
|
||
if prompt_id == "playtest.actor":
|
||
try:
|
||
actor_view, _context = materialize_eval_actor_input(input_row)
|
||
except ValueError as exc:
|
||
return {"ok": False, "code": "actor_eval_context_invalid", "message": str(exc)}
|
||
return invoke_production_parser(
|
||
"actor-selection", content, evidence_context=actor_view
|
||
)
|
||
if prompt_id in PLAYTEST_JUDGE_IDS:
|
||
package = input_row.get("judgePackage")
|
||
candidates = package.get("proofObligations", []) if isinstance(package, dict) else []
|
||
return invoke_production_parser("judge", content, candidates, package)
|
||
obj = parse_json_object_strict(content)
|
||
if obj is None:
|
||
return {"ok": False, "code": "invalid_json", "message": "模型必须只输出一个合法 JSON 对象"}
|
||
return {"ok": True, "normalized": obj}
|
||
|
||
|
||
_TYPE_PY = {"boolean": bool, "string": str, "number": (int, float),
|
||
"integer": int, "object": dict, "array": list}
|
||
|
||
|
||
def gate1_schema_ok(obj, schema):
|
||
"""闸1：产物合 output_schema（是对象 + 含 required 字段 + 字段类型匹配）。返回 (ok, 失败字段列表)。"""
|
||
if not isinstance(obj, dict):
|
||
return False, ["<非 JSON 对象>"]
|
||
if schema is None:
|
||
return True, [] # 无 schema 声明 → 只要求是对象（上面已保证）
|
||
bad = []
|
||
for field in schema.get("required", []):
|
||
if field not in obj:
|
||
bad.append(f"缺字段 {field}")
|
||
for field, tname in schema.get("types", {}).items():
|
||
if field in obj:
|
||
pytype = _TYPE_PY.get(tname)
|
||
if pytype and not isinstance(obj[field], pytype):
|
||
bad.append(f"字段 {field} 类型应为 {tname}")
|
||
return (len(bad) == 0), bad
|
||
|
||
|
||
_SHA256_RE = re.compile(r'^[0-9a-f]{64}$')
|
||
_ACTION_ID_RE = re.compile(r'^action-[1-9][0-9]*$')
|
||
_PRODUCTION_FRAME_RE = re.compile(r'^frames/(?:frame-initial|action-[1-9][0-9]*-(?:pre|post))\.png$')
|
||
_POST_FRAME_RE = re.compile(r'^frames/action-[1-9][0-9]*-post\.png$')
|
||
_ACTOR_VIEW_KEYS = {
|
||
"schemaVersion", "projectionVersion", "brief", "currentFrameRef", "currentFrameHash",
|
||
"viewport", "legalSelections", "visualTargetCatalog", "history", "protocolErrors",
|
||
"remainingBudget", "virtualTimeMs", "objectiveProgress",
|
||
}
|
||
_ACTOR_HISTORY_KEYS = {
|
||
"actionId", "selection", "normalized", "valid", "retryCount", "preFrameRef",
|
||
"preFrameHash", "preEventSeq", "virtualTimeBudgetMs", "postFrameRef",
|
||
"postFrameHash", "postEventSeq", "causalGameResponse", "gameEvents", "protocolErrors",
|
||
}
|
||
_OBJECTIVE_KEYS = {"candidateOnly", "snapshotSeq", "hash", "obligations"}
|
||
_OBJECTIVE_OBLIGATION_KEYS = {
|
||
"id", "title", "seenStepTitles", "missingStepTitles", "status",
|
||
}
|
||
_JUDGE_PACKAGE_KEYS = {
|
||
"schemaVersion", "projectionVersion", "packageType", "gameId", "genre",
|
||
"templateRoute", "proofProfileId", "proofRegistryVersion", "taskBindingHash",
|
||
"acceptanceRequestHash", "evidenceMode", "brief", "artifactHash", "briefHash",
|
||
"registry", "proofObligations", "actions", "frames", "gameEvents", "textReferenceScope",
|
||
"environment", "floor", "proofRef", "proofHash",
|
||
}
|
||
_TEXT_REFERENCE_SCOPE_KEYS = {"actionRefs", "frameRefs", "eventRefs"}
|
||
_JUDGE_REGISTRY_KEYS = {
|
||
"schemaVersion", "registryVersion", "proofProfileId", "genre", "templateRoute",
|
||
"briefRuleMatches",
|
||
}
|
||
_JUDGE_ACTION_KEYS = {
|
||
"actionId", "normalized", "valid", "preFrameRef", "preFrameHash", "preEventSeq",
|
||
"virtualTimeBudgetMs", "postFrameRef", "postFrameHash", "postEventSeq", "gameEvents",
|
||
}
|
||
_JUDGE_FRAME_KEYS = {"actionId", "kind", "ref", "hash", "artifactHash"}
|
||
_JUDGE_FRAME_OPTIONAL_KEYS = {"frameId"}
|
||
_CANDIDATE_REQUIRED_KEYS = {
|
||
"id", "required", "candidateOnly", "candidateState", "candidateProblems",
|
||
"sequenceRefs", "actionRefs", "postFrameRefs", "eventRefs", "evidenceRefs",
|
||
"seenStepTitles", "missingStepTitles",
|
||
}
|
||
_CANDIDATE_OPTIONAL_KEYS = {"title", "description", "evidence"}
|
||
_SEQUENCE_REF_KEYS = {"stepId", "eventRef", "actionRef", "postFrameRef"}
|
||
_GAME_EVENT_KEYS = {
|
||
"eventVersion", "evidenceMode", "scope", "namespace", "type", "message", "payload",
|
||
"payloadCanonical", "payloadHash", "producer", "seq", "actionId", "virtualTimeMs",
|
||
"sourceEventVersion", "adapterVersion", "sourceMessageHash", "eventRole",
|
||
}
|
||
_GAME_EVENT_OPTIONAL_KEYS = set()
|
||
_ACTION_TYPES = ["tap", "key", "drag", "wait"]
|
||
_SAFE_KEYS = [
|
||
"ArrowLeft", "ArrowRight", "ArrowUp", "ArrowDown", "Enter", "Escape", "Space",
|
||
"KeyW", "KeyA", "KeyS", "KeyD",
|
||
]
|
||
|
||
|
||
def _exact_keys(value, expected, label):
|
||
"""投影容器必须逐字段同形；任意答案侧信道字段都在这里 fail-closed。"""
|
||
if not isinstance(value, dict):
|
||
return [f"{label} 必须是对象"]
|
||
actual = set(value)
|
||
return [] if actual == expected else [
|
||
f"{label} 字段不等于生产投影，缺={sorted(expected - actual)} 多={sorted(actual - expected)}"
|
||
]
|
||
|
||
|
||
def _unique(values):
|
||
return list(dict.fromkeys(values))
|
||
|
||
|
||
def _validate_sequence_ref(row, label):
|
||
errors = _exact_keys(row, _SEQUENCE_REF_KEYS, label)
|
||
if errors:
|
||
return errors
|
||
if not isinstance(row.get("stepId"), str) or not row["stepId"]:
|
||
errors.append(f"{label}.stepId 非法")
|
||
if isinstance(row.get("eventRef"), bool) or not isinstance(row.get("eventRef"), int) \
|
||
or row["eventRef"] < 1:
|
||
errors.append(f"{label}.eventRef 非法")
|
||
if not isinstance(row.get("actionRef"), str) or not _ACTION_ID_RE.fullmatch(row["actionRef"]):
|
||
errors.append(f"{label}.actionRef 非法")
|
||
if not isinstance(row.get("postFrameRef"), str) \
|
||
or not _POST_FRAME_RE.fullmatch(row["postFrameRef"]):
|
||
errors.append(f"{label}.postFrameRef 不是生产帧引用")
|
||
return errors
|
||
|
||
|
||
def _validate_game_event(event, label):
|
||
if not isinstance(event, dict):
|
||
return [f"{label} 必须是对象"]
|
||
actual = set(event)
|
||
if not _GAME_EVENT_KEYS <= actual or not actual <= (_GAME_EVENT_KEYS | _GAME_EVENT_OPTIONAL_KEYS):
|
||
errors = [
|
||
f"{label} 字段不等于生产投影，缺={sorted(_GAME_EVENT_KEYS - actual)} "
|
||
f"多={sorted(actual - (_GAME_EVENT_KEYS | _GAME_EVENT_OPTIONAL_KEYS))}"
|
||
]
|
||
return errors
|
||
errors = []
|
||
event_type = event.get("type")
|
||
if event.get("eventVersion") != "game-event/1" or event.get("evidenceMode") != "native" \
|
||
or event.get("scope") != "game" or event.get("namespace") != f"game/{event_type}":
|
||
errors.append(f"{label} 信封不是 native game-event/1")
|
||
if not isinstance(event_type, str) or not re.fullmatch(r'[a-z][a-z0-9-]*\.[a-z][a-z0-9-]*', event_type):
|
||
errors.append(f"{label}.type 非法")
|
||
if event.get("eventRole") not in {
|
||
"profile-proof", "cross-profile-observation"}:
|
||
errors.append(f"{label}.eventRole 非法")
|
||
if not isinstance(event.get("seq"), int) or isinstance(event.get("seq"), bool) or event["seq"] < 1:
|
||
errors.append(f"{label}.seq 非法")
|
||
if not isinstance(event.get("actionId"), str) or not _ACTION_ID_RE.fullmatch(event["actionId"]):
|
||
errors.append(f"{label}.actionId 非法")
|
||
canonical = event.get("payloadCanonical")
|
||
if not isinstance(canonical, str) or hashlib.sha256(canonical.encode("utf-8")).hexdigest() \
|
||
!= event.get("payloadHash"):
|
||
errors.append(f"{label}.payloadCanonical/hash 不一致")
|
||
else:
|
||
try:
|
||
parsed_payload = json.loads(canonical)
|
||
except json.JSONDecodeError:
|
||
parsed_payload = object()
|
||
if parsed_payload != event.get("payload"):
|
||
errors.append(f"{label}.payloadCanonical 与 payload 语义不一致")
|
||
if event.get("sourceEventVersion") is not None or event.get("adapterVersion") is not None \
|
||
or event.get("sourceMessageHash") is not None:
|
||
errors.append(f"{label} native 事件不得伪装 legacy 来源")
|
||
return errors
|
||
|
||
|
||
def _validate_actor_view(actor_view):
|
||
"""校验 ActorView/3 可见投影；完整 TargetSet 坐标和 proposer 分数不得进入模型输入。"""
|
||
errors = _exact_keys(actor_view, _ACTOR_VIEW_KEYS, "ActorView")
|
||
if errors:
|
||
return errors
|
||
if actor_view.get("schemaVersion") != "playtest/3" \
|
||
or actor_view.get("projectionVersion") != "ActorView/3":
|
||
errors.append("ActorView 版本不是生产 playtest/3 + ActorView/3")
|
||
if not isinstance(actor_view.get("brief"), str) or not actor_view["brief"].strip():
|
||
errors.append("ActorView.brief 必须是非空字符串")
|
||
if not isinstance(actor_view.get("currentFrameRef"), str) \
|
||
or not _PRODUCTION_FRAME_RE.fullmatch(actor_view["currentFrameRef"]):
|
||
errors.append("ActorView.currentFrameRef 必须使用无语义生产帧名")
|
||
if not isinstance(actor_view.get("currentFrameHash"), str) \
|
||
or not _SHA256_RE.fullmatch(actor_view["currentFrameHash"]):
|
||
errors.append("ActorView.currentFrameHash 非 sha256")
|
||
if actor_view.get("viewport") != {"width": 390, "height": 844, "dpr": 1}:
|
||
errors.append("ActorView.viewport 必须严格为 390x844@1")
|
||
legal_actions = actor_view.get("legalSelections")
|
||
legal_errors = _exact_keys(
|
||
legal_actions, {"types", "safeKeys", "actionBudgetMs", "waitMs"},
|
||
"ActorView.legalSelections",
|
||
)
|
||
errors.extend(legal_errors)
|
||
if not legal_errors:
|
||
action_types = legal_actions.get("types")
|
||
safe_keys = legal_actions.get("safeKeys")
|
||
if not isinstance(action_types, list) or not action_types \
|
||
or action_types != [item for item in _ACTION_TYPES if item in action_types] \
|
||
or len(action_types) != len(set(action_types)):
|
||
errors.append("ActorView.legalSelections.types 必须是按生产顺序排列的非空动作子集")
|
||
if not isinstance(safe_keys, list) \
|
||
or safe_keys != [item for item in _SAFE_KEYS if item in safe_keys] \
|
||
or len(safe_keys) != len(set(safe_keys)):
|
||
errors.append("ActorView.legalSelections.safeKeys 必须是按生产顺序排列的按键子集")
|
||
if isinstance(action_types, list) and isinstance(safe_keys, list):
|
||
if "key" in action_types and not safe_keys:
|
||
errors.append("ActorView 允许 key 时必须至少提供一个 safeKey")
|
||
if "key" not in action_types and safe_keys:
|
||
errors.append("ActorView 未允许 key 时 safeKeys 必须为空")
|
||
if legal_actions.get("actionBudgetMs") != 600 or legal_actions.get("waitMs") != [100, 600]:
|
||
errors.append("ActorView.legalSelections 时间预算与生产常量不一致")
|
||
|
||
catalog = actor_view.get("visualTargetCatalog")
|
||
catalog_errors = _exact_keys(
|
||
catalog, {"schemaVersion", "targetSetHash", "targets"},
|
||
"ActorView.visualTargetCatalog",
|
||
)
|
||
errors.extend(catalog_errors)
|
||
if not catalog_errors:
|
||
if catalog.get("schemaVersion") != "VisualTargetSet/1" \
|
||
or not isinstance(catalog.get("targetSetHash"), str) \
|
||
or not _SHA256_RE.fullmatch(catalog["targetSetHash"]):
|
||
errors.append("ActorView.visualTargetCatalog 版本或 targetSetHash 非法")
|
||
targets = catalog.get("targets")
|
||
if not isinstance(targets, list) or not targets:
|
||
errors.append("ActorView.visualTargetCatalog.targets 必须是非空数组")
|
||
else:
|
||
seen_ids = set()
|
||
for index, target in enumerate(targets):
|
||
prefix = f"ActorView.visualTargetCatalog.targets[{index}]"
|
||
if not isinstance(target, dict):
|
||
errors.append(f"{prefix} 必须是对象")
|
||
continue
|
||
allowed = {"id", "kind", "allowedGestures", "dimensions", "subAnchors", "fallbackOnly"}
|
||
if not {"id", "kind", "allowedGestures"} <= set(target) or set(target) - allowed:
|
||
errors.append(f"{prefix} 字段不在可见 catalog 白名单")
|
||
continue
|
||
target_id = target.get("id")
|
||
kind = target.get("kind")
|
||
if not isinstance(target_id, str) or not re.fullmatch(r'(?:r|l|g)[0-9]{2}', target_id) \
|
||
or target_id in seen_ids:
|
||
errors.append(f"{prefix}.id 非法或重复")
|
||
seen_ids.add(target_id)
|
||
if kind not in {"region", "lattice", "spatial_grid"}:
|
||
errors.append(f"{prefix}.kind 非法")
|
||
gestures = target.get("allowedGestures")
|
||
if not isinstance(gestures, list) or not gestures \
|
||
or any(item not in {"tap", "drag"} for item in gestures):
|
||
errors.append(f"{prefix}.allowedGestures 非法")
|
||
if any(field in target for field in ("bounds", "safePoint", "score")):
|
||
errors.append(f"{prefix} 泄漏坐标或 proposer 分数")
|
||
if kind == "region" and set(target) != {"id", "kind", "allowedGestures"}:
|
||
errors.append(f"{prefix} region 不得携带 cell/subAnchor 元数据")
|
||
if kind in {"lattice", "spatial_grid"}:
|
||
dimensions = target.get("dimensions")
|
||
anchors = target.get("subAnchors")
|
||
if not isinstance(dimensions, dict) or set(dimensions) != {"rows", "columns"} \
|
||
or not all(isinstance(dimensions.get(field), int)
|
||
and not isinstance(dimensions.get(field), bool)
|
||
and dimensions[field] >= 1
|
||
for field in ("rows", "columns")):
|
||
errors.append(f"{prefix}.dimensions 非法")
|
||
if not isinstance(anchors, list) or anchors != [
|
||
"center", "north", "south", "east", "west", "northwest",
|
||
"northeast", "southwest", "southeast",
|
||
]:
|
||
errors.append(f"{prefix}.subAnchors 非法")
|
||
if kind == "spatial_grid" and target.get("fallbackOnly") is not True:
|
||
errors.append(f"{prefix} spatial_grid 必须标 fallbackOnly=true")
|
||
if not isinstance(actor_view.get("remainingBudget"), int) \
|
||
or isinstance(actor_view.get("remainingBudget"), bool) or actor_view["remainingBudget"] < 0:
|
||
errors.append("ActorView.remainingBudget 非法")
|
||
if not isinstance(actor_view.get("virtualTimeMs"), int) \
|
||
or isinstance(actor_view.get("virtualTimeMs"), bool) or actor_view["virtualTimeMs"] < 0:
|
||
errors.append("ActorView.virtualTimeMs 非法")
|
||
|
||
history = actor_view.get("history")
|
||
if not isinstance(history, list):
|
||
errors.append("ActorView.history 必须是数组")
|
||
else:
|
||
for index, row in enumerate(history):
|
||
prefix = f"ActorView.history[{index}]"
|
||
row_errors = _exact_keys(row, _ACTOR_HISTORY_KEYS, prefix)
|
||
errors.extend(row_errors)
|
||
if row_errors:
|
||
continue
|
||
if not isinstance(row.get("actionId"), str) or not _ACTION_ID_RE.fullmatch(row["actionId"]):
|
||
errors.append(f"{prefix}.actionId 非法")
|
||
selection = invoke_production_parser(
|
||
"actor-selection", stable_json(row.get("selection")), evidence_context=actor_view,
|
||
)
|
||
if not selection.get("ok") or selection.get("normalized") != row.get("selection"):
|
||
errors.append(f"{prefix}.selection 不是生产规范 ActorSelection")
|
||
normalized = invoke_production_parser("actor", stable_json(row.get("normalized")))
|
||
if not normalized.get("ok") or normalized.get("normalized") != row.get("normalized"):
|
||
errors.append(f"{prefix}.normalized 不是生产规范坐标动作")
|
||
for frame_field, hash_field in (
|
||
("preFrameRef", "preFrameHash"), ("postFrameRef", "postFrameHash")):
|
||
frame_ref = row.get(frame_field)
|
||
frame_hash = row.get(hash_field)
|
||
if not isinstance(frame_ref, dict) or set(frame_ref) != {"path", "hash"} \
|
||
or not isinstance(frame_ref.get("path"), str) \
|
||
or not _PRODUCTION_FRAME_RE.fullmatch(frame_ref["path"]) \
|
||
or frame_ref.get("hash") != frame_hash \
|
||
or not isinstance(frame_hash, str) or not _SHA256_RE.fullmatch(frame_hash):
|
||
errors.append(f"{prefix}.{frame_field}/{hash_field} 未按生产 ref/hash 绑定")
|
||
if not isinstance(row.get("valid"), bool) \
|
||
or not isinstance(row.get("retryCount"), int) or isinstance(row.get("retryCount"), bool) \
|
||
or not 0 <= row["retryCount"] <= 2:
|
||
errors.append(f"{prefix}.valid/retryCount 非法")
|
||
for field in ("preEventSeq", "virtualTimeBudgetMs", "postEventSeq"):
|
||
if not isinstance(row.get(field), int) or isinstance(row.get(field), bool) or row[field] < 0:
|
||
errors.append(f"{prefix}.{field} 非法")
|
||
if not isinstance(row.get("causalGameResponse"), bool):
|
||
errors.append(f"{prefix}.causalGameResponse 非法")
|
||
if not isinstance(row.get("gameEvents"), list):
|
||
errors.append(f"{prefix}.gameEvents 必须是数组")
|
||
continue
|
||
for event_index, event in enumerate(row.get("gameEvents", [])):
|
||
errors.extend(_validate_game_event(event, f"{prefix}.gameEvents[{event_index}]"))
|
||
if not isinstance(row.get("protocolErrors"), list) \
|
||
or not all(isinstance(item, str) for item in row["protocolErrors"]):
|
||
errors.append(f"{prefix}.protocolErrors 必须是字符串数组")
|
||
protocol_errors = actor_view.get("protocolErrors")
|
||
if not isinstance(protocol_errors, list):
|
||
errors.append("ActorView.protocolErrors 必须是数组")
|
||
else:
|
||
for index, row in enumerate(protocol_errors):
|
||
errors.extend(_exact_keys(row, {"code", "message"}, f"ActorView.protocolErrors[{index}]"))
|
||
|
||
progress = actor_view.get("objectiveProgress")
|
||
progress_errors = _exact_keys(progress, _OBJECTIVE_KEYS, "ActorView.objectiveProgress")
|
||
errors.extend(progress_errors)
|
||
if not progress_errors:
|
||
if progress.get("candidateOnly") is not True or not isinstance(progress.get("snapshotSeq"), int) \
|
||
or isinstance(progress.get("snapshotSeq"), bool) or progress["snapshotSeq"] < 1:
|
||
errors.append("ActorView.objectiveProgress 元数据非法")
|
||
obligations = progress.get("obligations")
|
||
if not isinstance(obligations, list) or not obligations:
|
||
errors.append("ActorView.objectiveProgress.obligations 必须是非空数组")
|
||
else:
|
||
for index, row in enumerate(obligations):
|
||
prefix = f"ActorView.objectiveProgress.obligations[{index}]"
|
||
row_errors = _exact_keys(row, _OBJECTIVE_OBLIGATION_KEYS, prefix)
|
||
errors.extend(row_errors)
|
||
if row_errors:
|
||
continue
|
||
if row.get("status") not in {"unobserved", "partial", "observed"}:
|
||
errors.append(f"{prefix}.status 非法")
|
||
for field in ("seenStepTitles", "missingStepTitles"):
|
||
if not isinstance(row.get(field), list) or not all(
|
||
isinstance(item, str) for item in row[field]
|
||
):
|
||
errors.append(f"{prefix}.{field} 非字符串数组")
|
||
payload = {
|
||
"candidateOnly": progress.get("candidateOnly"),
|
||
"snapshotSeq": progress.get("snapshotSeq"),
|
||
"obligations": progress.get("obligations"),
|
||
}
|
||
expected_hash = hashlib.sha256(stable_json(payload).encode("utf-8")).hexdigest()
|
||
if progress.get("hash") != expected_hash:
|
||
errors.append("ActorView.objectiveProgress.hash 与生产稳定投影不一致")
|
||
return errors
|
||
|
||
|
||
def _validate_candidate(candidate, label):
|
||
if not isinstance(candidate, dict):
|
||
return [f"{label} 必须是对象"]
|
||
actual = set(candidate)
|
||
allowed = _CANDIDATE_REQUIRED_KEYS | _CANDIDATE_OPTIONAL_KEYS
|
||
errors = []
|
||
if not _CANDIDATE_REQUIRED_KEYS <= actual or not actual <= allowed:
|
||
errors.append(
|
||
f"{label} 字段不在生产候选白名单，缺={sorted(_CANDIDATE_REQUIRED_KEYS - actual)} "
|
||
f"多={sorted(actual - allowed)}"
|
||
)
|
||
return errors
|
||
if candidate.get("candidateOnly") is not True \
|
||
or candidate.get("candidateState") not in {"complete", "missing", "contradicted"}:
|
||
errors.append(f"{label} 候选元数据非法")
|
||
sequence = candidate.get("sequenceRefs")
|
||
if not isinstance(sequence, list):
|
||
errors.append(f"{label}.sequenceRefs 必须是数组")
|
||
return errors
|
||
for index, row in enumerate(sequence):
|
||
errors.extend(_validate_sequence_ref(row, f"{label}.sequenceRefs[{index}]"))
|
||
action_refs = _unique([row.get("actionRef") for row in sequence if isinstance(row, dict)])
|
||
frame_refs = _unique([row.get("postFrameRef") for row in sequence if isinstance(row, dict)])
|
||
event_refs = _unique([row.get("eventRef") for row in sequence if isinstance(row, dict)])
|
||
evidence_refs = [*action_refs, *frame_refs, *[f"event:{seq}" for seq in event_refs]]
|
||
if candidate.get("actionRefs") != action_refs or candidate.get("postFrameRefs") != frame_refs \
|
||
or candidate.get("eventRefs") != event_refs or candidate.get("evidenceRefs") != evidence_refs:
|
||
errors.append(f"{label} flat refs 不是 sequenceRefs 的确定性投影")
|
||
if candidate.get("candidateState") == "missing" and sequence:
|
||
errors.append(f"{label} missing 候选不得携带 sequenceRefs")
|
||
for field in ("candidateProblems", "seenStepTitles", "missingStepTitles"):
|
||
if not isinstance(candidate.get(field), list) or not all(
|
||
isinstance(item, str) for item in candidate[field]
|
||
):
|
||
errors.append(f"{label}.{field} 非字符串数组")
|
||
return errors
|
||
|
||
|
||
def _validate_judge_package(package):
|
||
errors = _exact_keys(package, _JUDGE_PACKAGE_KEYS, "JudgePackage")
|
||
if errors:
|
||
return errors
|
||
if package.get("schemaVersion") != "playtest/3" \
|
||
or package.get("projectionVersion") != "JudgePackage/1" \
|
||
or package.get("packageType") != "JudgePackage":
|
||
errors.append("JudgePackage 版本或类型不是生产投影")
|
||
for field in (
|
||
"taskBindingHash", "acceptanceRequestHash", "artifactHash", "briefHash", "proofHash",
|
||
):
|
||
if not isinstance(package.get(field), str) or not _SHA256_RE.fullmatch(package[field]):
|
||
errors.append(f"JudgePackage.{field} 非 sha256")
|
||
if hashlib.sha256(str(package.get("brief", "")).encode("utf-8")).hexdigest() \
|
||
!= package.get("briefHash"):
|
||
errors.append("JudgePackage.briefHash 与 brief 不一致")
|
||
if package.get("evidenceMode") != "native":
|
||
errors.append("JudgePackage.evidenceMode 必须为 native")
|
||
|
||
registry = package.get("registry")
|
||
registry_errors = _exact_keys(registry, _JUDGE_REGISTRY_KEYS, "JudgePackage.registry")
|
||
errors.extend(registry_errors)
|
||
if not registry_errors and (
|
||
registry.get("registryVersion") != package.get("proofRegistryVersion")
|
||
or registry.get("proofProfileId") != package.get("proofProfileId")
|
||
or registry.get("genre") != package.get("genre")
|
||
or registry.get("templateRoute") != package.get("templateRoute")
|
||
or not isinstance(registry.get("briefRuleMatches"), list)
|
||
):
|
||
errors.append("JudgePackage.registry 与包身份不一致")
|
||
|
||
candidates = package.get("proofObligations")
|
||
if not isinstance(candidates, list):
|
||
errors.append("JudgePackage.proofObligations 必须是数组")
|
||
else:
|
||
for index, candidate in enumerate(candidates):
|
||
errors.extend(_validate_candidate(candidate, f"JudgePackage.proofObligations[{index}]"))
|
||
|
||
actions = package.get("actions")
|
||
if not isinstance(actions, list):
|
||
errors.append("JudgePackage.actions 必须是数组")
|
||
else:
|
||
for index, action in enumerate(actions):
|
||
prefix = f"JudgePackage.actions[{index}]"
|
||
row_errors = _exact_keys(action, _JUDGE_ACTION_KEYS, prefix)
|
||
errors.extend(row_errors)
|
||
if row_errors:
|
||
continue
|
||
parsed = invoke_production_parser("actor", stable_json(action.get("normalized")))
|
||
if not parsed.get("ok") or parsed.get("normalized") != action.get("normalized"):
|
||
errors.append(f"{prefix}.normalized 不是生产规范动作")
|
||
for event_index, event in enumerate(action.get("gameEvents", [])):
|
||
errors.extend(_validate_game_event(event, f"{prefix}.gameEvents[{event_index}]"))
|
||
|
||
frames = package.get("frames")
|
||
if not isinstance(frames, list):
|
||
errors.append("JudgePackage.frames 必须是数组")
|
||
else:
|
||
for index, frame in enumerate(frames):
|
||
prefix = f"JudgePackage.frames[{index}]"
|
||
if not isinstance(frame, dict) or not _JUDGE_FRAME_KEYS <= set(frame) \
|
||
or not set(frame) <= (_JUDGE_FRAME_KEYS | _JUDGE_FRAME_OPTIONAL_KEYS):
|
||
errors.append(f"{prefix} 字段不在生产帧白名单")
|
||
continue
|
||
frame_ref = str(frame.get("ref", ""))
|
||
initial_frame = frame.get("kind") == "pre" and frame_ref == "frames/frame-initial.png" \
|
||
and frame.get("actionId") == ""
|
||
action_frame = _ACTION_ID_RE.fullmatch(str(frame.get("actionId", ""))) is not None
|
||
if not (initial_frame or action_frame) or frame.get("kind") not in {"pre", "post"} \
|
||
or not _PRODUCTION_FRAME_RE.fullmatch(frame_ref) \
|
||
or not _SHA256_RE.fullmatch(str(frame.get("hash", ""))) \
|
||
or frame.get("artifactHash") != package.get("artifactHash"):
|
||
errors.append(f"{prefix} 身份/hash 非法")
|
||
|
||
events = package.get("gameEvents")
|
||
if not isinstance(events, list):
|
||
errors.append("JudgePackage.gameEvents 必须是数组")
|
||
else:
|
||
for index, event in enumerate(events):
|
||
errors.extend(_validate_game_event(event, f"JudgePackage.gameEvents[{index}]"))
|
||
|
||
# textReferenceScope 是 runner 从包内事实生成的只读白名单；任何漏项、乱序或伪造都必须在调模型前拦截。
|
||
scope = package.get("textReferenceScope")
|
||
scope_errors = _exact_keys(
|
||
scope, _TEXT_REFERENCE_SCOPE_KEYS, "JudgePackage.textReferenceScope",
|
||
)
|
||
errors.extend(scope_errors)
|
||
if not scope_errors:
|
||
for field in ("actionRefs", "frameRefs", "eventRefs"):
|
||
refs = scope.get(field)
|
||
if not isinstance(refs, list) or not all(
|
||
isinstance(item, str) and bool(item) for item in refs):
|
||
errors.append(f"JudgePackage.textReferenceScope.{field} 必须是字符串数组")
|
||
expected_scope = {
|
||
"actionRefs": _unique([
|
||
action.get("actionId") for action in actions or []
|
||
if isinstance(action, dict)
|
||
and isinstance(action.get("actionId"), str)
|
||
and _ACTION_ID_RE.fullmatch(action["actionId"])
|
||
]),
|
||
"frameRefs": _unique([
|
||
frame.get("ref") for frame in frames or []
|
||
if isinstance(frame, dict)
|
||
and isinstance(frame.get("ref"), str)
|
||
and bool(frame["ref"])
|
||
]),
|
||
"eventRefs": _unique([
|
||
f"event:{event['seq']}" for event in events or []
|
||
if isinstance(event, dict)
|
||
and isinstance(event.get("seq"), int)
|
||
and not isinstance(event.get("seq"), bool)
|
||
and event["seq"] >= 1
|
||
]),
|
||
}
|
||
if scope != expected_scope:
|
||
errors.append(
|
||
"JudgePackage.textReferenceScope 必须与 actions/frames/gameEvents 的生产投影逐项一致"
|
||
)
|
||
|
||
environment = package.get("environment")
|
||
environment_errors = _exact_keys(
|
||
environment, {"host", "chromeVersion", "viewport", "buildRef", "virtualTime"},
|
||
"JudgePackage.environment",
|
||
)
|
||
errors.extend(environment_errors)
|
||
if not environment_errors:
|
||
if environment.get("viewport") != {"width": 390, "height": 844, "dpr": 1} \
|
||
or environment.get("buildRef") != package.get("artifactHash") \
|
||
or environment.get("virtualTime") != {
|
||
"mode": "cdp", "actionQuantumMs": 600, "waitMinMs": 100, "waitMaxMs": 600,
|
||
}:
|
||
errors.append("JudgePackage.environment 与生产 390x844@1/虚拟时间口径不一致")
|
||
if package.get("floor") is not None and not isinstance(package.get("floor"), dict):
|
||
errors.append("JudgePackage.floor 只能是对象或 null")
|
||
if not isinstance(package.get("proofRef"), str) or not package["proofRef"]:
|
||
errors.append("JudgePackage.proofRef 必须是非空引用")
|
||
return errors
|
||
|
||
|
||
_JUDGE_DECISIONS = {"accept", "reject", "inconclusive", "tester_error"}
|
||
_JUDGE_FAILURE_CLASSES = {"none", "broken", "hollow", "off_brief"}
|
||
_JUDGE_OBLIGATION_STATUSES = {"satisfied", "failed", "missing", "contradicted"}
|
||
|
||
|
||
def _actor_brief_leaks_answer(actor_view, expected_action):
|
||
"""Actor 目标投影不得用自然语言、语义文件名或阶段字段直接给出规范动作。"""
|
||
brief = actor_view.get("brief") if isinstance(actor_view, dict) else None
|
||
if not isinstance(brief, str) or not brief.strip():
|
||
return "brief 必须是非空字符串"
|
||
progress = actor_view.get("objectiveProgress") if isinstance(actor_view, dict) else None
|
||
obligations = progress.get("obligations", []) if isinstance(progress, dict) else []
|
||
goal_parts = [brief, str(actor_view.get("currentFrameRef") or "")]
|
||
for row in obligations if isinstance(obligations, list) else []:
|
||
if not isinstance(row, dict):
|
||
continue
|
||
goal_parts.extend([str(row.get("title") or "")])
|
||
goal_parts.extend(str(item) for item in row.get("seenStepTitles", []))
|
||
goal_parts.extend(str(item) for item in row.get("missingStepTitles", []))
|
||
visible_goal = "\n".join(goal_parts).casefold()
|
||
if re.search(r'[\(（\[]\s*\d+\s*[,，]\s*\d+\s*[\)）\]]', visible_goal):
|
||
return "brief 泄漏了坐标答案"
|
||
if isinstance(expected_action, dict) and expected_action.get("type") == "key":
|
||
expected_key = expected_action.get("key")
|
||
aliases = {
|
||
"ArrowLeft": ("arrowleft", "向左", "左移", "左侧", "最左"),
|
||
"ArrowRight": ("arrowright", "向右", "右移", "右侧", "最右"),
|
||
"ArrowUp": ("arrowup", "向上", "上移", "顶部"),
|
||
"ArrowDown": ("arrowdown", "向下", "下移", "底部"),
|
||
}.get(expected_key, (str(expected_key or "").casefold(),))
|
||
if any(alias and alias.casefold() in visible_goal for alias in aliases):
|
||
return "ActorView 目标文本泄漏了规范 key 方向"
|
||
if isinstance(expected_action, dict) and expected_action.get("type") == "wait" \
|
||
and any(term in visible_goal for term in ("sequence_playback", "等待", "wait", "播放中")):
|
||
return "ActorView 目标文本泄漏了 wait 动作"
|
||
return None
|
||
|
||
|
||
def _actor_action_matches(actual, expected, tolerance):
|
||
"""ActorSelection 是有限 ID/cell 协议，行为金标必须精确命中，不再存在坐标容差。"""
|
||
del tolerance
|
||
return isinstance(actual, dict) and isinstance(expected, dict) and actual == expected
|
||
|
||
|
||
_TEXT_EXPECTATION_KEYS = {"requiredConcepts", "anyTerms", "evidenceRefs"}
|
||
|
||
|
||
def _validate_text_expectation(value, label):
|
||
"""单条说明使用显式同义概念组和证据引用，不依赖输出语言或通用 NLP 猜测。"""
|
||
if not isinstance(value, dict) or set(value) != _TEXT_EXPECTATION_KEYS:
|
||
return f"{label} 必须精确包含 requiredConcepts/anyTerms/evidenceRefs"
|
||
concepts = value.get("requiredConcepts")
|
||
if not isinstance(concepts, list) or not all(
|
||
isinstance(group, list) and group
|
||
and all(isinstance(item, str) and item.strip() for item in group)
|
||
for group in concepts):
|
||
return f"{label}.requiredConcepts 必须是非空同义词组数组"
|
||
for field in ("anyTerms", "evidenceRefs"):
|
||
items = value.get(field)
|
||
if not isinstance(items, list) or not all(
|
||
isinstance(item, str) and item.strip() for item in items
|
||
):
|
||
return f"{label}.{field} 必须是字符串数组"
|
||
return None
|
||
|
||
|
||
def _validate_list_expectation(value, label):
|
||
if not isinstance(value, list):
|
||
return f"{label} 必须是逐项文本期望数组"
|
||
for index, item in enumerate(value):
|
||
problem = _validate_text_expectation(item, f"{label}[{index}]")
|
||
if problem:
|
||
return problem
|
||
return None
|
||
|
||
|
||
def _evidence_reference_present(reference, text):
|
||
"""证据引用允许等价的 event 序号写法；frame/action 等身份仍要求逐字匹配。"""
|
||
folded_reference = str(reference or "").casefold()
|
||
folded_text = str(text or "").casefold()
|
||
if folded_reference in folded_text:
|
||
return True
|
||
event_match = re.fullmatch(r"event:(\d+)", folded_reference)
|
||
if not event_match:
|
||
return False
|
||
event_number = re.escape(event_match.group(1))
|
||
# 模型常把同一 event:N 写成 event seq=N / 事件 seq:N；只归一编号格式，不接受别的编号。
|
||
if re.search(
|
||
rf"(?:event|事件)\s*(?:seq(?:uence)?\s*)?[:=#]?\s*{event_number}\b",
|
||
folded_text,
|
||
):
|
||
return True
|
||
# 兼容 `event game/foo (seq 1)`，但要求 event 与 seq 位于同一句局部窗口内。
|
||
if re.search(
|
||
rf"(?:event|事件)[^。；\n]{{0,120}}\bseq(?:uence)?\s*[:=#]?\s*{event_number}\b",
|
||
folded_text,
|
||
):
|
||
return True
|
||
return re.search(
|
||
rf"\bseq(?:uence)?\s*[:=#]?\s*{event_number}\b[^。；\n]{{0,120}}(?:event|事件)",
|
||
folded_text,
|
||
) is not None
|
||
|
||
|
||
def _text_expectation_matches(actual, expected, evidence_context=None):
|
||
"""自由文本校验同义概念、结论词与证据引用；裁决轴仍由结构化字段精确校验。"""
|
||
if not isinstance(actual, str) or not actual.strip() or not isinstance(expected, dict):
|
||
return False
|
||
refs = expected.get("evidenceRefs")
|
||
concepts = expected.get("requiredConcepts")
|
||
any_terms = expected.get("anyTerms")
|
||
semantic_text = str(evidence_context if evidence_context is not None else actual).casefold()
|
||
compact_text = re.sub(r"\s+", "", semantic_text)
|
||
|
||
def term_present(term):
|
||
folded = str(term).casefold()
|
||
return folded in semantic_text or re.sub(r"\s+", "", folded) in compact_text
|
||
|
||
concepts_ok = isinstance(concepts, list) and all(
|
||
isinstance(group, list) and group
|
||
and any(term_present(term) for term in group)
|
||
for group in concepts
|
||
)
|
||
any_terms_ok = isinstance(any_terms, list) and (
|
||
not any_terms or any(term_present(term) for term in any_terms)
|
||
)
|
||
refs_ok = isinstance(refs, list) and all(
|
||
_evidence_reference_present(ref, semantic_text) for ref in refs
|
||
)
|
||
return concepts_ok and any_terms_ok and refs_ok
|
||
|
||
|
||
def _list_expectation_matches(actual, expected, evidence_context=None):
|
||
"""数组只校验必需存在性和整份解释的证据闭合，允许拆句、合句和语言切换。"""
|
||
if not isinstance(actual, list) or not all(isinstance(item, str) for item in actual) \
|
||
or not isinstance(expected, list):
|
||
return False
|
||
if not expected:
|
||
return actual == []
|
||
joined_evidence = evidence_context if evidence_context is not None else " ".join(actual)
|
||
return bool(actual) and all(
|
||
_text_expectation_matches(actual[0], rule, joined_evidence) for rule in expected
|
||
)
|
||
|
||
|
||
def _obligation_projection(rows):
|
||
"""提取 Judge 真正影响发布语义的 obligation 字段，并按 id 稳定排序。"""
|
||
if not isinstance(rows, list):
|
||
return None
|
||
projected = []
|
||
for row in rows:
|
||
if not isinstance(row, dict):
|
||
return None
|
||
projected.append({
|
||
"id": row.get("id"),
|
||
"status": row.get("status"),
|
||
"sequenceRefs": row.get("sequenceRefs"),
|
||
"evidenceRefs": row.get("evidenceRefs"),
|
||
})
|
||
return sorted(projected, key=lambda row: str(row.get("id")))
|
||
|
||
|
||
def _validate_expected_obligations(eval_key, package, expected_rows):
|
||
"""金标 obligation 必须覆盖全部 required 候选，并保持 refs 的生产边界。"""
|
||
errors = []
|
||
candidates = package.get("proofObligations") if isinstance(package, dict) else None
|
||
if not isinstance(candidates, list) or not isinstance(expected_rows, list):
|
||
return [f"{eval_key}: proofObligations/expectedObligations 必须是数组"]
|
||
candidate_by_id = {
|
||
row.get("id"): row for row in candidates
|
||
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
||
}
|
||
required_ids = sorted(
|
||
row["id"] for row in candidates
|
||
if isinstance(row, dict) and row.get("required") is True and isinstance(row.get("id"), str)
|
||
)
|
||
expected_ids = []
|
||
for index, row in enumerate(expected_rows):
|
||
prefix = f"{eval_key}: expectedObligations[{index}]"
|
||
if not isinstance(row, dict) or set(row) != {"id", "status", "sequenceRefs", "evidenceRefs"}:
|
||
errors.append(f"{prefix} 字段必须精确匹配生产 obligation 输出")
|
||
continue
|
||
obligation_id = row.get("id")
|
||
expected_ids.append(obligation_id)
|
||
candidate = candidate_by_id.get(obligation_id)
|
||
if candidate is None:
|
||
errors.append(f"{prefix} 引用了未知义务:{obligation_id}")
|
||
continue
|
||
if row.get("status") not in _JUDGE_OBLIGATION_STATUSES:
|
||
errors.append(f"{prefix} status 非法:{row.get('status')}")
|
||
if row.get("sequenceRefs") != candidate.get("sequenceRefs") \
|
||
or row.get("evidenceRefs") != candidate.get("evidenceRefs"):
|
||
errors.append(f"{prefix} 改写了候选 sequenceRefs/evidenceRefs 边界")
|
||
if row.get("status") == "missing" \
|
||
and (row.get("sequenceRefs") != [] or row.get("evidenceRefs") != []):
|
||
errors.append(f"{prefix} missing 必须使用空引用")
|
||
if sorted(expected_ids) != required_ids:
|
||
errors.append(
|
||
f"{eval_key}: expectedObligations 必须完整覆盖 required ids，"
|
||
f"expected={sorted(expected_ids)} required={required_ids}"
|
||
)
|
||
return errors
|
||
|
||
|
||
def validate_golden_samples(prompt_id, samples):
|
||
"""校验角色金标、语义标签和 canonical 输出；坏数据在模型调用前 fail-closed。"""
|
||
errors = []
|
||
seen_image_hashes = {}
|
||
for sample in samples:
|
||
eval_key = sample["evalKey"]
|
||
input_row = sample["input"]
|
||
label = sample["label"]
|
||
if prompt_id == "playtest.actor":
|
||
allowed_inputs = {
|
||
"evalKey", "actorView", "expectedTargetSetHash", "imagePath", "imageDataUrl",
|
||
}
|
||
if set(input_row) - allowed_inputs \
|
||
or not {"evalKey", "actorView", "expectedTargetSetHash"}.issubset(input_row) \
|
||
or (("imagePath" in input_row) == ("imageDataUrl" in input_row)):
|
||
errors.append(
|
||
f"{eval_key}: Actor input 必须含 ActorView/3 基础投影与 expectedTargetSetHash，"
|
||
"且当前帧只能选择 imagePath/imageDataUrl 之一"
|
||
)
|
||
try:
|
||
actor_view, _context = materialize_eval_actor_input(input_row)
|
||
except ValueError as exc:
|
||
errors.append(f"{eval_key}: ActorView/TargetSet 不可物化:{exc}")
|
||
actor_view = None
|
||
if set(label) != {"evalKey", "expectedSelections", "expectedError"}:
|
||
errors.append(
|
||
f"{eval_key}: Actor label 必须含 expectedSelections/expectedError"
|
||
)
|
||
continue
|
||
expected_actions = label.get("expectedSelections")
|
||
expected_error = label.get("expectedError")
|
||
has_actions = isinstance(expected_actions, list) and bool(expected_actions)
|
||
if has_actions == (expected_error is not None):
|
||
errors.append(f"{eval_key}: expectedSelections 与 expectedError 必须且只能启用一种")
|
||
continue
|
||
if has_actions:
|
||
for action_index, expected_action in enumerate(expected_actions):
|
||
parsed = invoke_production_parser(
|
||
"actor-selection", stable_json(expected_action),
|
||
evidence_context=actor_view,
|
||
)
|
||
if not parsed.get("ok") or parsed.get("normalized") != expected_action:
|
||
errors.append(
|
||
f"{eval_key}: expectedSelections[{action_index}] 不是生产 parser 的规范 selection:{parsed}"
|
||
)
|
||
leak = _actor_brief_leaks_answer(actor_view, expected_action)
|
||
if leak:
|
||
errors.append(f"{eval_key}: {leak}")
|
||
objective_progress = actor_view.get("objectiveProgress") \
|
||
if isinstance(actor_view, dict) else None
|
||
if not isinstance(objective_progress, dict) or not objective_progress:
|
||
errors.append(f"{eval_key}: Actor 必须给出非空 objectiveProgress，由画面与目标共同选动作")
|
||
frame_hash = actor_view.get("currentFrameHash") if isinstance(actor_view, dict) else None
|
||
previous = seen_image_hashes.get(frame_hash)
|
||
if previous:
|
||
errors.append(f"{eval_key}: 与 {previous} 复用了同一截图 hash")
|
||
elif isinstance(frame_hash, str):
|
||
seen_image_hashes[frame_hash] = eval_key
|
||
elif not isinstance(expected_error, str) or not expected_error:
|
||
errors.append(f"{eval_key}: expectedError 必须是非空 parser 错误码")
|
||
elif prompt_id in PLAYTEST_JUDGE_IDS:
|
||
if set(input_row) != {"evalKey", "judgePackage", "images", "preImages"}:
|
||
errors.append(
|
||
f"{eval_key}: Judge input 只允许 evalKey/judgePackage/images/preImages"
|
||
)
|
||
errors.extend(
|
||
f"{eval_key}: {problem}" for problem in _validate_judge_package(
|
||
input_row.get("judgePackage")
|
||
)
|
||
)
|
||
expected_label_keys = {
|
||
"evalKey", "expectedDecision", "expectedFailureClass", "expectedObligations",
|
||
"expectedProblems", "expectedContradictions", "expectedSummary", "canonicalOutput",
|
||
}
|
||
if set(label) != expected_label_keys:
|
||
errors.append(f"{eval_key}: Judge label 字段必须精确为 {sorted(expected_label_keys)}")
|
||
continue
|
||
if label.get("expectedDecision") not in _JUDGE_DECISIONS:
|
||
errors.append(f"{eval_key}: expectedDecision 非法")
|
||
if label.get("expectedFailureClass") not in _JUDGE_FAILURE_CLASSES:
|
||
errors.append(f"{eval_key}: expectedFailureClass 非法")
|
||
for field in ("expectedProblems", "expectedContradictions"):
|
||
problem = _validate_list_expectation(label.get(field), f"{eval_key}: {field}")
|
||
if problem:
|
||
errors.append(problem)
|
||
summary_problem = _validate_text_expectation(
|
||
label.get("expectedSummary"), f"{eval_key}: expectedSummary"
|
||
)
|
||
if summary_problem:
|
||
errors.append(summary_problem)
|
||
package = input_row.get("judgePackage")
|
||
errors.extend(_validate_expected_obligations(
|
||
eval_key, package, label.get("expectedObligations")
|
||
))
|
||
images = input_row.get("images")
|
||
pre_images = input_row.get("preImages")
|
||
frames = package.get("frames", []) if isinstance(package, dict) else []
|
||
frame_by_ref = {
|
||
row.get("ref"): row for row in frames
|
||
if isinstance(row, dict) and isinstance(row.get("ref"), str)
|
||
}
|
||
actions = package.get("actions", []) if isinstance(package, dict) else []
|
||
expected_pre_identities = set()
|
||
expected_post_identities = set()
|
||
for action_index, action in enumerate(actions):
|
||
if not isinstance(action, dict):
|
||
continue
|
||
pre_ref = action.get("preFrameRef")
|
||
post_ref = action.get("postFrameRef")
|
||
if isinstance(pre_ref, dict):
|
||
expected_pre_identities.add((pre_ref.get("path"), action.get("preFrameHash")))
|
||
if isinstance(post_ref, dict):
|
||
expected_post_identities.add((post_ref.get("path"), action.get("postFrameHash")))
|
||
if action.get("preFrameHash") == action.get("postFrameHash"):
|
||
errors.append(
|
||
f"{eval_key}: actions[{action_index}] 的 pre/postFrameHash 不得相同"
|
||
)
|
||
if not isinstance(images, list):
|
||
errors.append(f"{eval_key}: images 必须是数组")
|
||
images = []
|
||
if not isinstance(pre_images, list):
|
||
errors.append(f"{eval_key}: preImages 必须是数组")
|
||
pre_images = []
|
||
requires_visual = label.get("expectedDecision") in {"accept", "reject"} \
|
||
or bool(label.get("expectedContradictions"))
|
||
if requires_visual and not images:
|
||
errors.append(f"{eval_key}: 该行为标签必须携带真实截图")
|
||
seen_post_identities = set()
|
||
for index, image_row in enumerate(images):
|
||
if not isinstance(image_row, dict):
|
||
errors.append(f"{eval_key}: images[{index}] 必须是对象")
|
||
continue
|
||
frame_ref = image_row.get("frameRef")
|
||
frame_hash = image_row.get("hash")
|
||
frame = frame_by_ref.get(frame_ref)
|
||
if not isinstance(frame, dict) or frame.get("hash") != frame_hash:
|
||
errors.append(f"{eval_key}: images[{index}] 未与 JudgePackage.frames 的 ref/hash 绑定")
|
||
elif frame.get("kind") != "post" or (frame_ref, frame_hash) not in expected_post_identities:
|
||
errors.append(f"{eval_key}: images[{index}] 未绑定动作的 post 帧")
|
||
identity = (frame_ref, frame_hash)
|
||
if identity in seen_post_identities:
|
||
errors.append(f"{eval_key}: images[{index}] 重复绑定同一 post 帧")
|
||
seen_post_identities.add(identity)
|
||
previous = seen_image_hashes.get(frame_hash)
|
||
if previous:
|
||
errors.append(f"{eval_key}: 与 {previous} 复用了同一截图 hash")
|
||
elif isinstance(frame_hash, str):
|
||
seen_image_hashes[frame_hash] = eval_key
|
||
seen_pre_identities = set()
|
||
for index, image_row in enumerate(pre_images):
|
||
prefix = f"{eval_key}: preImages[{index}]"
|
||
if not isinstance(image_row, dict) or set(image_row) not in (
|
||
{"frameRef", "hash", "imagePath"},
|
||
{"frameRef", "hash", "imageDataUrl"},
|
||
):
|
||
errors.append(f"{prefix} 必须含 frameRef/hash 且只选一种图片来源")
|
||
continue
|
||
frame_ref = image_row.get("frameRef")
|
||
frame_hash = image_row.get("hash")
|
||
frame = frame_by_ref.get(frame_ref)
|
||
identity = (frame_ref, frame_hash)
|
||
if not isinstance(frame, dict) or frame.get("hash") != frame_hash:
|
||
errors.append(f"{prefix} 未与 JudgePackage.frames 的 ref/hash 绑定")
|
||
elif frame.get("kind") != "pre" or identity not in expected_pre_identities:
|
||
errors.append(f"{prefix} 未绑定动作的 pre 帧")
|
||
if identity in seen_pre_identities:
|
||
errors.append(f"{prefix} 重复绑定同一 pre 帧")
|
||
seen_pre_identities.add(identity)
|
||
try:
|
||
_, image_bytes = _resolve_eval_image(image_row, prefix)
|
||
except ValueError as exc:
|
||
errors.append(f"{prefix} 图片不可用:{exc}")
|
||
continue
|
||
if hashlib.sha256(image_bytes).hexdigest() != frame_hash:
|
||
errors.append(f"{prefix} 的 hash 与图片原始字节不一致")
|
||
if seen_pre_identities != expected_pre_identities:
|
||
errors.append(
|
||
f"{eval_key}: preImages 必须完整且仅覆盖动作 pre 帧，"
|
||
f"expected={sorted(expected_pre_identities, key=lambda item: (str(item[0]), str(item[1])))} "
|
||
f"actual={sorted(seen_pre_identities, key=lambda item: (str(item[0]), str(item[1])))}"
|
||
)
|
||
if any(frame_hash in {item[1] for item in expected_post_identities}
|
||
for _, frame_hash in expected_pre_identities):
|
||
errors.append(f"{eval_key}: preImages 不得复用同动作集合的 post 图片 hash")
|
||
canonical_output = label.get("canonicalOutput")
|
||
if not isinstance(canonical_output, dict):
|
||
errors.append(f"{eval_key}: canonicalOutput 必须是对象")
|
||
continue
|
||
candidates = package.get("proofObligations", []) if isinstance(package, dict) else []
|
||
parsed = invoke_production_parser(
|
||
"judge", stable_json(canonical_output), candidates, package
|
||
)
|
||
if not parsed.get("ok"):
|
||
errors.append(f"{eval_key}: canonicalOutput 未通过生产 parser:{parsed}")
|
||
elif not judge_correct(prompt_id, parsed, label):
|
||
errors.append(f"{eval_key}: canonicalOutput 与行为标签不一致:{parsed.get('normalized')}")
|
||
return errors
|
||
|
||
|
||
def judge_correct(prompt_id, parsed, label):
|
||
"""闸2 单条判对：产物是否与金标 label 一致。按 prompt 语义适配。
|
||
|
||
Actor 必须命中 expectedSelections 中任一生产 normalized selection，或精确命中 expectedError；
|
||
Judge 除四态外还会复核 canonical obligation 与候选引用边界。未适配 id 直接不通过。
|
||
"""
|
||
obj = parsed.get("normalized") if isinstance(parsed, dict) else None
|
||
if prompt_id == "safety.prompt-check":
|
||
return bool(parsed.get("ok")) and isinstance(obj, dict) \
|
||
and obj.get("safe") == label.get("expectedSafe")
|
||
if prompt_id == "playtest.actor":
|
||
expected_error = label.get("expectedError")
|
||
if expected_error is not None:
|
||
return parsed.get("ok") is False and parsed.get("code") == expected_error
|
||
return parsed.get("ok") is True and any(
|
||
_actor_action_matches(obj, expected, 0)
|
||
for expected in label.get("expectedSelections", [])
|
||
)
|
||
if prompt_id in PLAYTEST_JUDGE_IDS:
|
||
text_evidence = " ".join(
|
||
[*(obj.get("problems") or []), *(obj.get("contradictions") or []), str(obj.get("summary") or "")]
|
||
) if isinstance(obj, dict) else ""
|
||
traceability_ok = isinstance(obj, dict) \
|
||
and _list_expectation_matches(
|
||
obj.get("problems"), label.get("expectedProblems"), text_evidence
|
||
) \
|
||
and _list_expectation_matches(
|
||
obj.get("contradictions"), label.get("expectedContradictions"), text_evidence
|
||
) \
|
||
and _text_expectation_matches(obj.get("summary"), label.get("expectedSummary"))
|
||
return parsed.get("ok") is True and isinstance(obj, dict) \
|
||
and obj.get("decision") == label.get("expectedDecision") \
|
||
and obj.get("failureClass") == label.get("expectedFailureClass") \
|
||
and _obligation_projection(obj.get("obligations")) \
|
||
== _obligation_projection(label.get("expectedObligations")) \
|
||
and traceability_ok
|
||
return None # 无判定口径（上层视为未过 + 报告标注需补判定适配）
|
||
|
||
|
||
def key_verdict(prompt_id, parsed, label=None):
|
||
"""闸3 稳定语义签名；坐标容差内变化和自由文本改写不得制造回归假阴。"""
|
||
obj = parsed.get("normalized") if isinstance(parsed, dict) else None
|
||
if prompt_id == "safety.prompt-check":
|
||
return {"safe": obj.get("safe")} if isinstance(obj, dict) else {"safe": None}
|
||
if prompt_id == "playtest.actor":
|
||
return {
|
||
"schemaOk": parsed.get("ok") is True,
|
||
"correct": bool(judge_correct(prompt_id, parsed, label or {})),
|
||
"actionType": obj.get("type") if isinstance(obj, dict) else None,
|
||
"error": None if parsed.get("ok") else parsed.get("code"),
|
||
}
|
||
if prompt_id in PLAYTEST_JUDGE_IDS:
|
||
if not isinstance(obj, dict):
|
||
return {
|
||
"decision": None, "failureClass": None, "obligations": None,
|
||
"textTraceability": {"problems": False, "contradictions": False, "summary": False},
|
||
}
|
||
text_evidence = " ".join(
|
||
[*(obj.get("problems") or []), *(obj.get("contradictions") or []), str(obj.get("summary") or "")]
|
||
)
|
||
return {
|
||
"decision": obj.get("decision"),
|
||
"failureClass": obj.get("failureClass"),
|
||
"obligations": _obligation_projection(obj.get("obligations")),
|
||
"textTraceability": {
|
||
"problems": _list_expectation_matches(
|
||
obj.get("problems"), (label or {}).get("expectedProblems"), text_evidence
|
||
),
|
||
"contradictions": _list_expectation_matches(
|
||
obj.get("contradictions"), (label or {}).get("expectedContradictions"), text_evidence
|
||
),
|
||
"summary": _text_expectation_matches(
|
||
obj.get("summary"), (label or {}).get("expectedSummary")
|
||
),
|
||
},
|
||
}
|
||
return {}
|
||
|
||
|
||
# ══════════════════════ 基线 / 台账 ══════════════════════
|
||
|
||
def baseline_path(eval_id):
|
||
return EVAL_DIR / eval_id / "baseline.json"
|
||
|
||
|
||
def load_baseline(eval_id):
|
||
p = baseline_path(eval_id)
|
||
if not p.exists():
|
||
return None
|
||
try:
|
||
return json.loads(p.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError):
|
||
return None
|
||
|
||
|
||
def baseline_identity_matches(baseline, snapshot, model, audit_identity=None):
|
||
"""基线只可比较同 Prompt、夹具、模型、seed、审计协议和执行代码的复跑。"""
|
||
if not isinstance(baseline, dict) or not isinstance(snapshot, dict):
|
||
return False
|
||
core_matches = baseline.get("promptBodySha256") == snapshot.get("promptBodySha256") \
|
||
and baseline.get("fixtureSnapshot") == snapshot.get("fixtures") \
|
||
and baseline.get("providerOutputSchemaSha256") \
|
||
== snapshot.get("providerOutputSchemaSha256") \
|
||
and baseline.get("model") == model \
|
||
and baseline.get("seedStrategy") == SEED_STRATEGY
|
||
if not core_matches:
|
||
return False
|
||
# 旧基线没有 Audit/2 身份时保持可读；真跑路径总会传 audit_identity，并据此要求重建。
|
||
if audit_identity is None:
|
||
return True
|
||
return baseline.get("auditSchemaVersion") == audit_identity.get("auditSchemaVersion") \
|
||
and baseline.get("codeIdentityHash") == audit_identity.get("codeIdentityHash")
|
||
|
||
|
||
def _json_bytes(value, *, pretty=True):
|
||
"""审计 JSON 统一使用 UTF-8；manifest 用稳定格式，便于逐字节 hash 和离线重放。"""
|
||
if pretty:
|
||
return (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n").encode("utf-8")
|
||
return stable_json(value).encode("utf-8")
|
||
|
||
|
||
def _safe_endpoint(base_url):
|
||
"""环境快照只保留 endpoint 的 scheme/host/port/path，永不保存 query、userinfo 或 fragment。"""
|
||
parsed = urllib.parse.urlsplit(str(base_url or ""))
|
||
host = parsed.hostname or ""
|
||
port = f":{parsed.port}" if parsed.port is not None else ""
|
||
return urllib.parse.urlunsplit((parsed.scheme, f"{host}{port}", parsed.path, "", ""))
|
||
|
||
|
||
def _audit_write(context, relative_path, data):
|
||
"""写一个审计 artifact，并把路径、字节数和 SHA-256 收进 run manifest。"""
|
||
raw = data if isinstance(data, bytes) else str(data).encode("utf-8")
|
||
target = context["stagingDir"] / relative_path
|
||
_write_bytes_atomic(target, raw)
|
||
ref = {
|
||
"path": Path(relative_path).as_posix(),
|
||
"size": len(raw),
|
||
"sha256": hashlib.sha256(raw).hexdigest(),
|
||
}
|
||
context["artifacts"].append(ref)
|
||
context["totalBytes"] += len(raw)
|
||
if context["totalBytes"] > MAX_AUDIT_RUN_BYTES:
|
||
raise ValueError(f"prompt eval 审计包超过 {MAX_AUDIT_RUN_BYTES}B 上限")
|
||
return ref
|
||
|
||
|
||
def _relevant_code_files(prompt_id, contract):
|
||
"""列出能改变请求物化、parser 或计分结果的最小代码集合；脏工作区也按当前字节封存。"""
|
||
files = [Path(__file__).resolve(), PRODUCTION_PARSER_BRIDGE.resolve(), contract["path"].resolve()]
|
||
shared_root = PROMPTS_DIR.parents[1] / "game-runtime/games/_wg1-gen/_shared"
|
||
if prompt_id == "playtest.actor" or prompt_id in PLAYTEST_JUDGE_IDS:
|
||
files.extend([
|
||
(shared_root / "playtest-v3.core.cjs").resolve(),
|
||
(shared_root / "playtest-v3-targets.cjs").resolve(),
|
||
])
|
||
if prompt_id == "playtest.actor":
|
||
files.extend([VISUAL_TARGET_RENDERER.resolve(), ACTOR_OUTPUT_SCHEMA_FILE.resolve()])
|
||
if prompt_id in PLAYTEST_JUDGE_IDS:
|
||
files.extend([
|
||
(PROMPTS_DIR.parents[1] / "game-runtime/games/_wg1-gen/_shared/playtest-v3-judge-consensus.cjs").resolve(),
|
||
JUDGE_OUTPUT_SCHEMA_FILE.resolve(),
|
||
])
|
||
return [path for path in files if path.is_file()]
|
||
|
||
|
||
def _dependency_version(*distribution_names):
|
||
"""读取已安装发行包版本；缺依赖只记 None，不执行网络安装。"""
|
||
for name in distribution_names:
|
||
try:
|
||
return importlib.metadata.version(name)
|
||
except importlib.metadata.PackageNotFoundError:
|
||
continue
|
||
return None
|
||
|
||
|
||
def _command_version(command):
|
||
"""读取本机工具版本，输出只做环境身份，不包含环境变量。"""
|
||
try:
|
||
proc = subprocess.run(command, capture_output=True, text=True, timeout=5)
|
||
return (proc.stdout or proc.stderr).strip().splitlines()[0] if proc.returncode == 0 else None
|
||
except (OSError, subprocess.SubprocessError, IndexError):
|
||
return None
|
||
|
||
|
||
def _renderer_runtime_identity():
|
||
"""读取实际 renderer 解释器的 Python/Pillow/OpenCV 版本，避免误记主进程环境。"""
|
||
executable = VISUAL_TARGET_PYTHON if VISUAL_TARGET_PYTHON.is_file() else Path(sys.executable)
|
||
script = (
|
||
"import json,platform; import cv2; import PIL; "
|
||
"print(json.dumps({'python':platform.python_version(),"
|
||
"'pillow':PIL.__version__,'opencv':cv2.__version__},sort_keys=True))"
|
||
)
|
||
try:
|
||
proc = subprocess.run(
|
||
[str(executable), "-c", script], capture_output=True, text=True, timeout=5,
|
||
)
|
||
value = json.loads(proc.stdout) if proc.returncode == 0 else None
|
||
if not isinstance(value, dict):
|
||
return {"executable": str(executable), "python": None, "pillow": None, "opencv": None}
|
||
return {
|
||
"executable": str(executable),
|
||
"python": value.get("python"),
|
||
"pillow": value.get("pillow"),
|
||
"opencv": value.get("opencv"),
|
||
}
|
||
except (OSError, subprocess.SubprocessError, json.JSONDecodeError):
|
||
return {"executable": str(executable), "python": None, "pillow": None, "opencv": None}
|
||
|
||
|
||
def begin_prompt_eval_audit(ledger_dir, eval_id, prompt_id, entry, contract, samples,
|
||
snapshot, model, base_url, provider_output_schema=None):
|
||
"""在隐藏 staging 目录物化共享输入；只有 finalize 成功才会出现最终 run 目录。"""
|
||
ledger_root = Path(ledger_dir) if ledger_dir else (EVAL_DIR / eval_id / "runs")
|
||
ledger_root.mkdir(parents=True, exist_ok=True)
|
||
run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") \
|
||
+ f"-p{os.getpid()}-n{time.time_ns()}"
|
||
raw_root = ledger_root / "raw"
|
||
raw_root.mkdir(parents=True, exist_ok=True)
|
||
staging = raw_root / f".{run_id}.tmp"
|
||
final = raw_root / run_id
|
||
staging.mkdir(mode=0o700)
|
||
context = {
|
||
"ledgerRoot": ledger_root, "runId": run_id,
|
||
"stagingDir": staging, "finalDir": final,
|
||
"artifacts": [], "samples": [], "totalBytes": 0,
|
||
"auditErrors": [], "requestReplayable": True,
|
||
}
|
||
try:
|
||
prompt_text = contract["path"].read_bytes()
|
||
_audit_write(context, "prompt/contract.md", prompt_text)
|
||
_audit_write(context, "prompt/body.txt", contract["body"].encode("utf-8"))
|
||
_audit_write(context, "prompt/registry-entry.json", _json_bytes(entry))
|
||
if provider_output_schema is not None:
|
||
_audit_write(
|
||
context, "prompt/provider-output-schema.json",
|
||
_json_bytes(provider_output_schema),
|
||
)
|
||
|
||
golden_id = "playtest.judge" if eval_id in PLAYTEST_JUDGE_IDS else eval_id
|
||
golden_dir = EVAL_DIR / golden_id
|
||
_audit_write(context, "fixtures/inputs.jsonl", (golden_dir / "inputs.jsonl").read_bytes())
|
||
_audit_write(context, "fixtures/labels.jsonl", (golden_dir / "labels.jsonl").read_bytes())
|
||
_audit_write(context, "fixtures/fixture-manifest.json", _json_bytes(snapshot["fixtures"]))
|
||
|
||
repo_root = PROMPTS_DIR.parents[1].resolve()
|
||
code_rows = []
|
||
for index, source in enumerate(_relevant_code_files(prompt_id, contract), start=1):
|
||
raw = source.read_bytes()
|
||
try:
|
||
source_name = source.relative_to(repo_root).as_posix()
|
||
except ValueError:
|
||
source_name = source.name
|
||
suffix = source.suffix if source.suffix else ".bin"
|
||
artifact = _audit_write(context, f"code/files/{index:02d}{suffix}", raw)
|
||
code_rows.append({"source": source_name, **artifact})
|
||
code_identity = {
|
||
"gitHead": _git_output(["rev-parse", "HEAD"], repo_root),
|
||
"gitDirty": bool(_git_output(["status", "--porcelain", "--untracked-files=no"], repo_root)),
|
||
"files": code_rows,
|
||
}
|
||
code_identity["codeIdentityHash"] = hashlib.sha256(
|
||
b"PromptEvalCode/2\0" + _json_bytes(code_identity, pretty=False)
|
||
).hexdigest()
|
||
_audit_write(context, "code/code-manifest.json", _json_bytes(code_identity))
|
||
|
||
environment = {
|
||
"python": platform.python_version(),
|
||
"implementation": platform.python_implementation(),
|
||
"os": platform.platform(), "machine": platform.machine(),
|
||
"hostname": socket.gethostname(),
|
||
"node": _command_version(["node", "--version"]),
|
||
"pillow": _dependency_version("Pillow"),
|
||
"opencv": _dependency_version("opencv-python-headless", "opencv-python"),
|
||
"endpoint": _safe_endpoint(base_url), "model": model,
|
||
"providerProtocol": PROVIDER_PROTOCOL,
|
||
"providerOutputSchemaSha256": snapshot.get("providerOutputSchemaSha256"),
|
||
"thinking": {
|
||
"type": "enabled",
|
||
"actorBudgetTokens": ANTHROPIC_ACTOR_THINKING_BUDGET,
|
||
"judgeBudgetTokens": ANTHROPIC_JUDGE_THINKING_BUDGET,
|
||
},
|
||
"maxTokens": {
|
||
"actor": ANTHROPIC_ACTOR_MAX_TOKENS,
|
||
"judge": ANTHROPIC_JUDGE_MAX_TOKENS,
|
||
},
|
||
"timezone": str(datetime.now().astimezone().tzinfo),
|
||
"credentialPresent": True,
|
||
}
|
||
if prompt_id == "playtest.actor":
|
||
environment["rendererRuntime"] = _renderer_runtime_identity()
|
||
environment["environmentHash"] = hashlib.sha256(
|
||
b"PromptEvalEnvironment/2\0" + _json_bytes(environment, pretty=False)
|
||
).hexdigest()
|
||
_audit_write(context, "environment.json", _json_bytes(environment))
|
||
context["codeIdentityHash"] = code_identity["codeIdentityHash"]
|
||
context["environmentHash"] = environment["environmentHash"]
|
||
return context
|
||
except Exception:
|
||
shutil.rmtree(staging, ignore_errors=True)
|
||
raise
|
||
|
||
|
||
def _git_output(arguments, cwd):
|
||
"""读取非敏感 Git 身份；失败返回 None，不把命令错误或环境变量落入审计包。"""
|
||
try:
|
||
proc = subprocess.run(["git", *arguments], cwd=str(cwd), capture_output=True, text=True, timeout=5)
|
||
return proc.stdout.strip() if proc.returncode == 0 else None
|
||
except (OSError, subprocess.SubprocessError):
|
||
return None
|
||
|
||
|
||
def normalize_model_call_result(value, model, messages, max_tokens, seed, output_schema=None):
|
||
"""兼容旧测试五元组；真实路径已经返回逐 attempt 结构。"""
|
||
if isinstance(value, dict) and "attempts" in value:
|
||
return value
|
||
if not isinstance(value, tuple) or len(value) != 5:
|
||
raise ValueError("call_model 返回值既不是 PromptEvalAudit/2 结构也不是兼容五元组")
|
||
content, prompt_tokens, completion_tokens, latency_ms, error = value
|
||
request_payload = build_provider_request_payload(
|
||
model, messages, max_tokens=max_tokens, seed=seed, output_schema=output_schema,
|
||
)
|
||
response_body = {
|
||
"id": "synthetic-test-response", "model": model,
|
||
"choices": [{"message": {"content": content}, "finish_reason": "stop"}],
|
||
"usage": {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens},
|
||
}
|
||
response_bytes = _json_bytes(response_body, pretty=False) if error is None else None
|
||
return {
|
||
"content": content, "promptTokens": prompt_tokens, "completionTokens": completion_tokens,
|
||
"latencyMs": latency_ms, "error": _sanitize_audit_error(error),
|
||
"requestPayload": request_payload, "requestReplayable": True,
|
||
"attempts": [{
|
||
"attempt": 1, "httpStatus": 200 if error is None else None,
|
||
"startedAt": _utc_now(), "finishedAt": _utc_now(),
|
||
"latencyMs": latency_ms, "headers": {}, "responseBytes": response_bytes,
|
||
"responseComplete": True, "provider": {"model": model, "synthetic": True},
|
||
"error": _sanitize_audit_error(error),
|
||
}],
|
||
}
|
||
|
||
|
||
def write_prompt_eval_sample_audit(context, sample_index, sample, messages, material,
|
||
call_result, parsed, gold_result, base_url):
|
||
"""物化一条样本的请求、全部 attempt、parser 与金标结果，返回兼容 raw 引用。"""
|
||
sample_key_hash = hashlib.sha256(sample["evalKey"].encode("utf-8")).hexdigest()[:12]
|
||
prefix = f"samples/sample-{sample_index:04d}-{sample_key_hash}"
|
||
input_ref = _audit_write(context, f"{prefix}/input-line.json", _json_bytes(sample["input"]))
|
||
label_ref = _audit_write(context, f"{prefix}/label-line.json", _json_bytes(sample["label"]))
|
||
messages_ref = _audit_write(context, f"{prefix}/messages.json", _json_bytes(messages))
|
||
|
||
image_refs = []
|
||
total_image_bytes = 0
|
||
for image_index, image in enumerate(material.get("images") or [], start=1):
|
||
image_bytes = image["bytes"]
|
||
total_image_bytes += len(image_bytes)
|
||
if total_image_bytes > MAX_AUDIT_SAMPLE_IMAGE_BYTES:
|
||
raise ValueError("单样本实际发送图片超过审计上限")
|
||
artifact = _audit_write(context, f"{prefix}/images/image-{image_index:04d}.png", image_bytes)
|
||
image_refs.append({key: value for key, value in image.items() if key != "bytes"} | artifact)
|
||
_audit_write(context, f"{prefix}/images/image-manifest.json", _json_bytes(image_refs))
|
||
|
||
actor = material.get("actor")
|
||
if isinstance(actor, dict):
|
||
_audit_write(context, f"{prefix}/target-guide/clean-frame.png", actor["cleanFrame"])
|
||
_audit_write(context, f"{prefix}/target-guide/visual-target-set.json", _json_bytes(actor["targetSet"]))
|
||
_audit_write(context, f"{prefix}/target-guide/guide.png", actor["guide"])
|
||
_audit_write(context, f"{prefix}/target-guide/guide-manifest.json", _json_bytes(actor["manifest"]))
|
||
|
||
request_payload = call_result.get("requestPayload") or b""
|
||
request_body_sha256 = hashlib.sha256(request_payload).hexdigest()
|
||
request_replayable = bool(call_result.get("requestReplayable")) \
|
||
and len(request_payload) <= MAX_AUDIT_REQUEST_BYTES
|
||
context["requestReplayable"] = context["requestReplayable"] and request_replayable
|
||
attempt_refs = []
|
||
for attempt in call_result.get("attempts") or []:
|
||
attempt_no = int(attempt.get("attempt") or len(attempt_refs) + 1)
|
||
attempt_started_at = attempt.get("startedAt") or _utc_now()
|
||
attempt_finished_at = attempt.get("finishedAt") or attempt_started_at
|
||
attempt_prefix = f"{prefix}/attempts/attempt-{attempt_no:02d}"
|
||
request_ref = _audit_write(context, f"{attempt_prefix}/request-body.json", request_payload)
|
||
request_meta = {
|
||
"method": "POST", "endpoint": _safe_endpoint(base_url).rstrip("/") + "/v1/messages",
|
||
"headers": {"content-type": "application/json"},
|
||
"authorizationInjected": True, "requestReplayable": request_replayable,
|
||
"body": request_ref,
|
||
}
|
||
_audit_write(context, f"{attempt_prefix}/request-meta.json", _json_bytes(request_meta))
|
||
response_ref = None
|
||
response_bytes = attempt.get("responseBytes")
|
||
if isinstance(response_bytes, bytes):
|
||
response_ref = _audit_write(context, f"{attempt_prefix}/response-body.bin", response_bytes)
|
||
headers_ref = _audit_write(
|
||
context, f"{attempt_prefix}/response-headers.json",
|
||
_json_bytes(_audit_headers((attempt.get("headers") or {}))),
|
||
)
|
||
error_ref = None
|
||
if attempt.get("error"):
|
||
error_ref = _audit_write(
|
||
context, f"{attempt_prefix}/attempt-error.json",
|
||
_json_bytes({"message": _sanitize_audit_error(attempt.get("error"))}),
|
||
)
|
||
attempt_refs.append({
|
||
"attempt": attempt_no, "httpStatus": attempt.get("httpStatus"),
|
||
"startedAt": attempt_started_at, "finishedAt": attempt_finished_at,
|
||
"latencyMs": attempt.get("latencyMs"), "backoffMs": attempt.get("backoffMs"),
|
||
"responseComplete": attempt.get("responseComplete") is True,
|
||
"provider": attempt.get("provider") or {},
|
||
"request": request_ref, "response": response_ref,
|
||
"headers": headers_ref, "error": error_ref,
|
||
})
|
||
if attempt.get("responseComplete") is not True:
|
||
context["auditErrors"].append(f"{sample['evalKey']}:provider_response_incomplete")
|
||
|
||
parser_ref = _audit_write(context, f"{prefix}/parser-result.json", _json_bytes(parsed))
|
||
normalized_ref = _audit_write(
|
||
context, f"{prefix}/normalized-result.json", _json_bytes(parsed.get("normalized")),
|
||
)
|
||
gold_ref = _audit_write(context, f"{prefix}/gold-result.json", _json_bytes(gold_result))
|
||
|
||
# 兼容旧消费者：继续保留只含模型 message.content 的 response-NNNN.txt。
|
||
content_bytes = str(call_result.get("content") or "").encode("utf-8")
|
||
raw_ref = _audit_write(context, f"response-{sample_index:04d}.txt", content_bytes)
|
||
sample_manifest = {
|
||
"schemaVersion": AUDIT_SCHEMA_VERSION, "evalKey": sample["evalKey"],
|
||
"sampleIndex": sample_index, "input": input_ref, "label": label_ref,
|
||
"messages": messages_ref, "images": image_refs, "attempts": attempt_refs,
|
||
"parser": parser_ref, "normalized": normalized_ref, "gold": gold_ref,
|
||
"requestReplayable": request_replayable,
|
||
"requestBodySha256": request_body_sha256,
|
||
"providerDeterministic": "unknown",
|
||
}
|
||
sample_hash = hashlib.sha256(
|
||
b"PromptEvalSample/2\0" + _json_bytes(sample_manifest, pretty=False)
|
||
).hexdigest()
|
||
sample_manifest["sampleAuditHash"] = sample_hash
|
||
manifest_ref = _audit_write(context, f"{prefix}/sample-manifest.json", _json_bytes(sample_manifest))
|
||
context["samples"].append({
|
||
"evalKey": sample["evalKey"], "sampleAuditHash": sample_hash,
|
||
"requestBodySha256": request_body_sha256, "manifest": manifest_ref,
|
||
})
|
||
raw_ref["requestBodySha256"] = request_body_sha256
|
||
return raw_ref
|
||
|
||
|
||
def finalize_prompt_eval_audit(context, prompt_id, version, model, snapshot, expected_samples,
|
||
secret_values=()):
|
||
"""manifest 最后写并扫描凭据，随后原子封口；失败时不产生最终目录。"""
|
||
incomplete_reasons = list(context["auditErrors"])
|
||
if not context["requestReplayable"]:
|
||
incomplete_reasons.append("request_not_replayable")
|
||
audit_complete = len(context["samples"]) == expected_samples \
|
||
and not incomplete_reasons
|
||
manifest = {
|
||
"schemaVersion": AUDIT_SCHEMA_VERSION,
|
||
"runId": context["runId"], "promptId": prompt_id, "version": version, "model": model,
|
||
"promptBodySha256": snapshot["promptBodySha256"], "fixtureSnapshot": snapshot["fixtures"],
|
||
"seedStrategy": SEED_STRATEGY, "codeIdentityHash": context["codeIdentityHash"],
|
||
"environmentHash": context["environmentHash"], "samples": context["samples"],
|
||
"artifacts": context["artifacts"], "auditComplete": audit_complete,
|
||
"requestReplayable": audit_complete and context["requestReplayable"],
|
||
"providerDeterministic": "unknown", "incompleteReasons": incomplete_reasons,
|
||
}
|
||
manifest["runAuditHash"] = hashlib.sha256(
|
||
b"PromptEvalRun/2\0" + _json_bytes(manifest, pretty=False)
|
||
).hexdigest()
|
||
manifest_bytes = _json_bytes(manifest)
|
||
manifest_path = context["stagingDir"] / "run-manifest.json"
|
||
_write_bytes_atomic(manifest_path, manifest_bytes)
|
||
|
||
forbidden = [str(value).encode("utf-8") for value in secret_values if value]
|
||
for path in context["stagingDir"].rglob("*"):
|
||
if not path.is_file():
|
||
continue
|
||
raw = path.read_bytes()
|
||
if any(secret in raw for secret in forbidden):
|
||
shutil.rmtree(context["stagingDir"], ignore_errors=True)
|
||
raise ValueError("Prompt eval 审计包检测到凭据字节，已拒绝封口")
|
||
_fsync_directory(context["stagingDir"])
|
||
os.replace(context["stagingDir"], context["finalDir"])
|
||
_fsync_directory(context["finalDir"].parent)
|
||
relative = (context["finalDir"] / "run-manifest.json").relative_to(context["ledgerRoot"])
|
||
return {
|
||
"schemaVersion": AUDIT_SCHEMA_VERSION,
|
||
"path": relative.as_posix(), "sha256": hashlib.sha256(manifest_bytes).hexdigest(),
|
||
"runAuditHash": manifest["runAuditHash"], "auditComplete": audit_complete,
|
||
"requestReplayable": manifest["requestReplayable"],
|
||
"providerDeterministic": "unknown",
|
||
}
|
||
|
||
|
||
def _fsync_directory(path):
|
||
"""尽力刷新目录项；部分平台不支持目录 fsync 时不削弱原子 rename 语义。"""
|
||
descriptor = None
|
||
try:
|
||
descriptor = os.open(Path(path), os.O_RDONLY)
|
||
os.fsync(descriptor)
|
||
except OSError:
|
||
pass
|
||
finally:
|
||
if descriptor is not None:
|
||
os.close(descriptor)
|
||
|
||
|
||
def _write_bytes_atomic(path, data):
|
||
"""同目录临时文件写满并 fsync 后替换，避免 raw/baseline 留半文件。"""
|
||
target = Path(path)
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
temp = target.with_name(f".{target.name}.tmp-{os.getpid()}-{time.time_ns()}")
|
||
try:
|
||
fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||
with os.fdopen(fd, "wb") as fh:
|
||
fh.write(bytes(data))
|
||
fh.flush()
|
||
os.fsync(fh.fileno())
|
||
os.replace(temp, target)
|
||
finally:
|
||
try:
|
||
temp.unlink()
|
||
except FileNotFoundError:
|
||
pass
|
||
|
||
|
||
def create_raw_audit_dir(ledger_dir, eval_id):
|
||
"""为本轮创建固定格式目录；目录名只来自时钟、pid，不接收 evalKey 或模型文本。"""
|
||
ledger_root = Path(ledger_dir) if ledger_dir else (EVAL_DIR / eval_id / "runs")
|
||
ledger_root.mkdir(parents=True, exist_ok=True)
|
||
run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") \
|
||
+ f"-p{os.getpid()}-n{time.time_ns()}"
|
||
raw_dir = ledger_root / "raw" / run_id
|
||
raw_dir.mkdir(parents=True, exist_ok=False)
|
||
return ledger_root, run_id, raw_dir
|
||
|
||
|
||
def write_raw_audit(ledger_root, raw_dir, sample_index, content):
|
||
"""原子保存单条原始响应；超限只留有界前缀，并由调用方把本轮判为基础设施不完整。"""
|
||
raw_bytes = str(content).encode("utf-8")
|
||
oversized = len(raw_bytes) > MAX_RAW_AUDIT_BYTES
|
||
stored = raw_bytes
|
||
if oversized:
|
||
marker = (
|
||
f"\n\n[RAW_AUDIT_TRUNCATED originalBytes={len(raw_bytes)} "
|
||
f"limitBytes={MAX_RAW_AUDIT_BYTES}]\n"
|
||
).encode("ascii")
|
||
prefix_limit = max(0, MAX_RAW_AUDIT_BYTES - len(marker))
|
||
prefix = raw_bytes[:prefix_limit].decode("utf-8", errors="ignore").encode("utf-8")
|
||
stored = prefix + marker
|
||
target = Path(raw_dir) / f"response-{sample_index:04d}.txt"
|
||
_write_bytes_atomic(target, stored)
|
||
return {
|
||
"path": target.relative_to(Path(ledger_root)).as_posix(),
|
||
"sha256": hashlib.sha256(stored).hexdigest(),
|
||
}, oversized
|
||
|
||
|
||
def write_baseline_atomic(eval_id, baseline):
|
||
"""基线使用原子替换；旧基线要么完整保留，要么完整切到新快照。"""
|
||
payload = (json.dumps(baseline, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
|
||
_write_bytes_atomic(baseline_path(eval_id), payload)
|
||
|
||
|
||
def write_ledger(ledger_dir, eval_id, record):
|
||
"""台账 append-only：eval/<id>/runs/run-<date>.jsonl 追加一行运行记录。"""
|
||
d = Path(ledger_dir) if ledger_dir else (EVAL_DIR / eval_id / "runs")
|
||
d.mkdir(parents=True, exist_ok=True)
|
||
day = datetime.now(timezone.utc).strftime("%Y%m%d")
|
||
f = d / f"run-{day}.jsonl"
|
||
with f.open("a", encoding="utf-8") as fh:
|
||
fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
|
||
try:
|
||
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||
fh.flush()
|
||
os.fsync(fh.fileno())
|
||
finally:
|
||
fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
|
||
return f
|
||
|
||
|
||
def _read_stability_records(ledger_root):
|
||
"""按台账顺序读取历史；坏行不参与稳定性证明，避免用不完整记录凑轮次。"""
|
||
records = []
|
||
for ledger_file in sorted(Path(ledger_root).glob("run-*.jsonl")):
|
||
try:
|
||
lines = ledger_file.read_text(encoding="utf-8").splitlines()
|
||
except OSError:
|
||
continue
|
||
for line in lines:
|
||
try:
|
||
value = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
if isinstance(value, dict):
|
||
records.append(value)
|
||
return records
|
||
|
||
|
||
def evaluate_repeat_stability(
|
||
ledger_root, prompt_id, identity, request_hashes, signatures,
|
||
per_key_correct, per_key_schema_ok, qualified):
|
||
"""计算同身份、同请求下的金标稳定性；失败或身份漂移会从当前轮重新计数。"""
|
||
if not qualified:
|
||
return {
|
||
"qualified": False,
|
||
"consecutiveRuns": 0,
|
||
"previousRunIds": [],
|
||
"cohortReady": False,
|
||
}
|
||
expected_identity = stable_json(identity)
|
||
expected_requests = stable_json(request_hashes)
|
||
previous_run_ids = []
|
||
cohort_rows = [{
|
||
"correct": per_key_correct,
|
||
"schema": per_key_schema_ok,
|
||
"signatures": signatures,
|
||
}]
|
||
for record in reversed(_read_stability_records(ledger_root)):
|
||
stability = record.get("stability") if isinstance(record.get("stability"), dict) else None
|
||
if not stability or stability.get("qualified") is not True:
|
||
break
|
||
if stable_json(stability.get("identity")) != expected_identity \
|
||
or stable_json(stability.get("requestBodySha256")) != expected_requests:
|
||
break
|
||
previous_correct = stability.get("perKeyCorrect")
|
||
previous_schema = stability.get("perKeySchemaOk")
|
||
if prompt_id == "playtest.actor" or prompt_id in PLAYTEST_JUDGE_IDS:
|
||
if not isinstance(previous_correct, dict) or not isinstance(previous_schema, dict) \
|
||
or set(previous_correct) != set(per_key_correct) \
|
||
or set(previous_schema) != set(per_key_schema_ok):
|
||
break
|
||
elif stable_json(stability.get("signatures")) != stable_json(signatures):
|
||
# 非 playtest 条目沿用结构化结果完全一致的旧口径。
|
||
break
|
||
cohort_rows.append({
|
||
"correct": previous_correct,
|
||
"schema": previous_schema,
|
||
"signatures": stability.get("signatures"),
|
||
})
|
||
previous_run_ids.append(str(record.get("runId") or ""))
|
||
consecutive = len(cohort_rows)
|
||
cohort = cohort_rows[:MIN_STABLE_RUNS]
|
||
per_key_correct_counts = {
|
||
key: sum(1 for row in cohort if (row.get("correct") or {}).get(key) is True)
|
||
for key in per_key_correct
|
||
}
|
||
schema_total = sum(
|
||
1 for row in cohort for value in (row.get("schema") or {}).values() if value is True
|
||
)
|
||
correct_total = sum(per_key_correct_counts.values())
|
||
cohort_ready = False
|
||
if len(cohort) == MIN_STABLE_RUNS:
|
||
if prompt_id == "playtest.actor":
|
||
cohort_ready = bool(
|
||
schema_total == len(per_key_schema_ok) * MIN_STABLE_RUNS
|
||
and correct_total >= 12
|
||
and all(count >= 2 for count in per_key_correct_counts.values())
|
||
)
|
||
elif prompt_id in PLAYTEST_JUDGE_IDS:
|
||
expected_total = len(per_key_correct) * MIN_STABLE_RUNS
|
||
cohort_ready = schema_total == expected_total and correct_total == expected_total
|
||
else:
|
||
cohort_ready = True
|
||
return {
|
||
"qualified": True,
|
||
"consecutiveRuns": consecutive,
|
||
"previousRunIds": previous_run_ids,
|
||
"cohortReady": cohort_ready,
|
||
"cohortRuns": len(cohort),
|
||
"cohortCorrect": correct_total,
|
||
"cohortSchemaOk": schema_total,
|
||
"perKeyCorrectCounts": per_key_correct_counts,
|
||
}
|
||
|
||
|
||
# ══════════════════════ 单条目四闸主流程 ══════════════════════
|
||
|
||
def run_offline_validation_for_id(prompt_id, registry):
|
||
"""只验证 registry/version、金标、图片和生产 parser；绝不调用模型或写台账。"""
|
||
print(f"\n{'='*70}\n[离线消费门] 目标条目：{prompt_id}\n{'='*70}")
|
||
entry = registry.get(prompt_id)
|
||
if not entry:
|
||
print(f"[FAIL] {prompt_id} 不在 registry.yaml", file=sys.stderr)
|
||
return 1
|
||
contract, contract_errors = load_prompt_contract(prompt_id, entry)
|
||
if contract_errors:
|
||
print(f"[FAIL] {prompt_id} registry/version 契约非法：")
|
||
for error in contract_errors:
|
||
print(f" · {error}")
|
||
return 1
|
||
samples, golden_error = load_golden(prompt_id)
|
||
if golden_error:
|
||
print(f"[FAIL·fail-closed] {golden_error}")
|
||
return 1
|
||
sample_errors = validate_golden_samples(prompt_id, samples)
|
||
if sample_errors:
|
||
print(f"[FAIL·fail-closed] {prompt_id} 金标契约非法：")
|
||
for error in sample_errors:
|
||
print(f" · {error}")
|
||
return 1
|
||
try:
|
||
provider_output_schema = load_provider_output_schema(prompt_id)
|
||
except ValueError as exc:
|
||
print(f"[FAIL·fail-closed] {prompt_id} structured output 契约非法:{exc}")
|
||
return 1
|
||
|
||
image_count = 0
|
||
for sample in samples:
|
||
try:
|
||
messages, _ = build_model_messages(prompt_id, contract["body"], sample["input"])
|
||
except ValueError as exc:
|
||
print(f"[FAIL·fail-closed] {sample['evalKey']} 无法构造生产同形消息:{exc}")
|
||
return 1
|
||
user_content = messages[-1].get("content") if messages else None
|
||
if isinstance(user_content, list):
|
||
image_count += sum(1 for part in user_content if part.get("type") == "image_url")
|
||
|
||
parser_note = "生产 parser canonical 自校验通过" \
|
||
if prompt_id == "playtest.actor" or prompt_id in PLAYTEST_JUDGE_IDS \
|
||
else "无需角色 parser 自校验"
|
||
print(
|
||
f"[OK] registry/frontmatter id+version 一致；金标 {len(samples)} 条；"
|
||
f"真实图片 {image_count} 张；{parser_note}；"
|
||
f"structured output={'有' if provider_output_schema else '无'}；未调用模型、未写台账。"
|
||
)
|
||
return 0
|
||
|
||
def run_gate_for_id(prompt_id, registry, base_url, api_key, model, *, update_baseline, ledger_dir):
|
||
"""对单个 prompt 条目跑段 B。返回 exit code（0 绿/SKIP，1 未过/金标缺失，2 跑不通）。"""
|
||
print(f"\n{'='*70}\n[段B·真模型闸] 目标条目：{prompt_id}\n{'='*70}")
|
||
|
||
# ── 三态门：非 live 一律 SKIP 豁免（不花真模型钱焊死面）──
|
||
if prompt_id not in LIVE_PROMPT_IDS:
|
||
print(f"[SKIP] {prompt_id} 非 live（fossil / batch-relic，见 registry 头部三态对账）→ 真模型闸豁免，放行。")
|
||
return 0
|
||
|
||
entry = registry.get(prompt_id)
|
||
if not entry:
|
||
print(f"[FAIL] {prompt_id} 不在 registry.yaml（白名单与 registry 漂移？）", file=sys.stderr)
|
||
return 1
|
||
|
||
# ── 金标集：live 但无金标 → fail-closed（绝不假绿）──
|
||
eval_id = prompt_id # eval 目录名 = prompt id（registry eval 字段亦指向 eval/<id>/）
|
||
samples, gerr = load_golden(eval_id)
|
||
if gerr:
|
||
print(f"[FAIL·fail-closed] {prompt_id} 是 live 面但金标集不可用：{gerr}")
|
||
print(" → 按纪律 fail-closed（绝不假绿）。补齐 eval/<id>/inputs.jsonl+labels.jsonl 后本闸即真跑。")
|
||
return 1
|
||
if ledger_dir is None and any(
|
||
_contains_mapping_key(sample.get("input"), "imageDataUrl") for sample in samples
|
||
):
|
||
print("[FAIL·fail-closed] 仓内持久化模式禁止 input.imageDataUrl；请改用 eval 根内 imagePath。")
|
||
return 1
|
||
sample_errors = validate_golden_samples(prompt_id, samples)
|
||
if sample_errors:
|
||
print(f"[FAIL·fail-closed] {prompt_id} 金标契约非法：")
|
||
for error in sample_errors:
|
||
print(f" · {error}")
|
||
return 1
|
||
|
||
# ── 加载 prompt 正文，并对账 registry/frontmatter id + version ──
|
||
contract, contract_errors = load_prompt_contract(prompt_id, entry)
|
||
if contract_errors:
|
||
print(f"[FAIL·fail-closed] {prompt_id} prompt 身份契约非法：")
|
||
for error in contract_errors:
|
||
print(f" · {error}")
|
||
return 1
|
||
body = contract["body"]
|
||
schema = contract["schema"]
|
||
version = contract["version"]
|
||
try:
|
||
provider_output_schema = load_provider_output_schema(prompt_id)
|
||
snapshot = build_eval_snapshot(
|
||
eval_id, contract, samples, provider_output_schema=provider_output_schema,
|
||
)
|
||
audit_context = begin_prompt_eval_audit(
|
||
ledger_dir, eval_id, prompt_id, entry, contract, samples, snapshot, model, base_url,
|
||
provider_output_schema,
|
||
)
|
||
ledger_root = audit_context["ledgerRoot"]
|
||
run_id = audit_context["runId"]
|
||
except (OSError, ValueError) as exc:
|
||
print(f"[FAIL·fail-closed] 无法封存 prompt eval 输入快照:{exc}")
|
||
return 1
|
||
print(f"[信息] version={version} 金标集={len(samples)} 条 output_schema={'有' if schema else '无(仅校验JSON对象)'}")
|
||
|
||
# ── 逐条真跑模型 → 收集裁决 ──
|
||
results = []
|
||
total_pt = total_ct = 0
|
||
total_latency = 0
|
||
acc_cost = 0.0
|
||
n_error = 0
|
||
for sample_index, s in enumerate(samples, start=1):
|
||
policy_seed = eval_policy_seed(prompt_id, s["evalKey"])
|
||
try:
|
||
messages, max_tokens, material = build_model_request_material(prompt_id, body, s["input"])
|
||
except ValueError as exc:
|
||
shutil.rmtree(audit_context["stagingDir"], ignore_errors=True)
|
||
print(f"[FAIL·fail-closed] {s['evalKey']} 无法构造生产同形消息:{exc}")
|
||
return 1
|
||
try:
|
||
call_result = normalize_model_call_result(
|
||
call_model(
|
||
base_url, api_key, model, messages, max_tokens=max_tokens,
|
||
seed=policy_seed, output_schema=provider_output_schema,
|
||
),
|
||
model, messages, max_tokens, policy_seed, provider_output_schema,
|
||
)
|
||
except (OSError, UnicodeError, ValueError) as exc:
|
||
shutil.rmtree(audit_context["stagingDir"], ignore_errors=True)
|
||
print(f"[FAIL·审计] {s['evalKey']} 模型调用结果不可封存:{exc}")
|
||
return 1
|
||
content = call_result.get("content")
|
||
pt = int(call_result.get("promptTokens") or 0)
|
||
ct = int(call_result.get("completionTokens") or 0)
|
||
latency_ms = int(call_result.get("latencyMs") or 0)
|
||
cerr = call_result.get("error")
|
||
total_pt += pt
|
||
total_ct += ct
|
||
total_latency += latency_ms
|
||
acc_cost += estimate_cost_rmb(pt, ct)
|
||
# 预算 cap 硬闸：累计估算成本超 ¥5 立即中止
|
||
if acc_cost > PER_RUN_BUDGET_CAP_RMB:
|
||
shutil.rmtree(audit_context["stagingDir"], ignore_errors=True)
|
||
print(f"[FAIL·预算] 累计估算成本 ¥{acc_cost:.4f} 超单跑 cap ¥{PER_RUN_BUDGET_CAP_RMB} → 中止。")
|
||
return 1
|
||
if cerr:
|
||
n_error += 1
|
||
parsed = {"ok": False, "code": "model_call_failed", "message": _sanitize_audit_error(cerr)}
|
||
gold_result = {
|
||
"schemaOk": False, "schemaBad": ["model_call_failed"],
|
||
"correct": False, "verdict": None,
|
||
}
|
||
try:
|
||
raw_audit = write_prompt_eval_sample_audit(
|
||
audit_context, sample_index, s, messages, material,
|
||
call_result, parsed, gold_result, base_url,
|
||
)
|
||
except (OSError, UnicodeError, ValueError) as exc:
|
||
shutil.rmtree(audit_context["stagingDir"], ignore_errors=True)
|
||
print(f"[FAIL·审计] {s['evalKey']} 审计包写入失败:{exc}")
|
||
return 1
|
||
results.append({
|
||
"evalKey": s["evalKey"],
|
||
"schema_ok": False,
|
||
"correct": False,
|
||
"verdict": None,
|
||
"schema_bad": ["model_call_failed"],
|
||
"error": str(cerr)[:1000],
|
||
"seed": policy_seed,
|
||
"rawAudit": raw_audit,
|
||
})
|
||
print(f" · {s['evalKey']}: [调用失败] {cerr}")
|
||
continue
|
||
parsed = parse_model_output(prompt_id, content, s["input"])
|
||
obj = parsed.get("normalized") if parsed.get("ok") else None
|
||
s_ok, bad = gate1_schema_ok(obj, schema)
|
||
if not parsed.get("ok"):
|
||
s_ok = False
|
||
bad = [*bad, f"生产 parser:{parsed.get('code', 'unknown')}:{parsed.get('message', '')}"]
|
||
correct = judge_correct(prompt_id, parsed, s["label"])
|
||
verdict = key_verdict(prompt_id, parsed, s["label"])
|
||
gold_result = {
|
||
"schemaOk": s_ok, "schemaBad": bad, "correct": bool(correct), "verdict": verdict,
|
||
}
|
||
try:
|
||
raw_audit = write_prompt_eval_sample_audit(
|
||
audit_context, sample_index, s, messages, material,
|
||
call_result, parsed, gold_result, base_url,
|
||
)
|
||
except (OSError, UnicodeError, ValueError) as exc:
|
||
shutil.rmtree(audit_context["stagingDir"], ignore_errors=True)
|
||
print(f"[FAIL·审计] {s['evalKey']} 审计包写入失败:{exc}")
|
||
return 1
|
||
results.append({
|
||
"evalKey": s["evalKey"], "schema_ok": s_ok,
|
||
"correct": bool(correct), "verdict": verdict,
|
||
"schema_bad": bad,
|
||
"error": None,
|
||
"seed": policy_seed,
|
||
"rawAudit": raw_audit,
|
||
})
|
||
flag = "OK" if (s_ok and correct) else ("schema✘" if not s_ok else "判错✘")
|
||
display = json.dumps(obj, ensure_ascii=False) if obj is not None else repr(content)
|
||
print(f" · {s['evalKey']}: {flag} 产物={display}")
|
||
|
||
# ── 响应覆盖：任何调用失败都使本轮基础设施不确定，绝不能用缩小后的分母给 allGreen ──
|
||
# 已响应样本仍计算并逐条落账，便于复盘；最终退出码固定为 2，要求完整复跑。
|
||
responded = [r for r in results if not r.get("error")]
|
||
infrastructure_complete = n_error == 0 and len(responded) == len(samples)
|
||
n_judged = len(responded)
|
||
|
||
# ── 闸1 Schema：必须覆盖全部样本，不能把失败调用排除后假绿 ──
|
||
n_schema_ok = sum(1 for r in responded if r["schema_ok"])
|
||
gate1 = infrastructure_complete and n_schema_ok == len(samples)
|
||
# ── 闸2 成功率：仅完整响应时形成有效统计；不完整时保留观测值但闸为红 ──
|
||
n_correct = sum(1 for r in responded if r["correct"])
|
||
rate = n_correct / n_judged if n_judged else 0.0
|
||
required_correct = minimum_correct_required(prompt_id, len(samples))
|
||
gate2 = infrastructure_complete and n_correct >= required_correct
|
||
# ── 闸3 回归 diff（有基线才判；无基线 advisory）──
|
||
baseline = load_baseline(eval_id)
|
||
audit_identity = {
|
||
"auditSchemaVersion": AUDIT_SCHEMA_VERSION,
|
||
"codeIdentityHash": audit_context["codeIdentityHash"],
|
||
}
|
||
baseline_identity_ok = baseline_identity_matches(
|
||
baseline, snapshot, model, audit_identity,
|
||
) if baseline else True
|
||
gate3, gate3_note = True, "无基线，首次跑（advisory，未拦）"
|
||
if not infrastructure_complete:
|
||
gate3, gate3_note = False, "模型响应不完整，回归比较无效"
|
||
elif baseline and not baseline_identity_ok:
|
||
gate3, gate3_note = False, "基线身份与本轮 Prompt/fixture/model/seed 不一致，必须重建基线"
|
||
elif baseline and prompt_id == "playtest.actor" and baseline.get("perKeyCorrect"):
|
||
# Actor 允许多个等价正确动作；回归比较金标达标，不比较某个具体等价动作。
|
||
gate3 = gate1 and gate2
|
||
gate3_note = "Actor 金标单轮标准保持达标" if gate3 else "Actor 金标单轮标准回退"
|
||
elif baseline and prompt_id in PLAYTEST_JUDGE_IDS and baseline.get("perKeyCorrect"):
|
||
gate3 = gate1 and gate2
|
||
gate3_note = "Judge 金标全对标准保持达标" if gate3 else "Judge 金标全对标准回退"
|
||
elif baseline and baseline.get("perKey"):
|
||
flipped = []
|
||
for r in responded:
|
||
bk = baseline["perKey"].get(r["evalKey"])
|
||
if bk is not None and bk != r.get("verdict"):
|
||
flipped.append(r["evalKey"])
|
||
gate3 = (len(flipped) == 0)
|
||
gate3_note = "无关键裁决翻转" if gate3 else f"关键裁决翻转：{flipped}"
|
||
# ── 闸4 成本/延迟（有基线才判增量；无基线 advisory）──
|
||
avg_cost = acc_cost / n_judged if n_judged else 0.0
|
||
avg_latency = total_latency / n_judged if n_judged else 0.0
|
||
gate4, gate4_note = True, "无基线，首次跑（advisory，未拦）"
|
||
if not infrastructure_complete:
|
||
gate4, gate4_note = False, "模型响应不完整，成本/延迟比较无效"
|
||
elif baseline and not baseline_identity_ok:
|
||
gate4, gate4_note = False, "基线身份不一致，成本/延迟不可比较"
|
||
elif baseline and baseline.get("cost") is not None:
|
||
base_cost = baseline["cost"] or 1e-9
|
||
base_lat = baseline.get("latency") or 1e-9
|
||
cost_delta = (avg_cost - base_cost) / base_cost
|
||
lat_delta = (avg_latency - base_lat) / base_lat
|
||
cost_ok = cost_delta <= MAX_COST_DELTA_RATIO
|
||
lat_ok = lat_delta <= MAX_LATENCY_DELTA_RATIO
|
||
gate4 = cost_ok and lat_ok
|
||
gate4_note = f"成本Δ={cost_delta:+.1%}(限+{MAX_COST_DELTA_RATIO:.0%}) 延迟Δ={lat_delta:+.1%}(限+{MAX_LATENCY_DELTA_RATIO:.0%})"
|
||
|
||
# ── 闸5 重复稳定性：同一 exact request 连续三轮满足金标聚合门，第三轮只准建线 ──
|
||
request_hashes = {
|
||
result["evalKey"]: (result.get("rawAudit") or {}).get("requestBodySha256")
|
||
for result in results
|
||
}
|
||
signatures = {result["evalKey"]: result.get("verdict") for result in results}
|
||
per_key_correct = {result["evalKey"]: result.get("correct") is True for result in results}
|
||
per_key_schema_ok = {result["evalKey"]: result.get("schema_ok") is True for result in results}
|
||
request_hashes_complete = len(request_hashes) == len(samples) and all(
|
||
isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value)
|
||
for value in request_hashes.values()
|
||
)
|
||
stability_identity = {
|
||
"promptBodySha256": snapshot["promptBodySha256"],
|
||
"fixtureSnapshot": snapshot["fixtures"],
|
||
"model": model,
|
||
"providerProtocol": PROVIDER_PROTOCOL,
|
||
"providerOutputSchemaSha256": snapshot.get("providerOutputSchemaSha256"),
|
||
"thinkingBudgetTokens": ANTHROPIC_ACTOR_THINKING_BUDGET
|
||
if prompt_id == "playtest.actor" else ANTHROPIC_JUDGE_THINKING_BUDGET,
|
||
"maxTokens": ANTHROPIC_ACTOR_MAX_TOKENS
|
||
if prompt_id == "playtest.actor" else ANTHROPIC_JUDGE_MAX_TOKENS,
|
||
"seedStrategy": SEED_STRATEGY,
|
||
"auditSchemaVersion": AUDIT_SCHEMA_VERSION,
|
||
"codeIdentityHash": audit_context["codeIdentityHash"],
|
||
}
|
||
stability_qualified = bool(
|
||
infrastructure_complete and gate1 and gate2 and gate4 and request_hashes_complete
|
||
)
|
||
stability = evaluate_repeat_stability(
|
||
ledger_root, prompt_id, stability_identity, request_hashes, signatures,
|
||
per_key_correct, per_key_schema_ok, stability_qualified,
|
||
)
|
||
stability.update({
|
||
"identity": stability_identity,
|
||
"requestBodySha256": request_hashes,
|
||
"signatures": signatures,
|
||
"perKeyCorrect": per_key_correct,
|
||
"perKeySchemaOk": per_key_schema_ok,
|
||
"requiredRuns": MIN_STABLE_RUNS,
|
||
})
|
||
baseline_stability = baseline.get("stability") if isinstance(baseline, dict) \
|
||
and isinstance(baseline.get("stability"), dict) else {}
|
||
baseline_cohort_ready = bool(
|
||
baseline and baseline_identity_ok
|
||
and baseline_stability.get("strategy") == PLAYTEST_STABILITY_STRATEGY
|
||
and baseline_stability.get("cohortReady") is True
|
||
and stable_json(baseline_stability.get("requestBodySha256")) == stable_json(request_hashes)
|
||
)
|
||
# 第四轮及以后由已固定的三轮 cohort 证明稳定性，本轮仍须独立满足单轮标准。
|
||
gate5 = bool(
|
||
stability_qualified
|
||
and (baseline_cohort_ready or stability.get("cohortReady") is True)
|
||
)
|
||
gate5_note = (
|
||
f"同请求合格 {stability['consecutiveRuns']} 轮；"
|
||
f"三轮 cohort 正确 {stability.get('cohortCorrect', 0)}/"
|
||
f"{len(samples) * MIN_STABLE_RUNS}"
|
||
if stability_qualified else "本轮基础闸未全绿，稳定性计数归零"
|
||
)
|
||
baseline_ready = bool(baseline and baseline_identity_ok)
|
||
gates_green = infrastructure_complete and gate1 and gate2 and gate3 and gate4 and gate5 \
|
||
and baseline_ready
|
||
audit_ready = len(audit_context["samples"]) == len(samples) \
|
||
and not audit_context["auditErrors"] and audit_context["requestReplayable"]
|
||
baseline_update_eligible = bool(
|
||
update_baseline and audit_ready and infrastructure_complete and gate1 and gate2 and gate4
|
||
and gate5
|
||
)
|
||
# 建基线是一次状态变更，不得用刚写出的基线批准同一次运行。
|
||
all_green = gates_green and not update_baseline
|
||
|
||
# ── 报告 ──
|
||
print(f"\n{'-'*70}")
|
||
print(
|
||
f"[响应覆盖 ] {'绿' if infrastructure_complete else '不确定'} "
|
||
f"{len(responded)}/{len(samples)} 条收到响应；失败 {n_error} 条"
|
||
)
|
||
print(f"[闸1 Schema ] {'绿' if gate1 else '红'} {n_schema_ok}/{len(samples)} 条产物合 output_schema")
|
||
print(
|
||
f"[闸2 成功率 ] {'绿' if gate2 else '红'} 观测判对率 {rate:.0%}"
|
||
f"（专项门 ≥{required_correct}/{len(samples)}，当前 {n_correct}/{n_judged}）"
|
||
)
|
||
print(f"[闸3 回归diff] {'绿' if gate3 else '红'} {gate3_note}")
|
||
print(f"[闸4 成本延迟] {'绿' if gate4 else '红'} {gate4_note}")
|
||
print(f"[闸5 重复稳定] {'绿' if gate5 else '红'} {gate5_note}")
|
||
print(f"[成本] 本跑估算 ¥{acc_cost:.4f}（tokens: prompt={total_pt} completion={total_ct}；cap ¥{PER_RUN_BUDGET_CAP_RMB}）")
|
||
print(f"[延迟] 累计 {total_latency}ms（均 {avg_latency:.0f}ms/条）")
|
||
gate_decision = "五闸全绿且第四轮确认 → 放行" if all_green else (
|
||
"符合建基线条件 → 写入后必须基于新基线复跑，本次不放行"
|
||
if baseline_update_eligible else (
|
||
"基础设施不确定 → 不形成 prompt 裁决，必须完整复跑"
|
||
if not infrastructure_complete else (
|
||
"已满足三轮稳定，请用 --update-baseline 建线；本轮不放行"
|
||
if gate5 and not baseline_ready and not update_baseline
|
||
else "五闸未全绿 → 拦截"
|
||
))
|
||
)
|
||
print(f"{'-'*70}\n[裁决] {gate_decision}")
|
||
|
||
try:
|
||
audit_ref = finalize_prompt_eval_audit(
|
||
audit_context, prompt_id, version, model, snapshot, len(samples), (api_key,),
|
||
)
|
||
except (OSError, UnicodeError, ValueError) as exc:
|
||
shutil.rmtree(audit_context["stagingDir"], ignore_errors=True)
|
||
print(f"[FAIL·审计] PromptEvalAudit/2 无法原子封口:{exc}")
|
||
return 1
|
||
if not audit_ref.get("auditComplete"):
|
||
print("[FAIL·审计] PromptEvalAudit/2 不完整；禁止更新基线和追加台账。")
|
||
return 1
|
||
|
||
# staging 内路径封口后统一投影成相对 ledgerRoot 的最终路径，兼容旧 rawAudit 消费方。
|
||
final_prefix = audit_context["finalDir"].relative_to(ledger_root)
|
||
for result in results:
|
||
raw_ref = result.get("rawAudit")
|
||
if isinstance(raw_ref, dict) and raw_ref.get("path"):
|
||
raw_ref["path"] = (final_prefix / raw_ref["path"]).as_posix()
|
||
|
||
record_ts = datetime.now(timezone.utc).isoformat()
|
||
baseline_updated = False
|
||
if baseline_update_eligible:
|
||
bl = {
|
||
"promptId": prompt_id, "version": version, "updatedAt": record_ts,
|
||
"promptBodySha256": snapshot["promptBodySha256"],
|
||
"fixtureSnapshot": snapshot["fixtures"],
|
||
"model": model,
|
||
"seedStrategy": SEED_STRATEGY,
|
||
"providerProtocol": PROVIDER_PROTOCOL,
|
||
"providerOutputSchemaSha256": snapshot.get("providerOutputSchemaSha256"),
|
||
"thinkingBudgetTokens": stability_identity["thinkingBudgetTokens"],
|
||
"maxTokens": stability_identity["maxTokens"],
|
||
"auditRunHash": audit_ref["runAuditHash"],
|
||
"auditSchemaVersion": AUDIT_SCHEMA_VERSION,
|
||
"codeIdentityHash": audit_context["codeIdentityHash"],
|
||
"stability": {
|
||
"strategy": PLAYTEST_STABILITY_STRATEGY
|
||
if prompt_id == "playtest.actor" or prompt_id in PLAYTEST_JUDGE_IDS
|
||
else "exact-signature-v1",
|
||
"requiredRuns": MIN_STABLE_RUNS,
|
||
"consecutiveRuns": stability["consecutiveRuns"],
|
||
"currentRunId": run_id,
|
||
"previousRunIds": stability["previousRunIds"],
|
||
"requestBodySha256": request_hashes,
|
||
"signatures": signatures,
|
||
"cohortReady": stability.get("cohortReady") is True,
|
||
"cohortCorrect": stability.get("cohortCorrect"),
|
||
"cohortSchemaOk": stability.get("cohortSchemaOk"),
|
||
"perKeyCorrectCounts": stability.get("perKeyCorrectCounts"),
|
||
},
|
||
"perKey": {r["evalKey"]: r.get("verdict") for r in results if not r.get("error")},
|
||
"perKeyCorrect": per_key_correct,
|
||
"perKeySchemaOk": per_key_schema_ok,
|
||
"cost": round(avg_cost, 6), "latency": round(avg_latency, 1),
|
||
"note": "基线快照：关键裁决 + 单条均成本/延迟；写入当次固定不放行，须复跑确认",
|
||
}
|
||
try:
|
||
write_baseline_atomic(eval_id, bl)
|
||
baseline_updated = True
|
||
print(f"[基线] 已更新 → {baseline_path(eval_id)}；本次固定不放行，请复跑。")
|
||
except OSError as exc:
|
||
print(f"[FAIL·基线] 原子写入失败:{exc}")
|
||
|
||
# ── 台账 append ──
|
||
record = {
|
||
"ts": record_ts,
|
||
"runId": run_id,
|
||
"promptId": prompt_id, "version": version, "model": model,
|
||
"promptBodySha256": snapshot["promptBodySha256"],
|
||
"fixtureSnapshot": snapshot["fixtures"],
|
||
"nSamples": len(samples), "nJudged": n_judged, "nSchemaOk": n_schema_ok, "nCorrect": n_correct,
|
||
"successRate": round(rate, 4),
|
||
"gate1_schema": gate1, "gate2_success": gate2, "gate3_regression": gate3,
|
||
"gate4_cost_latency": gate4, "gate5_stability": gate5,
|
||
"infrastructureComplete": infrastructure_complete,
|
||
"seedStrategy": SEED_STRATEGY,
|
||
"codeIdentityHash": audit_context["codeIdentityHash"],
|
||
"stability": stability,
|
||
"status": "passed" if all_green else (
|
||
"infrastructure_uncertain" if not infrastructure_complete else (
|
||
"baseline_updated_rerun_required" if baseline_updated else "failed"
|
||
)
|
||
),
|
||
"allGreen": all_green,
|
||
"baselineUpdateRequested": bool(update_baseline),
|
||
"baselineUpdateEligible": baseline_update_eligible,
|
||
"baselineUpdated": baseline_updated,
|
||
"auditManifest": audit_ref,
|
||
"estCostRmb": round(acc_cost, 6), "promptTokens": total_pt, "completionTokens": total_ct,
|
||
"totalLatencyMs": total_latency, "nError": n_error,
|
||
"gate3_note": gate3_note, "gate4_note": gate4_note, "gate5_note": gate5_note,
|
||
# 只落影响闸门的结构化结果；模型 raw 不进台账，避免膨胀和不可信文本扩散。
|
||
"perKey": {
|
||
result["evalKey"]: {
|
||
"schema_ok": result["schema_ok"],
|
||
"correct": result["correct"],
|
||
"verdict": result.get("verdict"),
|
||
"schema_bad": result.get("schema_bad") or [],
|
||
"error": result.get("error"),
|
||
"seed": result.get("seed"),
|
||
"rawAudit": result.get("rawAudit"),
|
||
}
|
||
for result in results
|
||
},
|
||
}
|
||
ledger_f = write_ledger(ledger_dir, eval_id, record)
|
||
print(f"[台账] 已追加 → {ledger_f}")
|
||
|
||
if not infrastructure_complete:
|
||
return 2
|
||
if update_baseline:
|
||
return 1
|
||
return 0 if all_green else 1
|
||
|
||
|
||
# ══════════════════════ 改动条目发现（--changed）══════════════════════
|
||
|
||
def changed_prompt_ids(registry, base):
|
||
"""git diff base 找改动的 prompt .md，映射回 registry id。"""
|
||
try:
|
||
root = subprocess.run(["git", "rev-parse", "--show-toplevel"], cwd=str(PROMPTS_DIR),
|
||
capture_output=True, text=True, check=True).stdout.strip()
|
||
out = subprocess.run(["git", "diff", "--name-only", base, "--", "contracts/prompts/"],
|
||
cwd=root, capture_output=True, text=True, check=True).stdout
|
||
except (subprocess.CalledProcessError, OSError) as e:
|
||
print(f"[错误] git diff 失败：{e}", file=sys.stderr)
|
||
return []
|
||
changed_files = {ln.strip() for ln in out.splitlines() if ln.strip().endswith(".md")}
|
||
ids = []
|
||
for pid, entry in registry.items():
|
||
rel = entry.get("file")
|
||
if rel and f"contracts/prompts/{rel}" in changed_files:
|
||
ids.append(pid)
|
||
return ids
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="Prompt 四道闸 · 真模型闸（段 B）")
|
||
g = parser.add_mutually_exclusive_group(required=True)
|
||
g.add_argument("--id", help="对指定 prompt id 跑真模型闸")
|
||
g.add_argument("--changed", action="store_true", help="对相对基线改动的 live 条目跑（配 --base）")
|
||
parser.add_argument("--base", default="HEAD~1", help="--changed 的对比基线（默认 HEAD~1）")
|
||
parser.add_argument("--model", default=os.environ.get("EVAL_MODEL", DEFAULT_MODEL), help="模型名（默认 MiniMax-M3）")
|
||
parser.add_argument("--update-baseline", action="store_true", help="真跑后建/更新基线快照（闸3/4 后续据此判）")
|
||
parser.add_argument("--ledger-dir", default=None, help="台账目录（默认 eval/<id>/runs/；演示可指临时目录避免污染仓）")
|
||
parser.add_argument(
|
||
"--validate-only", action="store_true",
|
||
help="只跑 registry/version、金标、真实图片与生产 parser 离线门；绝不调用模型",
|
||
)
|
||
args = parser.parse_args()
|
||
|
||
if args.validate_only and args.update_baseline:
|
||
parser.error("--validate-only 不得与 --update-baseline 同用")
|
||
|
||
# 凭据：从 env 读，绝不硬编码 key（权威源 docs/内网凭据与端点.md）。
|
||
api_key = os.environ.get("NEWAPI_KEY", "").strip()
|
||
base_url = os.environ.get("NEWAPI_BASE_URL", DEFAULT_BASE_URL).strip()
|
||
|
||
registry = parse_registry(REGISTRY_FILE)
|
||
|
||
# 白名单自检：LIVE_PROMPT_IDS 必须都在 registry（防漂移/typo）
|
||
drift = [pid for pid in LIVE_PROMPT_IDS if pid not in registry]
|
||
if drift:
|
||
print(f"[错误] 段B白名单与 registry 漂移，以下 id 不在 registry：{drift}", file=sys.stderr)
|
||
sys.exit(1)
|
||
|
||
if args.changed:
|
||
targets = changed_prompt_ids(registry, args.base)
|
||
if not targets:
|
||
print(f"[OK] 相对基线 {args.base} 无改动的 prompt 条目 → 真模型闸无需跑，放行。")
|
||
sys.exit(0)
|
||
print(f"[信息] 改动条目：{targets}")
|
||
else:
|
||
targets = [args.id]
|
||
|
||
if args.validate_only:
|
||
worst = 0
|
||
for target in targets:
|
||
worst = max(worst, run_offline_validation_for_id(target, registry))
|
||
sys.exit(worst)
|
||
|
||
# 真跑需要 key；仅当存在「live 且有金标」目标（真会调模型）时才强制 key。
|
||
def _will_call_model(pid):
|
||
if pid not in LIVE_PROMPT_IDS:
|
||
return False
|
||
samples, err = load_golden(pid)
|
||
return err is None
|
||
if any(_will_call_model(t) for t in targets) and not api_key:
|
||
print("[错误] 需真调模型但 NEWAPI_KEY 未设（读 docs/内网凭据与端点.md，经 env 传入，勿设长期 env）。",
|
||
file=sys.stderr)
|
||
sys.exit(1)
|
||
|
||
worst = 0
|
||
for t in targets:
|
||
code = run_gate_for_id(t, registry, base_url, api_key, args.model,
|
||
update_baseline=args.update_baseline, ledger_dir=args.ledger_dir)
|
||
worst = max(worst, code) if code != 2 else (worst if worst == 1 else 2)
|
||
sys.exit(worst)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|