97 lines
5.0 KiB
Python
97 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
||
"""常设不变量检查（harness 改造原则 5：从靠预防迁到靠发现）。
|
||
|
||
逻辑隔离不能全靠预防，必须靠持续发现——从物理隔离迁到逻辑隔离的那一刻，
|
||
必须同时从"靠预防"迁到"靠发现"。本脚本跑一组常设不变量，违反即报警（退出码非 0）。
|
||
建议常设运行（CI / 看板健康面），违反即报警。
|
||
|
||
检查项：
|
||
I1 没有评测候选被接受（run_type=eval/diagnostic 且 state=accepted）——引擎已防，此为发现兜底。
|
||
I2 没有正式正文的来源能追溯到评测候选（归因 source_object_id → 评测候选）。
|
||
I3 评测候选的质量结果可被生产视图过滤（带评测标记，不混入生产质量视图）。
|
||
I4 oracle 读侧红线：oracle/标准答案以 kind='oracle' 标记，生产模型输入不得包含（read-context 生产路径保证；此处校验标记完整）。
|
||
I5 COMPLETED 轮次封存：一轮声明的 raw 集合 == 实际落库集合（lease 声明的 content_hashes 数 == 实际 raw_content 行数）。
|
||
|
||
跑法：.venv/bin/python .agent/skills/record-run-evidence/scripts/invariant_checks.py [--run-id X]
|
||
"""
|
||
import argparse
|
||
import json
|
||
import sys
|
||
|
||
from muse_db import connect
|
||
|
||
|
||
def _count(conn, sql, params=()):
|
||
return conn.execute(sql, params).fetchone()[0]
|
||
|
||
|
||
def check_invariants(run_id=None):
|
||
"""返回 [(检查项, 是否通过, 说明)]。"""
|
||
results = []
|
||
with connect(readonly=True) as conn:
|
||
# I1 没有评测候选被接受
|
||
n = _count(conn, "SELECT count(*) FROM example_candidate "
|
||
"WHERE run_type IN ('eval','diagnostic') AND state='accepted'")
|
||
results.append(("I1 评测候选未被接受", n == 0,
|
||
f"{n} 个评测候选被接受" if n else "无评测候选被接受"))
|
||
|
||
# I2 没有正式正文来源追溯到评测候选
|
||
n = _count(conn, "SELECT count(*) FROM muse_content_block_source_attribution a "
|
||
"JOIN example_candidate c ON c.id::text = a.source_object_id "
|
||
"WHERE c.run_type IN ('eval','diagnostic')")
|
||
results.append(("I2 正式正文来源不追溯到评测候选", n == 0,
|
||
f"{n} 个正式正文来源追溯到评测候选" if n else "无正式正文来源追溯到评测候选"))
|
||
|
||
# I3 评测候选的质量结果可被生产视图过滤（评测候选的质量结果都能关联到评测候选，即可过滤）
|
||
n = _count(conn, "SELECT count(*) FROM example_quality_result q "
|
||
"JOIN example_candidate c ON c.candidate_sha256 = q.candidate_sha256 "
|
||
"WHERE c.run_type IN ('eval','diagnostic') AND q.candidate_sha256 IS NULL")
|
||
results.append(("I3 评测质量结果可过滤", n == 0,
|
||
f"{n} 个评测质量结果无法关联评测候选（无法过滤）" if n else "评测质量结果均可关联过滤"))
|
||
|
||
# I4 oracle 读侧红线：oracle raw 行都带 kind='oracle' 标记（生产路径据此排除）
|
||
n = _count(conn, "SELECT count(*) FROM example_raw_content "
|
||
"WHERE kind='oracle' AND content_sha256 IS NULL")
|
||
results.append(("I4 oracle 标记完整", n == 0,
|
||
f"{n} 个 oracle raw 缺哈希标记" if n else "oracle raw 标记完整"))
|
||
|
||
# I5 COMPLETED 轮次封存：lease 声明的 content_hashes 数 == 实际 raw_content 行数
|
||
if run_id:
|
||
leases = conn.execute(
|
||
"SELECT id, content_hashes FROM example_raw_lease WHERE run_id=%s", (run_id,)).fetchall()
|
||
else:
|
||
leases = conn.execute("SELECT id, content_hashes FROM example_raw_lease").fetchall()
|
||
seal_violations = 0
|
||
for lease_id, content_hashes in leases:
|
||
if isinstance(content_hashes, (list, dict)):
|
||
declared = len(content_hashes)
|
||
else:
|
||
declared = 0
|
||
actual = _count(conn, "SELECT count(*) FROM example_raw_content WHERE lease_id=%s", (lease_id,))
|
||
if declared != actual:
|
||
seal_violations += 1
|
||
results.append(("I5 轮次封存(声明==实际落库)", seal_violations == 0,
|
||
f"{seal_violations} 个租约声明与实际落库不符" if seal_violations else "所有租约封存一致"))
|
||
|
||
return results
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser(description="常设不变量检查（从靠预防迁到靠发现）")
|
||
ap.add_argument("--run-id", default=None, help="只检查某轮的轮次封存")
|
||
args = ap.parse_args()
|
||
results = check_invariants(run_id=args.run_id)
|
||
all_pass = True
|
||
for name, ok, msg in results:
|
||
print(f"[{'PASS' if ok else 'VIOLATION'}] {name}：{msg}")
|
||
if not ok:
|
||
all_pass = False
|
||
if not all_pass:
|
||
print("\n有不变量被违反——逻辑隔离出现越界，须排查。", file=sys.stderr)
|
||
sys.exit(1)
|
||
print("\n所有不变量通过。")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|