#!/usr/bin/env python3 """把 AI 味案例卡与来源重验证回执写入 agent-example 的 muse-example。 采集脚本保持确定性、可离线回放；本脚本是唯一的持久化边界，复用 ``access-database`` 的连接入口。案例卡做幂等当前投影，重验证批次/回执做 append-only 账本。研究限定来源只写 hash、位置和观察，不写第三方正文。 """ from __future__ import annotations import argparse import hashlib import json import sys from pathlib import Path from typing import Any import yaml SCRIPT_DIR = Path(__file__).resolve().parent DB_SCRIPTS = SCRIPT_DIR.parents[1] / "access-database" / "scripts" if str(SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(SCRIPT_DIR)) if str(DB_SCRIPTS) not in sys.path: sys.path.insert(0, str(DB_SCRIPTS)) from capture_cases import ( # noqa: E402 CaseCardError, REVALIDATION_SCHEMA_VERSION, SCHEMA_VERSION, load_verification, validate_card, ) from db import connect # noqa: E402 TENANT_ID = 1 CREATOR = "1" def _logical_ref(path: Path) -> str: """落库只保留可迁移的证据名，不把本机绝对路径当跨环境引用。""" return f"capture://ai-flavor/{path.name}" def _sha256_file(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def _json(value: Any) -> str: return json.dumps(value, ensure_ascii=False, separators=(",", ":")) def _load_object(path: Path) -> dict: try: value = yaml.safe_load(path.read_text(encoding="utf-8")) or {} except (OSError, UnicodeError, yaml.YAMLError) as exc: raise CaseCardError(f"{path}: 读取失败: {exc}") from exc if not isinstance(value, dict): raise CaseCardError(f"{path}: 顶层必须是对象") return value def load_inventory(path: Path) -> dict: """加载并逐卡验证 inventory；返回可安全写入的报告对象。""" report = _load_object(path) if report.get("schema_version") != "ai-flavor-inventory-v1": raise CaseCardError(f"{path}: 不是 ai-flavor-inventory-v1 清单") cards = report.get("cards") if not isinstance(cards, list): raise CaseCardError(f"{path}: cards 必须是数组") seen: set[str] = set() for card in cards: validate_card(card) if card["id"] in seen: raise CaseCardError(f"{path}: 案例卡 id 重复: {card['id']}") seen.add(card["id"]) totals = report.get("totals") or {} if totals.get("cards") != len(cards): raise CaseCardError(f"{path}: totals.cards 与 cards 数量不一致") return report def _validate_pair(inventory: dict, verification: dict, *, inventory_ref: str, report_ref: str, inventory_sha256: str, report_sha256: str) -> dict: """校验一次检测产出的卡片集合和同运行重验证回执。""" if inventory.get("schema_version") != "ai-flavor-inventory-v1": raise CaseCardError("inventory 不是 ai-flavor-inventory-v1 清单") cards = inventory.get("cards") if not isinstance(cards, list): raise CaseCardError("inventory.cards 必须是数组") for card in cards: validate_card(card) verification = verification if isinstance(verification, dict) else {} if verification.get("schema_version") != REVALIDATION_SCHEMA_VERSION: raise CaseCardError("重验证版本不支持") if verification.get("report_ref") and verification["report_ref"] != report_ref: raise CaseCardError("重验证 report_ref 与导入引用不一致") results = verification["cards"] if not isinstance(results, list): raise CaseCardError("重验证 cards 必须是数组") card_ids = {card["id"] for card in cards} result_ids = {item.get("card_id") for item in results} if card_ids != result_ids: missing = sorted(card_ids - result_ids) extra = sorted(result_ids - card_ids) raise CaseCardError(f"清单/重验证卡片集合不一致: missing={missing[:3]} extra={extra[:3]}") by_id = {card["id"]: card for card in cards} for result in results: card = by_id[result["card_id"]] source_sha = card["source"]["source_sha256"] if result.get("expected_source_sha256") != source_sha: raise CaseCardError(f"{result['card_id']}: expected_source_sha256 与卡片不一致") if result.get("checked_on") != verification.get("checked_on"): raise CaseCardError(f"{result['card_id']}: checked_on 与批次不一致") return { "inventory": inventory, "verification": verification, "inventory_sha256": inventory_sha256, "report_sha256": report_sha256, "inventory_ref": inventory_ref, "report_ref": report_ref, } def prepare_import(inventory_path: Path, verification_path: Path) -> dict: """准备恢复/迁移导入；正常检测不需要调用此函数。""" return _validate_pair( load_inventory(inventory_path), load_verification(verification_path), inventory_ref=_logical_ref(inventory_path), report_ref=_logical_ref(verification_path), inventory_sha256=_sha256_file(inventory_path), report_sha256=_sha256_file(verification_path), ) def persist_generated(*, inventory: dict, verification: dict, inventory_ref: str, report_ref: str, inventory_sha256: str, report_sha256: str, creator: str = CREATOR, tenant_id: int = TENANT_ID) -> dict: """检测命令的自动写入口；不要求先生成可导入文件。""" prepared = _validate_pair( inventory, verification, inventory_ref=inventory_ref, report_ref=report_ref, inventory_sha256=inventory_sha256, report_sha256=report_sha256, ) return persist(prepared, creator=creator, tenant_id=tenant_id) def _card_params(card: dict, prepared: dict, *, creator: str, tenant_id: int) -> tuple: source = card["source"] return ( card["id"], card["schema_version"], card["card_type"], card["state"], card["label"], card["layer"], card["carrier"], card["capture_mode"], source["kind"], source["license"], source["source_sha256"], source["excerpt_sha256"], source.get("source_ref", ""), source.get("work_ref"), _json(source["location"]), _json(source.get("surface_location")) if source.get("surface_location") else None, card.get("excerpt", ""), card.get("context", ""), _json(card["observation"]), _json(card["feedback"]) if card.get("feedback") is not None else None, _json(card["review"]) if card.get("review") is not None else None, _json(card.get("rule_candidate_ids", [])), prepared["inventory_ref"], prepared["inventory_sha256"], creator, creator, tenant_id, ) def persist(prepared: dict, *, creator: str = CREATOR, tenant_id: int = TENANT_ID) -> dict: """事务性导入；重复运行不重复插入批次/逐卡回执。""" inventory = prepared["inventory"] verification = prepared["verification"] cards = inventory["cards"] results = verification["cards"] batch_sql = ( "INSERT INTO example_ai_flavor_revalidation_batch " "(batch_ref,report_sha256,schema_version,checked_on,source_root_ref,usable,totals,works,report_path,creator,tenant_id) " "VALUES (%s,%s,%s,%s,%s,%s,%s::jsonb,%s::jsonb,%s,%s,%s) " "ON CONFLICT (tenant_id,report_sha256) DO NOTHING RETURNING id" ) card_sql = ( "INSERT INTO example_ai_flavor_case " "(card_id,schema_version,card_type,state,label,layer,carrier,capture_mode,source_kind,source_license," "source_sha256,excerpt_sha256,source_ref,work_ref,source_location,surface_location,excerpt,context,observation," "feedback,review,rule_candidate_ids,inventory_ref,inventory_sha256,creator,updater,tenant_id) " "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s::jsonb,%s::jsonb,%s,%s,%s::jsonb,%s::jsonb,%s::jsonb,%s::jsonb,%s,%s,%s,%s,%s) " "ON CONFLICT (tenant_id,card_id) DO UPDATE SET " "schema_version=EXCLUDED.schema_version,card_type=EXCLUDED.card_type,source_kind=EXCLUDED.source_kind," "source_license=EXCLUDED.source_license,source_sha256=EXCLUDED.source_sha256,excerpt_sha256=EXCLUDED.excerpt_sha256," "source_ref=EXCLUDED.source_ref,work_ref=EXCLUDED.work_ref,source_location=EXCLUDED.source_location," "surface_location=EXCLUDED.surface_location,observation=EXCLUDED.observation,feedback=EXCLUDED.feedback," "rule_candidate_ids=EXCLUDED.rule_candidate_ids,inventory_ref=EXCLUDED.inventory_ref," "inventory_sha256=EXCLUDED.inventory_sha256,updater=EXCLUDED.updater,update_time=CURRENT_TIMESTAMP " "WHERE example_ai_flavor_case.state NOT IN ('canonical','rejected','archived')" ) result_sql = ( "INSERT INTO example_ai_flavor_revalidation " "(batch_id,card_id,work_ref,source_ref,expected_source_sha256,actual_source_sha256,status,reason,checked_on,creator,tenant_id) " "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) " "ON CONFLICT (tenant_id,batch_id,card_id) DO NOTHING" ) with connect() as conn: try: batch_row = conn.execute( batch_sql, (prepared["report_ref"], prepared["report_sha256"], verification["schema_version"], verification["checked_on"], verification.get("source_root_ref"), verification["usable"], _json(verification.get("totals", {})), _json(verification.get("works", [])), prepared["report_ref"], creator, tenant_id), ).fetchone() if batch_row: batch_id = batch_row[0] else: batch_id = conn.execute( "SELECT id FROM example_ai_flavor_revalidation_batch WHERE tenant_id=%s AND report_sha256=%s", (tenant_id, prepared["report_sha256"]), ).fetchone()[0] for card in cards: conn.execute(card_sql, _card_params(card, prepared, creator=creator, tenant_id=tenant_id)) for result in results: conn.execute(result_sql, ( batch_id, result["card_id"], result.get("work_ref"), result.get("source_ref", ""), result["expected_source_sha256"], result.get("actual_source_sha256"), result["status"], result.get("reason", ""), result["checked_on"], creator, tenant_id, )) counts = conn.execute( "SELECT count(*) FILTER (WHERE deleted=false), " "count(*) FILTER (WHERE deleted=false AND state='shadow') " "FROM example_ai_flavor_case WHERE tenant_id=%s", (tenant_id,), ).fetchone() receipt_count = conn.execute( "SELECT count(*) FROM example_ai_flavor_revalidation WHERE tenant_id=%s AND batch_id=%s", (tenant_id, batch_id), ).fetchone()[0] conn.commit() except Exception: conn.rollback() raise return { "status": "written", "batch_id": batch_id, "inventory_cards": len(cards), "case_rows": int(counts[0]), "shadow_rows": int(counts[1]), "revalidation_rows": int(receipt_count), "report_sha256": prepared["report_sha256"], } def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="AI 味案例卡与重验证回执落库") parser.add_argument("inventory", type=Path, help="ai-flavor-inventory-v1 JSON/YAML") parser.add_argument("verification", type=Path, help="ai-flavor-revalidation-v1 JSON/YAML") parser.add_argument("--creator", default=CREATOR) parser.add_argument("--tenant-id", type=int, default=TENANT_ID) return parser def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) prepared = prepare_import(args.inventory, args.verification) print(json.dumps(persist(prepared, creator=args.creator, tenant_id=args.tenant_id), ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": try: raise SystemExit(main()) except (CaseCardError, OSError, KeyError, TypeError, IndexError) as exc: print(f"CASE_CARD_PERSIST_FAILED: {exc}", file=sys.stderr) raise SystemExit(2)