#!/usr/bin/env python3 """根据固定卡索引计划回读冻结历史原文。 抽取卡只提供定位与历史状态索引。任何由卡承载的历史事实都必须展开 sourceRefs 并成功读取 asOf 以内的原文；正式设定、Canonical 状态和细纲 新事实通过独立不可变引用进入事实证据。 """ from __future__ import annotations import copy import hashlib import pathlib import sys from typing import Any, Callable, Mapping, Protocol, Sequence from writer_contract import PLAN_VERSION, TIE_BREAK, canonical_json, normalize_text, retrieval_identity class RetrievalError(ValueError): """检索计划、授权、冻结或来源引用不能证明安全时抛出。""" class CardIndexRepository(Protocol): """卡索引仓储只接收固定计划，不允许写手自行追加查询。""" def search(self, plan: Mapping[str, Any]) -> list[dict[str, Any]]: """返回带稳定排序字段、历史里程碑和来源指针的卡索引。""" class ProseRepository(Protocol): """原文仓储只能读取冻结线内的不可变来源引用。""" def read_source_refs( self, *, work_id: int, as_of: int, source_refs: Sequence[Mapping[str, Any]], ) -> list[dict[str, Any]]: """展开已校验的来源指针。""" def _load_search_cards() -> Callable[..., list[dict[str, Any]]]: """延迟导入 search skill，保持纯数据测试不触发嵌入依赖。""" search_scripts = pathlib.Path(__file__).resolve().parents[2] / "search" / "scripts" sys.path.insert(0, str(search_scripts)) from search import search_cards return search_cards def _chapter(value: Any, path: str) -> int: """严格解析正整数章号，拒绝 bool 和猜测性字符串。""" if isinstance(value, bool): raise RetrievalError(f"{path} 必须是正整数章号") if isinstance(value, str) and value.isdigit(): value = int(value) if not isinstance(value, int) or value <= 0: raise RetrievalError(f"{path} 必须是正整数章号") return value def _milestone_chapter(value: Mapping[str, Any]) -> int: """兼容实验库中英文里程碑章号字段，但不解析模糊文本。""" for key in ("chapter", "章", "chapterNo", "order_no"): if key in value: return _chapter(value[key], f"milestone.{key}") raise RetrievalError("卡里程碑缺少可证明的绝对章号") def stable_sort_cards(cards: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: """按合同固定卡片顺序，同分召回不会因数据库执行计划漂移。""" copied = [copy.deepcopy(dict(card)) for card in cards] try: return sorted( copied, key=lambda item: ( -float(item["score"]), str(item["sourceVersion"]), str(item["sourceId"]), int(item.get("sourceOffset") or 0), ), ) except (KeyError, TypeError, ValueError) as error: raise RetrievalError("卡缺少稳定排序字段") from error def _outline_elements(fine_outline: Mapping[str, Any]) -> list[dict[str, str]]: """按合同顺序提取需要检索的人物、关系、物品、地点和力量体系。""" groups = ( ("entities", None), ("relations", "character_relation"), ("items", "item"), ("locations", "location"), ("powerSystems", "power_system"), ("stateNeeds", "unknown"), ) result: list[dict[str, str]] = [] seen: set[tuple[str, str]] = set() for field, fallback_type in groups: values = fine_outline.get(field, []) if not isinstance(values, list): raise RetrievalError(f"fineOutline.{field} 必须是数组") for index, raw in enumerate(values): if isinstance(raw, str): item = {"id": f"{field}:{index}", "type": fallback_type or "unknown", "name": raw} elif isinstance(raw, Mapping): item = { "id": str(raw.get("id") or f"{field}:{index}"), "type": str(raw.get("type") or fallback_type or "unknown"), "name": str(raw.get("name") or ""), } else: raise RetrievalError(f"fineOutline.{field}[{index}] 类型非法") if not item["name"]: raise RetrievalError(f"fineOutline.{field}[{index}] 缺少 name") identity = (item["type"], item["name"]) if identity not in seen: seen.add(identity) result.append(item) return result def build_retrieval_plan( *, run_id: str, work_id: int, target_chapter: int, as_of: int, fine_outline: Mapping[str, Any], card_index_version: str, prose_index_version: str, token_budget: Mapping[str, int], top_k: int = 5, ) -> dict[str, Any]: """从已确认细纲确定查询集合，并在执行前冻结计划身份。""" target = _chapter(target_chapter, "targetChapter") freeze = _chapter(as_of, "asOf") if freeze >= target: raise RetrievalError("asOf 必须早于 targetChapter") if not isinstance(work_id, int) or isinstance(work_id, bool) or work_id <= 0: raise RetrievalError("workId 必须是正整数") if not isinstance(top_k, int) or isinstance(top_k, bool) or top_k <= 0: raise RetrievalError("topK 必须是正整数") hard_constraints = fine_outline.get("hardConstraints", []) if not isinstance(hard_constraints, list) or any(not isinstance(item, str) for item in hard_constraints): raise RetrievalError("fineOutline.hardConstraints 必须是字符串数组") suffix = " ".join(hard_constraints) queries = [ { "queryId": f"query-{index:03d}-{item['id']}", "text": normalize_text(f"{item['name']} {suffix}".strip()), "entityTypes": [item["type"]], "purpose": "fact_and_prose_evidence", "topK": top_k, } for index, item in enumerate(_outline_elements(fine_outline), 1) ] plan = { "planVersion": PLAN_VERSION, "runId": run_id, "asOf": freeze, "queries": queries, "cardIndexVersion": str(card_index_version), "proseIndexVersion": str(prose_index_version), "filters": { "workId": work_id, "asOfChapter": freeze, "sourceStatus": "active", "authorizationRequired": True, }, "tieBreak": TIE_BREAK, "tokenBudget": copy.deepcopy(dict(token_budget)), } plan["planId"] = retrieval_identity(plan) return plan def _validate_source_ref(ref: Mapping[str, Any], *, as_of: int, path: str) -> dict[str, Any]: """校验块级原文引用完整且不越过冻结线。""" required = {"sourceId", "sourceVersion", "chapter", "blockId", "startCodePoint", "endCodePoint"} if not isinstance(ref, Mapping) or not required.issubset(ref): raise RetrievalError(f"{path} 缺少块级来源定位") chapter = _chapter(ref["chapter"], f"{path}.chapter") if chapter > as_of: raise RetrievalError(f"{path} 包含目标章或未来章") start = ref["startCodePoint"] end = ref["endCodePoint"] if any(isinstance(value, bool) or not isinstance(value, int) for value in (ref["blockId"], start, end)): raise RetrievalError(f"{path} 的块或字符区间非法") if ref["blockId"] <= 0 or start < 0 or end <= start: raise RetrievalError(f"{path} 的块或字符区间非法") if not str(ref["sourceId"]).strip() or not str(ref["sourceVersion"]).strip(): raise RetrievalError(f"{path} 缺少来源 ID 或版本") return copy.deepcopy(dict(ref)) def freeze_card(card: Mapping[str, Any], *, as_of: int) -> dict[str, Any]: """仅用冻结线内里程碑重建卡状态，不透传终态摘要和未来字段。""" freeze = _chapter(as_of, "asOf") milestones = card.get("milestones") if not isinstance(milestones, list): raise RetrievalError(f"卡 {card.get('cardId')} 缺少历史里程碑") kept: list[dict[str, Any]] = [] for index, milestone in enumerate(milestones): if not isinstance(milestone, Mapping): raise RetrievalError(f"卡里程碑[{index}] 必须是对象") chapter = _milestone_chapter(milestone) if chapter <= freeze: normalized = copy.deepcopy(dict(milestone)) normalized["chapter"] = chapter for alias in ("章", "chapterNo", "order_no"): normalized.pop(alias, None) kept.append(normalized) kept.sort(key=lambda item: (item["chapter"], str(item.get("id") or ""))) if not kept: raise RetrievalError(f"卡 {card.get('cardId')} 在冻结线内没有可证明状态") refs = [ _validate_source_ref(ref, as_of=freeze, path=f"card.sourceRefs[{index}]") for index, ref in enumerate(card.get("sourceRefs") or []) ] return { "cardId": str(card.get("cardId") or ""), "type": str(card.get("type") or ""), "name": str(card.get("name") or ""), "score": float(card.get("score") or 0), "sourceId": str(card.get("sourceId") or ""), "sourceVersion": str(card.get("sourceVersion") or ""), "sourceOffset": int(card.get("sourceOffset") or 0), "sourceRefs": refs, "stateAsOf": kept, "sourceKind": str(card.get("sourceKind") or ""), "productionRetrievalEligible": bool(card.get("productionRetrievalEligible")), } class ProductionCardIndexRepository: """生产卡仓储固定调用 search.py 的 work 授权面。""" def __init__(self, *, search_function: Callable[..., list[dict[str, Any]]] | None = None): self._search = search_function or _load_search_cards() def search(self, plan: Mapping[str, Any]) -> list[dict[str, Any]]: """按计划逐查询召回，并再次失败关闭验证生产资格。""" cards: list[dict[str, Any]] = [] for query in plan.get("queries", []): entity_types = query.get("entityTypes") or [] card_type = entity_types[0] if len(entity_types) == 1 and entity_types[0] != "unknown" else None cards.extend( self._search( intent=query["text"], scope="work", work_id=plan["filters"]["workId"], ttype=card_type, purpose="generation", top=query["topK"], ) ) unique: dict[str, dict[str, Any]] = {} for card in stable_sort_cards(cards): if ( card.get("sourceKind") != "canonical_entity" or card.get("sourceStatus") not in {"active", "authorized"} or card.get("bindingStatus") != "active" or card.get("productionRetrievalEligible") is not True ): raise RetrievalError("生产检索命中非 active Canonical entity 或无效 binding") unique.setdefault(str(card.get("cardId")), copy.deepcopy(card)) return stable_sort_cards(list(unique.values())) class ReplayCardIndexRepository: """回放卡仓储只接受预注册 upgrade_book 评测投影。""" def __init__( self, cards: Sequence[Mapping[str, Any]], *, preregistered_card_ids: Sequence[str], _validated_replay: bool = False, ): if not _validated_replay: raise RetrievalError("回放仓储必须通过 from_replay_config 完成授权与泄露门禁") expected = {str(item) for item in preregistered_card_ids} actual = {str(item.get("cardId")) for item in cards} if not expected or expected != actual: raise RetrievalError("回放卡与预注册 ID 不一致") self._cards = [] for card in cards: if ( card.get("sourceType") != "upgrade_book" or card.get("evaluationStatus") != "eval_draft" or card.get("productionRetrievalEligible") is not False ): raise RetrievalError("回放只允许不可生产检索的 upgrade_book 评测卡") self._cards.append(copy.deepcopy(dict(card))) @classmethod def from_replay_config( cls, config: Mapping[str, Any], *, cards: Sequence[Mapping[str, Any]], preregistered_card_ids: Sequence[str], ) -> "ReplayCardIndexRepository": """复用 replay-eval 的授权、冻结来源和内容泄露审计后构造仓储。""" scripts = pathlib.Path(__file__).resolve().parents[2] / "replay-eval" / "scripts" sys.path.insert(0, str(scripts)) from audit_leakage import audit_snapshot from check_snapshot import check_authorization, check_target_sources if not isinstance(config, Mapping): raise RetrievalError("回放配置必须是对象") target = _chapter(config.get("targetChapter"), "config.targetChapter") snapshot = config.get("snapshot") if not isinstance(snapshot, Mapping): raise RetrievalError("回放配置缺少冻结快照") as_of = _chapter(snapshot.get("asOfChapter"), "config.snapshot.asOfChapter") if target != as_of + 1: raise RetrievalError("回放 targetChapter 必须等于 asOfChapter+1") authorization_result = check_authorization(config.get("authorization")) if not authorization_result.get("ok"): raise RetrievalError("回放授权快照未通过既有门禁") source_result = check_target_sources(target, config.get("sources", [])) if not source_result.get("ok"): raise RetrievalError("回放来源包含目标章、未来章或不可证明区间") leakage = config.get("leakageAudit") target_facts = leakage.get("targetFacts") if isinstance(leakage, Mapping) else None audit_result = audit_snapshot(snapshot, target_facts, as_of=as_of, target=target) if not audit_result.get("ok"): raise RetrievalError("回放冻结快照未通过既有内容泄露审计") return cls( cards, preregistered_card_ids=preregistered_card_ids, _validated_replay=True, ) def search(self, plan: Mapping[str, Any]) -> list[dict[str, Any]]: """返回预注册集合；计划不能把评测卡转换为生产卡。""" del plan return stable_sort_cards(self._cards) class FrozenProseRepository: """复用 load_reference_work 的唯一冻结原文 SQL 入口。""" def __init__(self, *, dsn: str, tenant_id: int): self.dsn = dsn self.tenant_id = tenant_id def read_source_refs(self, *, work_id: int, as_of: int, source_refs: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: """延迟导入 replay-eval，避免复制 SQL 或建立第二套权限语义。""" scripts = pathlib.Path(__file__).resolve().parents[2] / "replay-eval" / "scripts" sys.path.insert(0, str(scripts)) from load_reference_work import load_frozen_prose_rows return load_frozen_prose_rows( dsn=self.dsn, tenant_id=self.tenant_id, work_id=work_id, as_of=as_of, source_refs=source_refs, ) def _content_hash(text: str) -> str: """对规范化文本计算带算法前缀的内容哈希。""" normalized = normalize_text(text) return "sha256:" + hashlib.sha256(normalized.encode("utf-8")).hexdigest() def _authoritative_evidence(facts: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: """把正式设定、Canonical 状态和细纲新事实转换为独立事实证据。""" allowed = {"formal_setting", "canonical_state", "fine_outline_declared_new"} result: list[dict[str, Any]] = [] for index, raw in enumerate(facts): source_type = raw.get("sourceType") source_ref = raw.get("sourceRef") fact = normalize_text(str(raw.get("fact") or "")) if source_type not in allowed or not fact or not isinstance(source_ref, Mapping): raise RetrievalError(f"authoritativeFacts[{index}] 合同非法") if not source_ref.get("sourceId") or not source_ref.get("sourceVersion"): raise RetrievalError(f"authoritativeFacts[{index}] 缺少不可变来源引用") result.append( { "evidenceId": str(raw.get("factId") or f"authoritative:{index}"), "fact": fact, "sourceType": source_type, "sourceRef": copy.deepcopy(dict(source_ref)), "contentSha256": _content_hash(fact), "riskLevel": str(raw.get("riskLevel") or "medium"), } ) return result def retrieve_writer_sources( *, plan: Mapping[str, Any], card_repository: CardIndexRepository, prose_repository: ProseRepository, authoritative_facts: Sequence[Mapping[str, Any]] = (), ) -> dict[str, Any]: """执行固定计划，展开卡来源并形成事实/原文两条证据线。""" as_of = _chapter(plan.get("asOf"), "plan.asOf") work_id = plan.get("filters", {}).get("workId") if not isinstance(work_id, int) or isinstance(work_id, bool) or work_id <= 0: raise RetrievalError("plan.filters.workId 非法") cards = [freeze_card(card, as_of=as_of) for card in stable_sort_cards(card_repository.search(plan))] source_refs: list[dict[str, Any]] = [] unverified: list[dict[str, Any]] = [] for card_item in cards: if card_item["sourceRefs"]: source_refs.extend(card_item["sourceRefs"]) else: unverified.append( { "cardId": card_item["cardId"], "reason": "missing_source_refs", "classification": "unverifiedIndexHint", } ) deduplicated = { (str(ref["sourceVersion"]), str(ref["sourceId"]), int(ref["startCodePoint"])): ref for ref in source_refs } ordered_refs = [deduplicated[key] for key in sorted(deduplicated)] prose_rows = prose_repository.read_source_refs(work_id=work_id, as_of=as_of, source_refs=ordered_refs) if ordered_refs else [] def ref_key(ref: Mapping[str, Any]) -> tuple[str, str, int]: """同一块可有多个字符区间，映射键必须包含偏移。""" return (str(ref.get("sourceVersion")), str(ref.get("sourceId")), int(ref.get("startCodePoint") or 0)) rows_by_source = {ref_key(row.get("sourceRef", {})): row for row in prose_rows} missing = [ref["sourceId"] for ref in ordered_refs if ref_key(ref) not in rows_by_source] if missing: raise RetrievalError(f"卡来源未能回读原文: {','.join(map(str, missing))}") prose_evidence: list[dict[str, Any]] = [] for index, ref in enumerate(ordered_refs): row = rows_by_source[ref_key(ref)] text = normalize_text(str(row.get("text") or "")) if not text: raise RetrievalError(f"来源 {ref['sourceId']} 原文为空") prose_evidence.append( { "evidenceId": f"prose:card:{index:04d}", "chapter": _chapter(row.get("chapter"), "prose.chapter"), "sourceRef": copy.deepcopy(ref), "contentSha256": _content_hash(text), "purpose": str(row.get("purpose") or "card_source"), "text": text, "isRecentBaseline": False, } ) fact_evidence = _authoritative_evidence(authoritative_facts) for card_item in cards: if not card_item["sourceRefs"]: continue latest = card_item["stateAsOf"][-1] fact_text = normalize_text(str(latest.get("fact") or latest.get("台阶") or canonical_json(latest))) fact_evidence.append( { "evidenceId": f"fact:card:{card_item['cardId']}", "fact": fact_text, "sourceType": "historical_prose", "sourceRef": copy.deepcopy(card_item["sourceRefs"][0]), "contentSha256": _content_hash(fact_text), "riskLevel": "medium", } ) fact_evidence.sort(key=lambda item: item["evidenceId"]) sources = sorted( [copy.deepcopy(ref) for ref in ordered_refs] + [copy.deepcopy(item["sourceRef"]) for item in fact_evidence if item["sourceType"] != "historical_prose"], key=lambda item: (str(item["sourceVersion"]), str(item["sourceId"]), int(item.get("startCodePoint") or 0)), ) manifest_payload = { "manifestVersion": "writer-retrieval-manifest-v1", "planId": plan["planId"], "sources": sources, "omittedSources": [ {"sourceId": f"card:{item['cardId']}", "reason": item["reason"]} for item in unverified ], } manifest = {**manifest_payload, "manifestId": retrieval_identity(manifest_payload)} return { "cards": cards, "factEvidence": fact_evidence, "proseEvidence": prose_evidence, "unverifiedIndexHints": unverified, "manifest": manifest, } __all__ = [ "RetrievalError", "CardIndexRepository", "ProseRepository", "ProductionCardIndexRepository", "ReplayCardIndexRepository", "FrozenProseRepository", "stable_sort_cards", "freeze_card", "build_retrieval_plan", "retrieve_writer_sources", ]