460 lines
19 KiB
Python
460 lines
19 KiB
Python
#!/usr/bin/env python3
|
||
"""确定性组装 WriterContext v1 与可审阅 RetrievalManifest。
|
||
|
||
组装器不访问数据库、不调用模型。调用方必须先完成检索计划、卡索引和
|
||
sourceRefs 原文回读，再把冻结结果交给本模块。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
import hashlib
|
||
import json
|
||
from typing import Any, Mapping, Sequence
|
||
|
||
from writer_contract import (
|
||
MANIFEST_VERSION,
|
||
canonical_json,
|
||
normalize_text,
|
||
retrieval_identity,
|
||
validate_writer_context,
|
||
)
|
||
|
||
|
||
class AssemblyError(ValueError):
|
||
"""上下文不连续、预算不足或证据合同非法时抛出。"""
|
||
|
||
|
||
def _hash_text(text: str) -> str:
|
||
"""对已经归一化的证据文本计算带算法前缀的哈希。"""
|
||
|
||
return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def _source_key(source_ref: Mapping[str, Any]) -> tuple[str, str, int]:
|
||
"""来源版本、来源 ID 与偏移共同定义片段身份。"""
|
||
|
||
return (
|
||
str(source_ref.get("sourceVersion") or ""),
|
||
str(source_ref.get("sourceId") or ""),
|
||
int(source_ref.get("startCodePoint") or 0),
|
||
)
|
||
|
||
|
||
def _whole_source_key(source_ref: Mapping[str, Any]) -> tuple[str, str]:
|
||
"""同一块全文已存在时，卡片子区间不再重复注入。"""
|
||
|
||
return (str(source_ref.get("sourceVersion") or ""), str(source_ref.get("sourceId") or ""))
|
||
|
||
|
||
def _normalize_prose(raw: Mapping[str, Any], *, recent: bool, purpose: str | None = None) -> dict[str, Any]:
|
||
"""规范化单条原文证据，并机械复核内容哈希和字符区间。"""
|
||
|
||
required = {"chapter", "sourceRef", "text"}
|
||
if not isinstance(raw, Mapping) or not required.issubset(raw):
|
||
raise AssemblyError("原文证据缺少 chapter/sourceRef/text")
|
||
chapter = raw["chapter"]
|
||
if isinstance(chapter, bool) or not isinstance(chapter, int) or chapter <= 0:
|
||
raise AssemblyError("原文证据 chapter 必须是正整数")
|
||
source_ref = copy.deepcopy(dict(raw["sourceRef"])) if isinstance(raw["sourceRef"], Mapping) else None
|
||
if source_ref is None or not source_ref.get("sourceId") or not source_ref.get("sourceVersion"):
|
||
raise AssemblyError("原文证据缺少不可变来源引用")
|
||
text = normalize_text(str(raw["text"]))
|
||
if not text:
|
||
raise AssemblyError("原文证据不能为空")
|
||
source_ref["chapter"] = chapter
|
||
source_ref.setdefault("startCodePoint", 0)
|
||
source_ref.setdefault("endCodePoint", len(text))
|
||
if source_ref["endCodePoint"] > len(text) and source_ref["startCodePoint"] == 0:
|
||
raise AssemblyError("原文来源字符区间超过文本长度")
|
||
return {
|
||
"evidenceId": str(raw.get("evidenceId") or f"prose:{source_ref['sourceId']}:{source_ref['startCodePoint']}"),
|
||
"chapter": chapter,
|
||
"sourceRef": source_ref,
|
||
"contentSha256": _hash_text(text),
|
||
"purpose": purpose or str(raw.get("purpose") or "card_source"),
|
||
"text": text,
|
||
"isRecentBaseline": recent,
|
||
}
|
||
|
||
|
||
def _normalize_fact(raw: Mapping[str, Any]) -> dict[str, Any]:
|
||
"""规范化事实证据，并保留来源类型与风险优先级。"""
|
||
|
||
required = {"evidenceId", "fact", "sourceType", "sourceRef"}
|
||
if not isinstance(raw, Mapping) or not required.issubset(raw):
|
||
raise AssemblyError("事实证据缺少 evidenceId/fact/sourceType/sourceRef")
|
||
source_type = str(raw["sourceType"])
|
||
if source_type not in {"historical_prose", "formal_setting", "canonical_state", "fine_outline_declared_new"}:
|
||
raise AssemblyError("事实证据 sourceType 非法")
|
||
source_ref = copy.deepcopy(dict(raw["sourceRef"])) if isinstance(raw["sourceRef"], Mapping) else None
|
||
if source_ref is None or not source_ref.get("sourceId") or not source_ref.get("sourceVersion"):
|
||
raise AssemblyError("事实证据缺少不可变来源引用")
|
||
fact = normalize_text(str(raw["fact"]))
|
||
if not fact:
|
||
raise AssemblyError("事实证据内容不能为空")
|
||
risk = str(raw.get("riskLevel") or "medium")
|
||
if risk not in {"low", "medium", "high"}:
|
||
raise AssemblyError("事实证据 riskLevel 非法")
|
||
return {
|
||
"evidenceId": str(raw["evidenceId"]),
|
||
"fact": fact,
|
||
"sourceType": source_type,
|
||
"sourceRef": source_ref,
|
||
"contentSha256": _hash_text(fact),
|
||
"riskLevel": risk,
|
||
}
|
||
|
||
|
||
def _select_recent_baseline(recent_chapters: Sequence[Mapping[str, Any]], *, as_of: int) -> list[dict[str, Any]]:
|
||
"""选择冻结章起连续四章全文；作品不足四章时从第一章开始。"""
|
||
|
||
by_chapter: dict[int, dict[str, Any]] = {}
|
||
for raw in recent_chapters:
|
||
normalized = _normalize_prose(raw, recent=True, purpose="recent_full_chapter")
|
||
chapter = normalized["chapter"]
|
||
if chapter > as_of:
|
||
raise AssemblyError("近章原文包含目标章或未来章")
|
||
if chapter in by_chapter:
|
||
raise AssemblyError(f"第 {chapter} 章完整原文重复")
|
||
by_chapter[chapter] = normalized
|
||
first = max(1, as_of - 3)
|
||
expected = list(range(first, as_of + 1))
|
||
missing = [chapter for chapter in expected if chapter not in by_chapter]
|
||
if missing:
|
||
raise AssemblyError(f"连续四章基线缺章: {','.join(map(str, missing))}")
|
||
return [by_chapter[chapter] for chapter in expected]
|
||
|
||
|
||
def _outline_contract(fine_outline: Mapping[str, Any]) -> dict[str, Any]:
|
||
"""只保留 WriterContext 合同字段，检索用实体列表不会被倾倒给写手。"""
|
||
|
||
required = {"sourceRef", "hardConstraints", "adjustableBeats", "declaredNewFacts"}
|
||
if not isinstance(fine_outline, Mapping) or not required.issubset(fine_outline):
|
||
raise AssemblyError("fineOutline 缺少严格合同字段")
|
||
return {key: copy.deepcopy(fine_outline[key]) for key in ("sourceRef", "hardConstraints", "adjustableBeats", "declaredNewFacts")}
|
||
|
||
|
||
def _coverage_elements(fine_outline: Mapping[str, Any]) -> list[dict[str, str]]:
|
||
"""从细纲提取人物、关系、物品、地点和力量体系覆盖目标。"""
|
||
|
||
groups = (
|
||
("entities", None),
|
||
("relations", "character_relation"),
|
||
("items", "item"),
|
||
("locations", "location"),
|
||
("powerSystems", "power_system"),
|
||
)
|
||
result: list[dict[str, str]] = []
|
||
seen: set[str] = set()
|
||
for field, fallback_type in groups:
|
||
values = fine_outline.get(field, [])
|
||
if not isinstance(values, list):
|
||
raise AssemblyError(f"fineOutline.{field} 必须是数组")
|
||
for index, raw in enumerate(values):
|
||
if isinstance(raw, str):
|
||
item = {"id": f"{field}:{index}", "type": fallback_type or "unknown", "name": raw}
|
||
elif isinstance(raw, Mapping):
|
||
item = {
|
||
"id": str(raw.get("id") or f"{field}:{index}"),
|
||
"type": str(raw.get("type") or fallback_type or "unknown"),
|
||
"name": str(raw.get("name") or ""),
|
||
}
|
||
else:
|
||
raise AssemblyError(f"fineOutline.{field}[{index}] 类型非法")
|
||
if not item["name"]:
|
||
raise AssemblyError(f"fineOutline.{field}[{index}] 缺少名称")
|
||
if item["id"] not in seen:
|
||
seen.add(item["id"])
|
||
result.append(item)
|
||
return result
|
||
|
||
|
||
def _build_coverage(
|
||
fine_outline: Mapping[str, Any],
|
||
facts: Sequence[Mapping[str, Any]],
|
||
prose: Sequence[Mapping[str, Any]],
|
||
cards: Sequence[Mapping[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
"""把五类细纲要素映射到最终保留的事实与原文证据。"""
|
||
|
||
prose_by_source = {
|
||
str(item["sourceRef"]["sourceId"]): item["evidenceId"]
|
||
for item in prose
|
||
}
|
||
declared = fine_outline.get("declaredNewFacts", [])
|
||
result: list[dict[str, Any]] = []
|
||
for element in _coverage_elements(fine_outline):
|
||
fact_ids = [item["evidenceId"] for item in facts if element["name"] in item["fact"]]
|
||
prose_ids: list[str] = []
|
||
for card in cards:
|
||
if str(card.get("name")) != element["name"] and str(card.get("type")) != element["type"]:
|
||
continue
|
||
for ref in card.get("sourceRefs") or []:
|
||
evidence_id = prose_by_source.get(str(ref.get("sourceId")))
|
||
if evidence_id and evidence_id not in prose_ids:
|
||
prose_ids.append(evidence_id)
|
||
is_declared = any(
|
||
isinstance(item, Mapping)
|
||
and (str(item.get("factId")) == element["id"] or element["name"] in str(item.get("text") or ""))
|
||
for item in declared
|
||
)
|
||
if is_declared:
|
||
status, reason = "declared_new", ""
|
||
elif fact_ids and prose_ids:
|
||
status, reason = "supported", ""
|
||
elif fact_ids:
|
||
status, reason = "style_gap", "缺少历史表现原文"
|
||
elif prose_ids:
|
||
status, reason = "card_gap", "原文可证但缺少冻结事实索引"
|
||
else:
|
||
status, reason = "unsupported", "没有可信事实证据"
|
||
result.append(
|
||
{
|
||
"elementId": element["id"],
|
||
"elementType": element["type"],
|
||
"name": element["name"],
|
||
"status": status,
|
||
"factEvidenceIds": sorted(fact_ids),
|
||
"proseEvidenceIds": sorted(prose_ids),
|
||
"gapReason": reason,
|
||
}
|
||
)
|
||
return result
|
||
|
||
|
||
def _manifest(
|
||
*,
|
||
plan_id: str,
|
||
facts: Sequence[Mapping[str, Any]],
|
||
prose: Sequence[Mapping[str, Any]],
|
||
pattern_references: Sequence[Mapping[str, Any]],
|
||
omitted: Sequence[Mapping[str, str]],
|
||
) -> dict[str, Any]:
|
||
"""由最终入包来源集合计算稳定 manifest，不含 runId 或时间戳。"""
|
||
|
||
unique: dict[tuple[str, str, int], dict[str, Any]] = {}
|
||
for item in [*facts, *prose]:
|
||
ref = copy.deepcopy(dict(item["sourceRef"]))
|
||
unique[_source_key(ref)] = ref
|
||
for raw_ref in pattern_references:
|
||
ref = copy.deepcopy(dict(raw_ref))
|
||
unique[_source_key(ref)] = ref
|
||
sources = [unique[key] for key in sorted(unique)]
|
||
omitted_rows = sorted(
|
||
[copy.deepcopy(dict(item)) for item in omitted],
|
||
key=lambda item: (str(item.get("reason")), str(item.get("sourceId"))),
|
||
)
|
||
payload = {
|
||
"manifestVersion": MANIFEST_VERSION,
|
||
"planId": plan_id,
|
||
"sources": sources,
|
||
"omittedSources": omitted_rows,
|
||
}
|
||
return {**payload, "manifestId": retrieval_identity(payload)}
|
||
|
||
|
||
def _markdown_manifest(manifest: Mapping[str, Any]) -> str:
|
||
"""渲染不含运行时元数据的人类审阅清单，保证字节稳定。"""
|
||
|
||
lines = [
|
||
"# 正文检索清单",
|
||
"",
|
||
f"- 计划：`{manifest['planId']}`",
|
||
f"- 清单：`{manifest['manifestId']}`",
|
||
f"- 纳入来源：{len(manifest['sources'])}",
|
||
f"- 排除来源：{len(manifest['omittedSources'])}",
|
||
"",
|
||
"## 纳入来源",
|
||
"",
|
||
]
|
||
if manifest["sources"]:
|
||
for source in manifest["sources"]:
|
||
location = f"第{source['chapter']}章" if "chapter" in source else "权威版本"
|
||
lines.append(f"- `{source['sourceId']}` @ `{source['sourceVersion']}`，{location}")
|
||
else:
|
||
lines.append("- 无")
|
||
lines.extend(["", "## 排除来源", ""])
|
||
if manifest["omittedSources"]:
|
||
for source in manifest["omittedSources"]:
|
||
lines.append(f"- `{source['sourceId']}`：{source['reason']}")
|
||
else:
|
||
lines.append("- 无")
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
def _context_size(context: Mapping[str, Any]) -> int:
|
||
"""用最终规范 JSON 的 Unicode code point 数作为确定性预算单位。"""
|
||
|
||
return len(canonical_json(context))
|
||
|
||
|
||
def assemble_context(
|
||
*,
|
||
run_id: str,
|
||
attempt: int,
|
||
mode: str,
|
||
purpose: str,
|
||
quality_policy_version: str,
|
||
work_id: int,
|
||
target_chapter: int,
|
||
as_of: int,
|
||
source_version: str,
|
||
authorization_snapshot: Mapping[str, Any],
|
||
source_status: str,
|
||
retrieval_plan: Mapping[str, Any],
|
||
retrieval_result: Mapping[str, Any],
|
||
fine_outline: Mapping[str, Any],
|
||
narrative_state: Mapping[str, Any],
|
||
recent_chapters: Sequence[Mapping[str, Any]],
|
||
output_contract: Mapping[str, Any],
|
||
token_budget: Mapping[str, int],
|
||
pattern_references: Sequence[Mapping[str, Any]] = (),
|
||
generated_at: str,
|
||
) -> dict[str, Any]:
|
||
"""组装稳定 WriterContext，并返回规范 JSON 与 Markdown manifest。"""
|
||
|
||
if retrieval_plan.get("runId") != run_id or retrieval_plan.get("asOf") != as_of:
|
||
raise AssemblyError("retrievalPlan 与当前 runId/asOf 不一致")
|
||
if retrieval_plan.get("filters", {}).get("workId") != work_id:
|
||
raise AssemblyError("retrievalPlan 与当前 workId 不一致")
|
||
max_chars = token_budget.get("maxContextChars")
|
||
if isinstance(max_chars, bool) or not isinstance(max_chars, int) or max_chars <= 0:
|
||
raise AssemblyError("tokenBudget.maxContextChars 必须是正整数")
|
||
|
||
baseline = _select_recent_baseline(recent_chapters, as_of=as_of)
|
||
baseline_blocks = {_whole_source_key(item["sourceRef"]) for item in baseline}
|
||
supplemental: list[dict[str, Any]] = []
|
||
seen_fragments = {_source_key(item["sourceRef"]) for item in baseline}
|
||
for raw in retrieval_result.get("proseEvidence", []):
|
||
item = _normalize_prose(raw, recent=False)
|
||
if item["chapter"] > as_of:
|
||
raise AssemblyError("卡来源原文包含目标章或未来章")
|
||
if _whole_source_key(item["sourceRef"]) in baseline_blocks:
|
||
continue
|
||
key = _source_key(item["sourceRef"])
|
||
if key not in seen_fragments:
|
||
seen_fragments.add(key)
|
||
supplemental.append(item)
|
||
supplemental.sort(key=lambda item: _source_key(item["sourceRef"]))
|
||
facts = sorted(
|
||
[_normalize_fact(item) for item in retrieval_result.get("factEvidence", [])],
|
||
key=lambda item: ({"high": 0, "medium": 1, "low": 2}[item["riskLevel"]], item["evidenceId"]),
|
||
)
|
||
cards = [copy.deepcopy(dict(item)) for item in retrieval_result.get("cards", [])]
|
||
inherited_omitted = [
|
||
{"sourceId": str(item.get("sourceId") or "unknown"), "reason": str(item.get("reason") or "not_relevant")}
|
||
for item in retrieval_result.get("manifest", {}).get("omittedSources", [])
|
||
if isinstance(item, Mapping)
|
||
]
|
||
|
||
selected_facts: list[dict[str, Any]] = []
|
||
# 连续前四章全文是正文实验 v1 的不可裁剪基线。预算容不下时必须失败关闭，
|
||
# 不能静默退化成只保留最近一两章。
|
||
selected_prose: list[dict[str, Any]] = list(baseline)
|
||
omitted = list(inherited_omitted)
|
||
candidates: list[tuple[str, dict[str, Any]]] = []
|
||
candidates.extend(("fact", item) for item in facts if item["riskLevel"] == "high")
|
||
candidates.extend(("fact", item) for item in facts if item["riskLevel"] == "medium")
|
||
candidates.extend(("fact", item) for item in facts if item["riskLevel"] == "low")
|
||
candidates.extend(("prose", item) for item in supplemental)
|
||
|
||
def make_context() -> dict[str, Any]:
|
||
"""用当前选择集生成完整上下文，供预算试算与最终冻结。"""
|
||
|
||
ordered_prose = sorted(
|
||
selected_prose,
|
||
key=lambda item: (
|
||
0 if item["isRecentBaseline"] else 1,
|
||
item["chapter"] if item["isRecentBaseline"] else 0,
|
||
_source_key(item["sourceRef"]),
|
||
),
|
||
)
|
||
ordered_facts = sorted(selected_facts, key=lambda item: item["evidenceId"])
|
||
manifest = _manifest(
|
||
plan_id=str(retrieval_plan["planId"]),
|
||
facts=ordered_facts,
|
||
prose=ordered_prose,
|
||
pattern_references=pattern_references,
|
||
omitted=omitted,
|
||
)
|
||
context = {
|
||
"schemaVersion": "writer-context-v1",
|
||
"runId": run_id,
|
||
"attempt": attempt,
|
||
"mode": mode,
|
||
"purpose": purpose,
|
||
"qualityPolicyVersion": quality_policy_version,
|
||
"workId": work_id,
|
||
"targetChapter": target_chapter,
|
||
"asOf": as_of,
|
||
"contextSnapshot": {
|
||
"manifestId": manifest["manifestId"],
|
||
"contextSha256": "sha256:" + "0" * 64,
|
||
"generatedAt": normalize_text(generated_at),
|
||
},
|
||
"sourceVersion": source_version,
|
||
"authorizationSnapshot": copy.deepcopy(dict(authorization_snapshot)),
|
||
"sourceStatus": source_status,
|
||
"retrievalPlan": copy.deepcopy(dict(retrieval_plan)),
|
||
"retrievalManifest": manifest,
|
||
"fineOutline": _outline_contract(fine_outline),
|
||
"narrativeState": copy.deepcopy(dict(narrative_state)),
|
||
"factEvidence": ordered_facts,
|
||
"proseEvidence": ordered_prose,
|
||
"patternReferences": [copy.deepcopy(dict(item)) for item in pattern_references],
|
||
"evidenceCoverage": _build_coverage(fine_outline, ordered_facts, ordered_prose, cards),
|
||
"outputContract": copy.deepcopy(dict(output_contract)),
|
||
"tokenBudget": {"maxContextChars": max_chars, "usedContextChars": 0},
|
||
"omittedSources": sorted(copy.deepcopy(omitted), key=lambda item: (item["reason"], item["sourceId"])),
|
||
"acceptanceEligible": mode == "production" and purpose == "production",
|
||
}
|
||
# usedContextChars 自身位数会影响 JSON 长度，迭代到数值稳定。
|
||
for _ in range(8):
|
||
used = _context_size(context)
|
||
if context["tokenBudget"]["usedContextChars"] == used:
|
||
break
|
||
context["tokenBudget"]["usedContextChars"] = used
|
||
context["contextSnapshot"]["contextSha256"] = retrieval_identity(context)
|
||
return context
|
||
|
||
baseline_context = make_context()
|
||
if _context_size(baseline_context) > max_chars:
|
||
raise AssemblyError("上下文预算不足以容纳细纲硬约束与连续前四章全文基线")
|
||
for kind, item in candidates:
|
||
target = selected_facts if kind == "fact" else selected_prose
|
||
target.append(item)
|
||
trial = make_context()
|
||
if _context_size(trial) > max_chars:
|
||
target.pop()
|
||
omitted.append({"sourceId": str(item["sourceRef"]["sourceId"]), "reason": "token_budget"})
|
||
|
||
context = make_context()
|
||
# 添加裁剪回显可能占用少量预算；若越界，按最低优先级继续移除。
|
||
while _context_size(context) > max_chars and (selected_prose or selected_facts):
|
||
removable_prose = [item for item in selected_prose if not item["isRecentBaseline"]]
|
||
if removable_prose:
|
||
removed = removable_prose[-1]
|
||
selected_prose.remove(removed)
|
||
elif selected_facts and selected_facts[-1]["riskLevel"] != "high":
|
||
removed = selected_facts.pop()
|
||
elif selected_facts:
|
||
removed = selected_facts.pop()
|
||
else:
|
||
raise AssemblyError("上下文预算不足以保留连续前四章全文基线")
|
||
omitted.append({"sourceId": str(removed["sourceRef"]["sourceId"]), "reason": "token_budget"})
|
||
context = make_context()
|
||
if _context_size(context) > max_chars:
|
||
raise AssemblyError("上下文预算不足以保留可用证据")
|
||
context["tokenBudget"]["usedContextChars"] = _context_size(context)
|
||
context["contextSnapshot"]["contextSha256"] = retrieval_identity(context)
|
||
normalized_context = validate_writer_context(context)
|
||
return {
|
||
"context": normalized_context,
|
||
"contextJson": canonical_json(normalized_context),
|
||
"manifestMarkdown": _markdown_manifest(normalized_context["retrievalManifest"]),
|
||
}
|
||
|
||
|
||
__all__ = ["AssemblyError", "assemble_context"]
|