686aaaa421
冻结地图1二十分钟纵切版的平衡、证据与金标登记。 新增 ReferenceAsset/2 清单、策略、release、受信快照及 CLI/Service/acceptance provenance /4 消费链；保持 survivor live、R1 签名与部署关闭。
813 lines
32 KiB
Python
813 lines
32 KiB
Python
"""参照资产 v2 的受信 fd 消费门与双身份验证器。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import unicodedata
|
||
from collections.abc import Mapping, Sequence
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from types import MappingProxyType
|
||
|
||
try:
|
||
import artifact_snapshot
|
||
except ImportError: # pragma: no cover - 直接以 cheap-worker 目录为 cwd 时使用绝对导入
|
||
from . import artifact_snapshot
|
||
|
||
|
||
POLICY_ID = "survivor-gold-v1"
|
||
POLICY_RECORD_ID = "gac-shanhai-xingji"
|
||
POLICY_ROLE = "game_content_gold"
|
||
POLICY_CONSUMER_REF = "generation-runtime@reference-assets/2"
|
||
POLICY_MODE = "frozen_preflight"
|
||
VERIFIER_VERSION = "reference-asset-verifier/1.0.0"
|
||
TRUSTED_ROOT_ID = "wanxiang-reference-assets-root-v1"
|
||
SNAPSHOT_HASH_DOMAIN = b"reference-asset-consumption-snapshot/1\n"
|
||
|
||
MAX_MANIFEST_BYTES = 1 * 1024 * 1024
|
||
MAX_MANIFEST_ENTRIES = 512
|
||
MAX_FILE_BYTES = 16 * 1024 * 1024
|
||
MAX_RECORD_BYTES = 64 * 1024 * 1024
|
||
MAX_TOTAL_BYTES = 128 * 1024 * 1024
|
||
|
||
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||
_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9._-]*$")
|
||
_VERIFIER_RE = re.compile(r"^[A-Za-z][A-Za-z0-9._-]*/[0-9]+\.[0-9]+\.[0-9]+$")
|
||
_TRUSTED_ROOT_ID_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9._:-]*$")
|
||
_SIGNED_AT_RE = re.compile(
|
||
r"^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2})?(Z|[+-]\d{2}:?\d{2})?)?$"
|
||
)
|
||
|
||
_ALLOWED_ROLES = {
|
||
"harness_fixture",
|
||
"prompt_eval_gold",
|
||
"generation_exemplar",
|
||
"game_content_gold",
|
||
}
|
||
_ALLOWED_LIFECYCLE = {"candidate", "migration_pending", "active", "retired"}
|
||
_RECORD_KEYS = {
|
||
"schemaVersion",
|
||
"recordId",
|
||
"role",
|
||
"lifecycleStatus",
|
||
"assetRef",
|
||
"assetVersion",
|
||
"artifactHash",
|
||
"consumerRef",
|
||
"designRef",
|
||
"evidenceRefs",
|
||
"signedBy",
|
||
"signedAt",
|
||
"artifactRef",
|
||
"consumptionManifestRef",
|
||
"consumptionManifestHash",
|
||
}
|
||
_RELEASE_KEYS = {
|
||
"schemaVersion",
|
||
"releaseId",
|
||
"registryRef",
|
||
"registryHash",
|
||
"policyRef",
|
||
"policyHash",
|
||
"verifierVersion",
|
||
"trustedRootId",
|
||
}
|
||
_POLICY_KEYS = {
|
||
"schemaVersion",
|
||
"policyId",
|
||
"recordId",
|
||
"role",
|
||
"consumerRef",
|
||
"route",
|
||
"autoSelect",
|
||
"mode",
|
||
}
|
||
_MANIFEST_KEYS = {"schemaVersion", "manifestId", "canonicalization", "entries"}
|
||
_MANIFEST_ENTRY_KEYS = {"path", "size", "sha256"}
|
||
|
||
|
||
class ReferenceAssetGateError(ValueError):
|
||
"""稳定的可信消费错误；异常正文只含 code、recordId 和逻辑路径。"""
|
||
|
||
def __init__(self, code: str, record_id: str | None = None, logical_path: str | None = None) -> None:
|
||
self.code = code
|
||
self.record_id = record_id
|
||
self.logical_path = logical_path
|
||
fields = [f"code={code}"]
|
||
if record_id is not None:
|
||
fields.append(f"recordId={record_id}")
|
||
if logical_path is not None:
|
||
fields.append(f"path={logical_path}")
|
||
super().__init__(" ".join(fields))
|
||
|
||
|
||
# 为调用方保留两个直观别名，实际异常类型只有一套稳定字段。
|
||
ReferenceAssetError = ReferenceAssetGateError
|
||
ReferenceAssetVerificationError = ReferenceAssetGateError
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class VerifiedReferenceAssets:
|
||
"""验证成功后的只读消费闭包；失败时不会构造或返回该对象。"""
|
||
|
||
constraint_records: tuple[Mapping[str, object], ...]
|
||
files: Mapping[str, bytes]
|
||
receipts: tuple[Mapping[str, object], ...]
|
||
snapshot_hash: str
|
||
reference_roots: Mapping[str, tuple[str, ...]]
|
||
|
||
@property
|
||
def records(self) -> tuple[Mapping[str, object], ...]:
|
||
"""兼容调用方使用 records 读取已冻结的约束记录。"""
|
||
return self.constraint_records
|
||
|
||
@property
|
||
def canonical_snapshot_hash(self) -> str:
|
||
"""返回消费快照 canonical hash 的显式别名。"""
|
||
return self.snapshot_hash
|
||
|
||
@property
|
||
def reference_files(self) -> Mapping[str, bytes]:
|
||
"""返回 Task 4 Toolkit 使用的只读文件映射。"""
|
||
return self.files
|
||
|
||
@property
|
||
def protected_roots(self) -> Mapping[str, tuple[str, ...]]:
|
||
"""返回按 recordId 索引的受保护根集合。"""
|
||
return self.reference_roots
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class _StagedReferenceAssets:
|
||
"""单条记录的内部暂存结果；batch 全部成功前不暴露公开结果对象。"""
|
||
|
||
record: Mapping[str, object]
|
||
files: Mapping[str, bytes]
|
||
receipt: Mapping[str, object]
|
||
roots: tuple[str, ...]
|
||
|
||
|
||
def _freeze(value):
|
||
"""递归冻结记录、回执和根索引，防止调用方改写验证结果。"""
|
||
if isinstance(value, Mapping):
|
||
return MappingProxyType({key: _freeze(item) for key, item in value.items()})
|
||
if isinstance(value, (list, tuple)):
|
||
return tuple(_freeze(item) for item in value)
|
||
return value
|
||
|
||
|
||
def to_json_value(value):
|
||
"""把只读验证结果显式复制成普通 JSON 容器，供落盘和 schema 校验边界使用。"""
|
||
if isinstance(value, Mapping):
|
||
return {key: to_json_value(item) for key, item in value.items()}
|
||
if isinstance(value, tuple):
|
||
return [to_json_value(item) for item in value]
|
||
return value
|
||
|
||
|
||
def canonical_json_bytes(value: object) -> bytes:
|
||
"""生成 manifest 使用的 UTF-8 canonical JSON 原始字节。"""
|
||
return json.dumps(
|
||
value,
|
||
ensure_ascii=False,
|
||
sort_keys=True,
|
||
separators=(",", ":"),
|
||
allow_nan=False,
|
||
).encode("utf-8")
|
||
|
||
|
||
def snapshot_hash(files: Mapping[str, bytes]) -> str:
|
||
"""按路径 UTF-8 字节序计算消费快照 hash 向量。"""
|
||
return artifact_snapshot.consumption_snapshot_hash(files)
|
||
|
||
|
||
def canonical_snapshot_hash(files: Mapping[str, bytes]) -> str:
|
||
"""snapshot_hash 的公开语义别名，供跨语言向量测试调用。"""
|
||
return snapshot_hash(files)
|
||
|
||
|
||
def _fail(code: str, record_id: str | None = None, logical_path: str | None = None):
|
||
"""统一抛出稳定异常，禁止把底层系统错误和输入内容向外传播。"""
|
||
raise ReferenceAssetGateError(code, record_id, logical_path)
|
||
|
||
|
||
def _path(value, *, record_id: str | None = None) -> str:
|
||
"""校验仓根相对 NFC POSIX 路径；越界与其它路径语法错误分码。"""
|
||
if isinstance(value, Path):
|
||
value = value.as_posix()
|
||
if not isinstance(value, str) or not value or len(value) > 1024 or "\x00" in value:
|
||
_fail("reference_path_invalid", record_id, "<path>")
|
||
if value.startswith("/"):
|
||
_fail("reference_path_escape", record_id, "<absolute>")
|
||
if "\\" in value or unicodedata.normalize("NFC", value) != value:
|
||
_fail("reference_path_invalid", record_id, "<path>")
|
||
parts = value.split("/")
|
||
if any(part == ".." for part in parts):
|
||
_fail("reference_path_escape", record_id, "<path>")
|
||
if any(part in ("", ".") for part in parts):
|
||
_fail("reference_path_invalid", record_id, "<path>")
|
||
if any(ord(char) < 0x20 or ord(char) == 0x7F for char in value):
|
||
_fail("reference_path_invalid", record_id, "<path>")
|
||
return value
|
||
|
||
|
||
def _keys(value, allowed: set[str], *, record_id: str | None = None, code: str = "reference_registry_invalid") -> None:
|
||
"""执行 strict object key 检查，拒绝未知字段和非对象。"""
|
||
if not isinstance(value, dict) or set(value) != allowed:
|
||
_fail(code, record_id)
|
||
|
||
|
||
def _keys_optional(value, required: set[str], optional: set[str], *, record_id: str | None = None) -> None:
|
||
"""执行带 optional 字段的 strict object key 检查。"""
|
||
if not isinstance(value, dict) or not required.issubset(value) or set(value) - required - optional:
|
||
_fail("reference_registry_invalid", record_id)
|
||
|
||
|
||
def _is_sha256(value) -> bool:
|
||
"""判断是否为小写 64 位 SHA-256 字符串。"""
|
||
return isinstance(value, str) and _SHA256_RE.fullmatch(value) is not None
|
||
|
||
|
||
def _is_identifier(value) -> bool:
|
||
"""判断 schema 中的稳定标识符。"""
|
||
return isinstance(value, str) and _IDENTIFIER_RE.fullmatch(value) is not None
|
||
|
||
|
||
def _load_json(raw: bytes, *, code: str, record_id: str | None, logical_path: str):
|
||
"""只解析可信 fd 已读取的 UTF-8 JSON，并拒绝重复 object key。"""
|
||
def pairs(pairs_list):
|
||
result = {}
|
||
for key, value in pairs_list:
|
||
if key in result:
|
||
raise ValueError("duplicate-key")
|
||
result[key] = value
|
||
return result
|
||
|
||
try:
|
||
text = raw.decode("utf-8")
|
||
value = json.loads(text, object_pairs_hook=pairs, parse_constant=lambda _: (_ for _ in ()).throw(ValueError()))
|
||
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
|
||
_fail(code, record_id, logical_path)
|
||
if not isinstance(value, dict):
|
||
_fail(code, record_id, logical_path)
|
||
return value
|
||
|
||
|
||
def _read_files(root_fd: int, paths: Sequence[str], *, limits: Mapping[str, int], record_id: str | None,
|
||
missing_code: str | None = None) -> dict[str, bytes]:
|
||
"""通过 artifact_snapshot 的受信选择性读取器取回文件，统一异常类型。"""
|
||
try:
|
||
snapshot = artifact_snapshot.capture_selected_files(root_fd, paths, limits)
|
||
except artifact_snapshot.ArtifactSnapshotError as exc:
|
||
code = missing_code if missing_code is not None and exc.code == "reference_missing" else exc.code
|
||
_fail(code, record_id, exc.logical_path)
|
||
except (OSError, ValueError, TypeError):
|
||
_fail("reference_unreadable", record_id)
|
||
return dict(snapshot.files)
|
||
|
||
|
||
def _open_trusted_root(trusted_root) -> int:
|
||
"""只打开可信根一次，之后所有引用均通过该 fd 解析。"""
|
||
try:
|
||
root_fd, _ = artifact_snapshot._selected_root_fd(trusted_root)
|
||
return root_fd
|
||
except artifact_snapshot.ArtifactSnapshotError as exc:
|
||
_fail(exc.code, logical_path=exc.logical_path)
|
||
except (OSError, TypeError, ValueError):
|
||
_fail("reference_unreadable", logical_path="<root>")
|
||
|
||
|
||
def _read_json_file(root_fd: int, logical_path: str, *, record_id: str | None, limits: Mapping[str, int],
|
||
missing_code: str | None = None) -> tuple[dict, bytes, str]:
|
||
"""从同一可信根读取 JSON，并返回对象、原始字节和原始 SHA-256。"""
|
||
logical_path = _path(logical_path, record_id=record_id)
|
||
raw = _read_files(
|
||
root_fd,
|
||
[logical_path],
|
||
limits=limits,
|
||
record_id=record_id,
|
||
missing_code=missing_code,
|
||
)[logical_path]
|
||
value = _load_json(raw, code="reference_registry_invalid", record_id=record_id, logical_path=logical_path)
|
||
return value, raw, hashlib.sha256(raw).hexdigest()
|
||
|
||
|
||
def _validate_release(release: dict, *, record_id: str | None = None) -> None:
|
||
"""校验 release/1 的 strict 结构和引用字段。"""
|
||
_keys(release, _RELEASE_KEYS, record_id=record_id)
|
||
if release.get("schemaVersion") != "ReferenceAssetRelease/1":
|
||
_fail("reference_registry_invalid", record_id)
|
||
if not _is_identifier(release.get("releaseId")):
|
||
_fail("reference_registry_invalid", record_id)
|
||
for field in ("registryRef", "policyRef"):
|
||
_path(release.get(field), record_id=record_id)
|
||
for field in ("registryHash", "policyHash"):
|
||
if not _is_sha256(release.get(field)):
|
||
_fail("reference_registry_invalid", record_id)
|
||
if release.get("verifierVersion") != VERIFIER_VERSION:
|
||
_fail("reference_registry_invalid", record_id)
|
||
if release.get("trustedRootId") != TRUSTED_ROOT_ID:
|
||
_fail("reference_registry_invalid", record_id)
|
||
|
||
|
||
def _validate_nullable_string(value) -> bool:
|
||
"""检查允许 null 的非空字符串字段。"""
|
||
return value is None or (isinstance(value, str) and bool(value))
|
||
|
||
|
||
def _validate_record(record: dict, *, record_id: str | None = None) -> None:
|
||
"""校验 Record/2 字段、生命周期条件和所有仓内路径。"""
|
||
_keys_optional(
|
||
record,
|
||
{
|
||
"schemaVersion",
|
||
"recordId",
|
||
"role",
|
||
"lifecycleStatus",
|
||
"assetRef",
|
||
"assetVersion",
|
||
"artifactHash",
|
||
"evidenceRefs",
|
||
},
|
||
_RECORD_KEYS - {
|
||
"schemaVersion",
|
||
"recordId",
|
||
"role",
|
||
"lifecycleStatus",
|
||
"assetRef",
|
||
"assetVersion",
|
||
"artifactHash",
|
||
"evidenceRefs",
|
||
},
|
||
record_id=record_id,
|
||
)
|
||
current_id = record.get("recordId")
|
||
if record.get("schemaVersion") != "ReferenceAssetRecord/2" or not _is_identifier(current_id):
|
||
# 只有已经通过 identifier 校验的上下文 ID 才能进入异常，避免回显原始输入。
|
||
safe_record_id = current_id if _is_identifier(current_id) else (
|
||
record_id if _is_identifier(record_id) else None
|
||
)
|
||
_fail("reference_registry_invalid", safe_record_id)
|
||
if record.get("role") not in _ALLOWED_ROLES or record.get("lifecycleStatus") not in _ALLOWED_LIFECYCLE:
|
||
_fail("reference_registry_invalid", current_id)
|
||
_path(record.get("assetRef"), record_id=current_id)
|
||
if not isinstance(record.get("assetVersion"), str) or not record["assetVersion"]:
|
||
_fail("reference_registry_invalid", current_id)
|
||
if not _is_sha256(record.get("artifactHash")):
|
||
_fail("reference_registry_invalid", current_id)
|
||
for field in ("consumerRef", "signedBy"):
|
||
if not _validate_nullable_string(record.get(field)):
|
||
_fail("reference_registry_invalid", current_id)
|
||
signed_at = record.get("signedAt")
|
||
if signed_at is not None and (not isinstance(signed_at, str) or _SIGNED_AT_RE.fullmatch(signed_at) is None):
|
||
_fail("reference_registry_invalid", current_id)
|
||
design_ref = record.get("designRef")
|
||
if design_ref is not None:
|
||
if not isinstance(design_ref, list) or not design_ref or not all(
|
||
isinstance(value, str) and value for value in design_ref
|
||
):
|
||
_fail("reference_registry_invalid", current_id)
|
||
for value in design_ref:
|
||
_path(value, record_id=current_id)
|
||
evidence_refs = record.get("evidenceRefs")
|
||
if not isinstance(evidence_refs, list) or not all(isinstance(value, str) and value for value in evidence_refs):
|
||
_fail("reference_registry_invalid", current_id)
|
||
for value in evidence_refs:
|
||
_path(value, record_id=current_id)
|
||
for field in ("artifactRef", "consumptionManifestRef"):
|
||
value = record.get(field)
|
||
if value is not None:
|
||
_path(value, record_id=current_id)
|
||
if not (record.get("consumptionManifestHash") is None or _is_sha256(record.get("consumptionManifestHash"))):
|
||
_fail("reference_registry_invalid", current_id)
|
||
|
||
active = record["lifecycleStatus"] == "active"
|
||
historical = record["lifecycleStatus"] == "retired" and any(
|
||
record.get(field) is not None for field in ("consumerRef", "signedBy", "signedAt")
|
||
)
|
||
if active or historical:
|
||
required_identity = (
|
||
"consumerRef",
|
||
"signedBy",
|
||
"signedAt",
|
||
"artifactRef",
|
||
"consumptionManifestRef",
|
||
"consumptionManifestHash",
|
||
)
|
||
if any(record.get(field) in (None, "") for field in required_identity):
|
||
_fail("reference_registry_invalid", current_id)
|
||
if active and record["role"] == "game_content_gold":
|
||
if not isinstance(design_ref, list) or not design_ref:
|
||
_fail("reference_registry_invalid", current_id)
|
||
|
||
|
||
def _validate_registry(registry: dict) -> dict[str, dict]:
|
||
"""校验 Registry/2 的 strict 结构、recordId 唯一性和迁移语义。"""
|
||
_keys_optional(
|
||
registry,
|
||
{"schemaVersion", "registryVersion", "sourceOfTruth", "records"},
|
||
{"migrationNotes"},
|
||
)
|
||
if registry.get("schemaVersion") != "ReferenceAssetRegistry/2":
|
||
_fail("reference_registry_invalid")
|
||
if not isinstance(registry.get("registryVersion"), str) or not registry["registryVersion"]:
|
||
_fail("reference_registry_invalid")
|
||
if not isinstance(registry.get("sourceOfTruth"), str) or not registry["sourceOfTruth"]:
|
||
_fail("reference_registry_invalid")
|
||
records = registry.get("records")
|
||
if not isinstance(records, list):
|
||
_fail("reference_registry_invalid")
|
||
notes = registry.get("migrationNotes", {})
|
||
if not isinstance(notes, dict) or any(
|
||
not _is_identifier(key) or not isinstance(value, str) or not value for key, value in notes.items()
|
||
):
|
||
_fail("reference_registry_invalid")
|
||
by_id: dict[str, dict] = {}
|
||
for record in records:
|
||
_validate_record(record)
|
||
current_id = record["recordId"]
|
||
if current_id in by_id:
|
||
_fail("reference_registry_invalid", current_id)
|
||
by_id[current_id] = record
|
||
for key in notes:
|
||
if key != "_registry" and key not in by_id:
|
||
_fail("reference_registry_invalid", key)
|
||
if registry["registryVersion"].endswith(".migration-list") and any(
|
||
record.get("lifecycleStatus") == "active" for record in records
|
||
):
|
||
_fail("reference_registry_invalid")
|
||
return by_id
|
||
|
||
|
||
def _validate_policy(policy: dict, requested_policy_id: str, requested_mode: str) -> None:
|
||
"""校验 release 绑定 policy；当前冻结策略另执行固定身份闭包。"""
|
||
_keys(policy, _POLICY_KEYS, code="reference_policy_missing")
|
||
if policy.get("schemaVersion") != "ReferenceAssetConsumptionPolicy/1":
|
||
_fail("reference_policy_missing")
|
||
if policy.get("policyId") != requested_policy_id or not _is_identifier(policy.get("policyId")):
|
||
_fail("reference_policy_missing")
|
||
if not _is_identifier(policy.get("recordId")) or policy.get("role") not in _ALLOWED_ROLES:
|
||
_fail("reference_policy_missing")
|
||
if not isinstance(policy.get("consumerRef"), str) or not policy["consumerRef"]:
|
||
_fail("reference_policy_missing")
|
||
if not isinstance(policy.get("route"), str) or not policy["route"] or policy.get("autoSelect") is not False:
|
||
_fail("reference_policy_missing")
|
||
if policy.get("mode") != POLICY_MODE:
|
||
_fail("reference_declaration_mismatch", policy["recordId"])
|
||
if requested_policy_id == POLICY_ID:
|
||
fixed = {
|
||
"recordId": POLICY_RECORD_ID,
|
||
"role": POLICY_ROLE,
|
||
"consumerRef": POLICY_CONSUMER_REF,
|
||
"route": "survivor-gold",
|
||
}
|
||
if any(policy.get(field) != expected for field, expected in fixed.items()):
|
||
_fail("reference_policy_missing")
|
||
if requested_mode != policy["mode"]:
|
||
_fail("reference_declaration_mismatch", policy["recordId"])
|
||
|
||
|
||
def _declarations_match(declarations, record: dict, policy: dict) -> bool:
|
||
"""把调用方声明投影成 recordId/role/consumerRef 三元组后逐项对账。"""
|
||
if declarations is None:
|
||
return True
|
||
if isinstance(declarations, Mapping):
|
||
record_ids = declarations.get("referenceAssetRecordIds", declarations.get("recordIds"))
|
||
if record_ids is None and "recordId" in declarations:
|
||
record_ids = [declarations.get("recordId")]
|
||
consumer_ref = declarations.get("consumerRef")
|
||
role = declarations.get("role")
|
||
elif isinstance(declarations, Sequence) and not isinstance(declarations, (str, bytes, bytearray)):
|
||
record_ids = list(declarations)
|
||
consumer_ref = None
|
||
role = None
|
||
else:
|
||
return False
|
||
if record_ids != [record["recordId"]] or consumer_ref not in (None, policy["consumerRef"]):
|
||
return False
|
||
return role in (None, policy["role"])
|
||
|
||
|
||
def _under(path: str, root: str) -> bool:
|
||
"""判断仓内路径是否位于目录根或等于批准的文件引用。"""
|
||
return path == root or path.startswith(root + "/")
|
||
|
||
|
||
def _validate_manifest(manifest: dict, raw: bytes, *, record_id: str, allowed_root: str,
|
||
design_refs: Sequence[str]) -> list[dict]:
|
||
"""校验 manifest canonical 字节、排序、路径范围和单记录资源预算。"""
|
||
if len(raw) > MAX_MANIFEST_BYTES:
|
||
_fail("reference_oversize", record_id)
|
||
if raw != canonical_json_bytes(manifest):
|
||
_fail("reference_manifest_hash_mismatch", record_id)
|
||
_keys(manifest, _MANIFEST_KEYS, record_id=record_id, code="reference_manifest_hash_mismatch")
|
||
if manifest.get("schemaVersion") != "ReferenceAssetConsumptionManifest/1":
|
||
_fail("reference_manifest_hash_mismatch", record_id)
|
||
if not _is_identifier(manifest.get("manifestId")):
|
||
_fail("reference_manifest_hash_mismatch", record_id)
|
||
if manifest.get("canonicalization") != "reference-asset-consumption-manifest/1":
|
||
_fail("reference_manifest_hash_mismatch", record_id)
|
||
entries = manifest.get("entries")
|
||
if not isinstance(entries, list) or not entries:
|
||
_fail("reference_manifest_hash_mismatch", record_id)
|
||
if len(entries) > MAX_MANIFEST_ENTRIES:
|
||
_fail("reference_oversize", record_id)
|
||
|
||
normalized_paths: list[str] = []
|
||
seen: set[str] = set()
|
||
total_declared = 0
|
||
for entry in entries:
|
||
_keys(entry, _MANIFEST_ENTRY_KEYS, record_id=record_id, code="reference_manifest_hash_mismatch")
|
||
path = _path(entry.get("path"), record_id=record_id)
|
||
if path in seen:
|
||
_fail("reference_path_invalid", record_id, path)
|
||
seen.add(path)
|
||
normalized_paths.append(path)
|
||
size = entry.get("size")
|
||
if not isinstance(size, int) or isinstance(size, bool) or size < 0 or size > 9007199254740991:
|
||
_fail("reference_manifest_hash_mismatch", record_id, path)
|
||
if not _is_sha256(entry.get("sha256")):
|
||
_fail("reference_manifest_hash_mismatch", record_id, path)
|
||
if size > MAX_FILE_BYTES:
|
||
_fail("reference_oversize", record_id, path)
|
||
total_declared += size
|
||
if total_declared > MAX_RECORD_BYTES:
|
||
_fail("reference_oversize", record_id, path)
|
||
if not _under(path, allowed_root) and path not in design_refs:
|
||
_fail("reference_path_escape", record_id, path)
|
||
if normalized_paths != sorted(normalized_paths, key=lambda value: value.encode("utf-8")):
|
||
_fail("reference_path_invalid", record_id)
|
||
return entries
|
||
|
||
|
||
def _stage_one_policy(
|
||
policy_id: str,
|
||
*,
|
||
release_ref: str,
|
||
expected_release_hash: str,
|
||
trusted_root,
|
||
mode: str,
|
||
declarations=None,
|
||
) -> _StagedReferenceAssets:
|
||
"""在已锚定根 fd 上完成单条 policy 验证，仅返回 batch 内部暂存值。"""
|
||
if not _is_sha256(expected_release_hash):
|
||
_fail("reference_registry_untrusted")
|
||
|
||
release_ref = _path(release_ref)
|
||
root_fd = _open_trusted_root(trusted_root)
|
||
try:
|
||
release, _, release_hash = _read_json_file(
|
||
root_fd,
|
||
release_ref,
|
||
record_id=None,
|
||
limits={"max_files": 1, "max_file_bytes": MAX_MANIFEST_BYTES, "max_record_bytes": MAX_MANIFEST_BYTES},
|
||
)
|
||
if release_hash != expected_release_hash:
|
||
_fail("reference_registry_untrusted", logical_path=release_ref)
|
||
_validate_release(release)
|
||
|
||
registry_ref = _path(release["registryRef"])
|
||
registry, _, registry_hash = _read_json_file(
|
||
root_fd,
|
||
registry_ref,
|
||
record_id=None,
|
||
limits={"max_files": 1, "max_file_bytes": MAX_RECORD_BYTES, "max_record_bytes": MAX_RECORD_BYTES},
|
||
)
|
||
if registry_hash != release["registryHash"]:
|
||
_fail("reference_registry_untrusted", logical_path=registry_ref)
|
||
by_id = _validate_registry(registry)
|
||
|
||
policy_ref = _path(release["policyRef"])
|
||
try:
|
||
policy, _, policy_hash = _read_json_file(
|
||
root_fd,
|
||
policy_ref,
|
||
record_id=None,
|
||
limits={"max_files": 1, "max_file_bytes": MAX_MANIFEST_BYTES, "max_record_bytes": MAX_MANIFEST_BYTES},
|
||
missing_code="reference_policy_missing",
|
||
)
|
||
except ReferenceAssetGateError as exc:
|
||
if exc.code == "reference_registry_invalid":
|
||
_fail("reference_policy_missing", logical_path=policy_ref)
|
||
raise
|
||
if policy_hash != release["policyHash"]:
|
||
_fail("reference_registry_untrusted", logical_path=policy_ref)
|
||
_validate_policy(policy, policy_id, mode)
|
||
|
||
record = by_id.get(policy["recordId"])
|
||
if record is None:
|
||
_fail("reference_registry_invalid", policy["recordId"])
|
||
if record.get("lifecycleStatus") != "active":
|
||
_fail("reference_registry_invalid", record["recordId"])
|
||
if record.get("role") != policy["role"] or record.get("consumerRef") != policy["consumerRef"]:
|
||
_fail("reference_declaration_mismatch", record["recordId"])
|
||
if not _declarations_match(declarations, record, policy):
|
||
_fail("reference_declaration_mismatch", record["recordId"])
|
||
|
||
record_id = record["recordId"]
|
||
asset_root = _path(record["assetRef"], record_id=record_id)
|
||
artifact_ref = _path(record.get("artifactRef"), record_id=record_id)
|
||
manifest_ref = _path(record.get("consumptionManifestRef"), record_id=record_id)
|
||
if not _under(artifact_ref, asset_root) or not _under(manifest_ref, asset_root):
|
||
_fail("reference_path_escape", record_id)
|
||
design_refs = tuple(record.get("designRef") or ())
|
||
|
||
# 先读取并核验 bundle；manifest 使用独立的 1 MiB 读前预算，避免把超限清单
|
||
# 先读入内存。两次读取仍共用同一个 trusted root fd 和逐级 O_NOFOLLOW 边界。
|
||
artifact_files = _read_files(
|
||
root_fd,
|
||
[artifact_ref],
|
||
limits={
|
||
"max_files": 1,
|
||
"max_file_bytes": MAX_FILE_BYTES,
|
||
"max_record_bytes": MAX_RECORD_BYTES,
|
||
},
|
||
record_id=record_id,
|
||
)
|
||
observed_artifact_hash = hashlib.sha256(artifact_files[artifact_ref]).hexdigest()
|
||
if observed_artifact_hash != record["artifactHash"]:
|
||
_fail("reference_artifact_hash_mismatch", record_id, artifact_ref)
|
||
manifest_files = _read_files(
|
||
root_fd,
|
||
[manifest_ref],
|
||
limits={
|
||
"max_files": 1,
|
||
"max_file_bytes": MAX_MANIFEST_BYTES,
|
||
"max_record_bytes": MAX_MANIFEST_BYTES,
|
||
},
|
||
record_id=record_id,
|
||
)
|
||
manifest_raw = manifest_files[manifest_ref]
|
||
observed_manifest_hash = hashlib.sha256(manifest_raw).hexdigest()
|
||
if observed_manifest_hash != record["consumptionManifestHash"]:
|
||
_fail("reference_manifest_hash_mismatch", record_id, manifest_ref)
|
||
manifest = _load_json(
|
||
manifest_raw,
|
||
code="reference_manifest_hash_mismatch",
|
||
record_id=record_id,
|
||
logical_path=manifest_ref,
|
||
)
|
||
entries = _validate_manifest(
|
||
manifest,
|
||
manifest_raw,
|
||
record_id=record_id,
|
||
allowed_root=asset_root,
|
||
design_refs=design_refs,
|
||
)
|
||
entry_paths = [entry["path"] for entry in entries]
|
||
selected = _read_files(
|
||
root_fd,
|
||
entry_paths,
|
||
limits={
|
||
"max_files": MAX_MANIFEST_ENTRIES,
|
||
"max_file_bytes": MAX_FILE_BYTES,
|
||
"max_record_bytes": MAX_RECORD_BYTES,
|
||
},
|
||
record_id=record_id,
|
||
)
|
||
for entry in entries:
|
||
path = entry["path"]
|
||
content = selected[path]
|
||
if len(content) != entry["size"] or hashlib.sha256(content).hexdigest() != entry["sha256"]:
|
||
_fail("reference_entry_hash_mismatch", record_id, path)
|
||
|
||
immutable_files = MappingProxyType(dict(selected))
|
||
receipt = {
|
||
"schemaVersion": "ReferenceAssetVerificationReceipt/1",
|
||
"receiptId": f"receipt-{policy_id}-{record_id}-{observed_manifest_hash[:12]}",
|
||
"releaseRef": release_ref,
|
||
"registryVersion": registry["registryVersion"],
|
||
"expectedRegistryHash": release["registryHash"],
|
||
"observedRegistryHash": registry_hash,
|
||
"policyHash": policy_hash,
|
||
"verifierVersion": release["verifierVersion"],
|
||
"trustedRootId": release["trustedRootId"],
|
||
"recordId": record_id,
|
||
"role": record["role"],
|
||
"consumerRef": record["consumerRef"],
|
||
"artifactRef": artifact_ref,
|
||
"consumptionManifestRef": manifest_ref,
|
||
"expected": {
|
||
"artifactHash": record["artifactHash"],
|
||
"consumptionManifestHash": record["consumptionManifestHash"],
|
||
},
|
||
"observed": {
|
||
"artifactHash": observed_artifact_hash,
|
||
"consumptionManifestHash": observed_manifest_hash,
|
||
},
|
||
"finalSnapshotHash": snapshot_hash(immutable_files),
|
||
}
|
||
return _StagedReferenceAssets(
|
||
record=dict(record),
|
||
files=immutable_files,
|
||
receipt=receipt,
|
||
roots=tuple((asset_root, *design_refs)),
|
||
)
|
||
finally:
|
||
os.close(root_fd)
|
||
|
||
|
||
def verify_policy(
|
||
policy_id: str,
|
||
*,
|
||
release_ref: str,
|
||
expected_release_hash: str,
|
||
trusted_root,
|
||
mode: str,
|
||
declarations=None,
|
||
) -> VerifiedReferenceAssets:
|
||
"""验证当前唯一受批准 policy，并委托单元素原子 batch。"""
|
||
if policy_id != POLICY_ID:
|
||
_fail("reference_policy_missing")
|
||
if mode != POLICY_MODE:
|
||
_fail("reference_declaration_mismatch")
|
||
return verify_policies([{
|
||
"policy_id": policy_id,
|
||
"release_ref": release_ref,
|
||
"expected_release_hash": expected_release_hash,
|
||
"trusted_root": trusted_root,
|
||
"mode": mode,
|
||
"declarations": declarations,
|
||
}])
|
||
|
||
|
||
def verify_policies(requests: Sequence[Mapping[str, object]]) -> VerifiedReferenceAssets:
|
||
"""原子验证并合并多条 policy，任一失败都不会构造公开结果。"""
|
||
if not isinstance(requests, Sequence) or isinstance(requests, (str, bytes, bytearray)) or not requests:
|
||
_fail("reference_policy_missing")
|
||
|
||
required_keys = {
|
||
"policy_id",
|
||
"release_ref",
|
||
"expected_release_hash",
|
||
"trusted_root",
|
||
"mode",
|
||
}
|
||
allowed_keys = required_keys | {"declarations"}
|
||
merged_files: dict[str, bytes] = {}
|
||
merged_records: list[Mapping[str, object]] = []
|
||
merged_receipts: list[Mapping[str, object]] = []
|
||
merged_roots: dict[str, tuple[str, ...]] = {}
|
||
total_bytes = 0
|
||
|
||
for request in requests:
|
||
if not isinstance(request, Mapping) or not required_keys.issubset(request) or set(request) - allowed_keys:
|
||
_fail("reference_declaration_mismatch")
|
||
item = _stage_one_policy(
|
||
request["policy_id"],
|
||
release_ref=request["release_ref"],
|
||
expected_release_hash=request["expected_release_hash"],
|
||
trusted_root=request["trusted_root"],
|
||
mode=request["mode"],
|
||
declarations=request.get("declarations"),
|
||
)
|
||
record_id = item.record["recordId"]
|
||
if record_id in merged_roots:
|
||
_fail("reference_registry_invalid", record_id)
|
||
for path, content in item.files.items():
|
||
existing = merged_files.get(path)
|
||
if existing is not None:
|
||
if existing != content:
|
||
_fail("reference_entry_hash_mismatch", record_id, path)
|
||
continue
|
||
total_bytes += len(content)
|
||
if total_bytes > MAX_TOTAL_BYTES:
|
||
_fail("reference_oversize", record_id, path)
|
||
merged_files[path] = content
|
||
# 记录和回执在当前 stage 合并成功后立即冻结，避免积存可变暂存对象。
|
||
merged_records.append(_freeze(item.record))
|
||
merged_receipts.append(_freeze(item.receipt))
|
||
merged_roots[record_id] = tuple(item.roots)
|
||
|
||
immutable_files = MappingProxyType(dict(merged_files))
|
||
return VerifiedReferenceAssets(
|
||
constraint_records=tuple(merged_records),
|
||
files=immutable_files,
|
||
receipts=tuple(merged_receipts),
|
||
snapshot_hash=snapshot_hash(immutable_files),
|
||
reference_roots=_freeze(merged_roots),
|
||
)
|
||
|
||
|
||
__all__ = [
|
||
"MAX_FILE_BYTES",
|
||
"MAX_MANIFEST_BYTES",
|
||
"MAX_MANIFEST_ENTRIES",
|
||
"MAX_RECORD_BYTES",
|
||
"MAX_TOTAL_BYTES",
|
||
"POLICY_ID",
|
||
"ReferenceAssetError",
|
||
"ReferenceAssetGateError",
|
||
"ReferenceAssetVerificationError",
|
||
"VerifiedReferenceAssets",
|
||
"canonical_json_bytes",
|
||
"canonical_snapshot_hash",
|
||
"snapshot_hash",
|
||
"to_json_value",
|
||
"verify_policy",
|
||
"verify_policies",
|
||
]
|