"""游戏包对象存储工具。 本模块只处理平台侧的分类、哈希、manifest 和 MinIO/S3 读写；具体游戏目录只是一次性上传输入， 不是生产消费来源。实现使用标准库完成本地快照，只有真正执行对象存储读写时才惰性导入 minio， 让契约测试和离线 manifest 构建不依赖网络或第三方 SDK。 """ from __future__ import annotations import hashlib import json import mimetypes import os import posixpath import re import shutil import tempfile import unicodedata from io import BytesIO from pathlib import Path from typing import Iterable, Mapping SCHEMA_VERSION = "GameArtifactManifest/1" _ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") _DB_ID_RE = re.compile(r"^[1-9][0-9]*$") _HASH_RE = re.compile(r"^[a-f0-9]{64}$") _ARTIFACT_MANIFEST_KEY_RE = re.compile( r"^tenants/[A-Za-z0-9][A-Za-z0-9._-]*/games/[1-9][0-9]*/" r"versions/[1-9][0-9]*/artifact-manifest\.json$" ) _CATEGORY_ORDER = ("asset", "runtime", "evidence") _CATEGORY_DIR = {"asset": "assets", "runtime": "runtime", "evidence": "evidence"} _CATEGORY_STORE = {"asset": "artifact", "runtime": "artifact", "evidence": "evidence"} _TRANSIENT_PARTS = frozenset({ ".git", ".idea", "node_modules", "__pycache__", ".pytest_cache", "target", "coverage", ".DS_Store", "_store", "_cheap-sessions", "_tier2-gen", }) _SECRET_NAMES = frozenset({ ".env", ".env.local", ".env.production", "credentials.json", "service-account.json", }) _MANIFEST_HASH_DOMAIN = b"GameArtifactManifest/1\0" _TREE_HASH_DOMAIN = b"GameArtifactTreeHash/1\0" _SOURCE_TREE_HASH_DOMAIN = b"GameSourceArchiveTreeHash/1\0" _MAX_OBJECTS = 10_000 _MAX_PATH_BYTES = 1_024 _MAX_OBJECT_BYTES = 100 * 1024 * 1024 _MAX_RUNTIME_ASSET_BYTES = 10 * 1024 * 1024 _MAX_MANIFEST_BYTES = 8 * 1024 * 1024 _REMOTE_READ_CHUNK_BYTES = 1024 * 1024 class ArtifactError(ValueError): """对象包不满足安全或一致性约束时抛出的稳定异常。""" def _validate_id(value: str, label: str) -> str: """校验 gameId/versionId，避免对象 key 被注入路径片段。""" if not isinstance(value, str) or not _ID_RE.fullmatch(value): raise ArtifactError(f"{label} 非法:{value!r}") return value def _validate_db_id(value: str, label: str) -> str: """生产 game/version 身份必须来自数据库正整数主键，禁止 Tier2 slug 冒充。""" if not isinstance(value, str) or not _DB_ID_RE.fullmatch(value): raise ArtifactError(f"{label} 必须是数据库正整数 ID:{value!r}") return value def _safe_rel(path: str) -> str: """把本地相对路径规范化为 POSIX，并拒绝绝对路径、父级跳转和空路径。""" normalized = unicodedata.normalize("NFC", posixpath.normpath(path.replace(os.sep, "/"))) if not normalized or normalized in (".", "..") or normalized.startswith("../") or normalized.startswith("/"): raise ArtifactError(f"对象相对路径非法:{path!r}") if any(part in ("", ".", "..") for part in normalized.split("/")): raise ArtifactError(f"对象相对路径含危险片段:{path!r}") if len(normalized.encode("utf-8")) > _MAX_PATH_BYTES: raise ArtifactError(f"对象相对路径超过 {_MAX_PATH_BYTES} 字节:{path!r}") return normalized def _canonical_json(value: Mapping) -> bytes: """使用无空格、排序键的 JSON 字节，保证跨语言 manifest hash 可复算。""" return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") def _hash_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def _file_digest(path: Path) -> tuple[int, str]: """流式计算本地文件大小与 SHA-256，避免把大文件整体读入内存。""" digest = hashlib.sha256() size = 0 with path.open("rb") as stream: while chunk := stream.read(1024 * 1024): size += len(chunk) if size > _MAX_OBJECT_BYTES: raise ArtifactError(f"单文件超过 {_MAX_OBJECT_BYTES} 字节:{path.name}") digest.update(chunk) return size, digest.hexdigest() def _source_project_hash(files: Iterable[tuple[str, Path]]) -> tuple[int, int, str]: """按 SourceProjectStore 的 path\0content\0 口径复算源工程 contentHash。""" rows = sorted(files, key=lambda item: item[0]) digest = hashlib.sha256() total_bytes = 0 for rel, path in rows: data = path.read_bytes() if len(data) > _MAX_OBJECT_BYTES: raise ArtifactError(f"源文件超过 {_MAX_OBJECT_BYTES} 字节:{rel}") try: content = data.decode("utf-8") except UnicodeDecodeError as exc: raise ArtifactError(f"源工程文件必须是 UTF-8 文本:{rel}") from exc digest.update((rel + "\0").encode("utf-8")) digest.update((content + "\0").encode("utf-8")) total_bytes += len(data) if not rows: raise ArtifactError("游戏包缺少源工程文件") return len(rows), total_bytes, digest.hexdigest() def _source_archive_rows(files: Iterable[tuple[str, Path]]) -> list[dict]: """按二进制内容构造引擎无关源归档的稳定文件行。""" rows: list[dict] = [] seen: set[str] = set() normalized_files = [(_safe_rel(rel), path) for rel, path in files] for normalized, path in sorted(normalized_files, key=lambda item: item[0]): if normalized in seen: raise ArtifactError(f"源归档存在重复路径:{normalized}") seen.add(normalized) size, sha256 = _file_digest(path) rows.append({"path": normalized, "bytes": size, "sha256": sha256}) if not rows: raise ArtifactError("游戏包缺少源工程文件") if len(rows) > _MAX_OBJECTS: raise ArtifactError(f"源归档对象数量超过 {_MAX_OBJECTS}") return rows def _source_archive_hash(files: Iterable[tuple[str, Path]]) -> tuple[int, int, str]: """计算 GameSourceArchive/1 的二进制安全源工程 hash。""" rows = _source_archive_rows(files) total_bytes = sum(item["bytes"] for item in rows) digest = _hash_bytes(_SOURCE_TREE_HASH_DOMAIN + _canonical_json(rows)) return len(rows), total_bytes, digest def _contract_schema(name: str) -> dict: """从仓内契约 SoT 读取 JSON Schema。""" path = Path(__file__).resolve().parents[3] / "contracts" / name try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise ArtifactError(f"读取契约失败:{name}:{exc}") from exc def _validate_contract(instance: Mapping, schema_name: str) -> None: """执行 Draft 2020-12 契约校验，包含 additionalProperties 边界。""" try: from jsonschema import Draft202012Validator, FormatChecker except Exception as exc: # pragma: no cover - 部署缺依赖时响亮失败 raise ArtifactError(f"缺少 jsonschema 依赖:{exc}") from exc errors = sorted( Draft202012Validator( _contract_schema(schema_name), format_checker=FormatChecker(), ).iter_errors(instance), key=lambda item: tuple(str(part) for part in item.absolute_path), ) if errors: first = errors[0] location = "/".join(str(part) for part in first.absolute_path) or "#" raise ArtifactError(f"{schema_name} 校验失败:{location}:{first.message}") def _validate_game_package(data: bytes, game_id: str, version_id: str) -> dict: """验证运行时 manifest 是完整 GamePackage/1，且身份与对象版本一致。""" try: game_package = json.loads(data.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise ArtifactError("GamePackage manifest 不是合法 UTF-8 JSON") from exc if not isinstance(game_package, Mapping): raise ArtifactError("GamePackage manifest 顶层必须是对象") _validate_contract(game_package, "game-package.schema.json") if str(game_package.get("gameId")) != game_id or str(game_package.get("versionId")) != version_id: raise ArtifactError("GamePackage gameId/versionId 与对象版本不匹配") return dict(game_package) def _hash_objects(objects: Iterable[Mapping]) -> str: """对对象清单的稳定摘要，不读取对象内容之外的机器路径和时间。""" rows = [ {"category": item["category"], "path": item["path"], "bytes": item["bytes"], "sha256": item["sha256"]} for item in objects ] rows.sort(key=lambda item: (item["category"], item["path"])) return _hash_bytes(_TREE_HASH_DOMAIN + _canonical_json(rows)) def _bundle_object(objects: Iterable[Mapping]) -> Mapping | None: """按固定优先级挑选 runtime bundle，避免排序变化导致身份漂移。""" runtime_js = [item for item in objects if item.get("category") == "runtime" and str(item.get("path", "")).endswith(".js")] if not runtime_js: return None runtime_js.sort(key=lambda item: (0 if str(item["path"]).endswith("bundle.iife.js") else 1, 0 if str(item["path"]).endswith("entry-bundle.js") else 1, str(item["path"]))) return runtime_js[0] def _classify(rel: str) -> tuple[str, str]: """按路径把游戏输入映射为 source/asset/runtime/evidence 和包内路径。""" path = _safe_rel(rel) parts = path.split("/") name = parts[-1] if parts[0] == "evidence": return "evidence", path if parts[0] == "assets": return "asset", path if parts[0] == "dist": # dist 中只有可执行 bundle/meta 进入 runtime，其余报告留作 evidence。 if name.endswith(".js") or name.endswith(".js.meta.json") or name.endswith(".html"): return "runtime", f"runtime/{path[5:]}" return "evidence", f"evidence/{path}" if name in {"bundle.iife.js", "entry-bundle.js"} or name.endswith(".bundle.js"): return "runtime", f"runtime/{path}" if parts[0] in {"screenshots", "reports"} or name.endswith((".png", ".jpg", ".jpeg", ".webp", ".mp3", ".wav")): # 游戏根下孤立的二进制不是源码；按证据保存，资产应放在 assets/ 下。 return "evidence", f"evidence/{path}" return "source", f"source/{path}" def _is_transient(path: Path, rel: str) -> bool: """过滤缓存、凭据和临时批跑；过滤规则宁可少收，不把敏感文件上传。""" if any(part in _TRANSIENT_PARTS for part in path.parts): return True if path.name in _SECRET_NAMES or path.name.startswith(".env.") or path.name.endswith((".pem", ".key", ".p12")): return True if "/raw/" in f"/{rel}/" or rel.startswith("runs/raw/"): return True return False def _iter_files(root: Path) -> Iterable[tuple[str, Path]]: """以排序顺序枚举真实文件，拒绝符号链接和越界路径。""" root = root.resolve() if not root.is_dir(): raise ArtifactError(f"游戏根目录不存在或不是目录:{root}") for item in sorted(root.rglob("*"), key=lambda p: p.as_posix()): rel = item.relative_to(root).as_posix() if _is_transient(item, rel): continue if item.is_symlink(): raise ArtifactError(f"游戏包禁止符号链接:{rel}") if item.is_file(): yield _safe_rel(rel), item def build_manifest(root: Path, tenant_id: str, game_id: str, version_id: str, *, package_type: str, runtime_manifest_path: str, source_revision: Mapping) -> dict: """扫描一款游戏目录，生成可上传的对象清单和分类 hash。""" _validate_id(tenant_id, "tenantId") _validate_db_id(game_id, "gameId") _validate_db_id(version_id, "versionId") if package_type not in {"phaser", "littlejs", "canvas"}: raise ArtifactError(f"packageType 非法:{package_type}") key_prefix = f"tenants/{tenant_id}/games/{game_id}/versions/{version_id}" objects: list[dict] = [] root = Path(root).resolve() runtime_manifest_rel = _safe_rel(runtime_manifest_path) runtime_manifest = None source_files: list[tuple[str, Path]] = [] for rel, file_path in _iter_files(root): if rel == runtime_manifest_rel: if file_path.stat().st_size > _MAX_OBJECT_BYTES: raise ArtifactError(f"GamePackage manifest 超过 {_MAX_OBJECT_BYTES} 字节") data = file_path.read_bytes() _validate_game_package(data, game_id, version_id) runtime_manifest = { "store": "artifact", "key": f"{key_prefix}/manifest.json", "sourcePath": rel, "bytes": len(data), "sha256": _hash_bytes(data), "mime": "application/json", } continue category, package_path = _classify(rel) if category == "source": source_files.append((rel, file_path)) continue size, sha256 = _file_digest(file_path) mime = mimetypes.guess_type(file_path.name)[0] or "application/octet-stream" key = f"{key_prefix}/{package_path}" objects.append({ "category": category, "store": _CATEGORY_STORE[category], "path": package_path, "sourcePath": rel, "key": key, "bytes": size, "sha256": sha256, "mime": mime, }) if runtime_manifest is None: raise ArtifactError(f"指定的 GamePackage manifest 不存在:{runtime_manifest_rel}") provider = str(source_revision.get("provider", "")) source_game_id = _validate_id(str(source_revision.get("gameId", "")), "sourceRevision.gameId") revision_id = _validate_id(str(source_revision.get("revisionId", "")), "sourceRevision.revisionId") manifest_ref = str(source_revision.get("manifestRef", "")) if provider not in {"tier2_source_project", "game_source_archive"}: raise ArtifactError(f"sourceRevision.provider 非法:{provider}") if provider == "tier2_source_project" and package_type != "phaser": raise ArtifactError("tier2_source_project 只允许 packageType=phaser") if provider == "game_source_archive": _validate_db_id(source_game_id, "sourceRevision.gameId") source_objects, source_bytes, source_hash = _source_archive_hash(source_files) expected_manifest_ref = f"game-source-archive:{source_game_id}:{revision_id}" else: source_objects, source_bytes, source_hash = _source_project_hash(source_files) expected_manifest_ref = f"tier2-source-project:{source_game_id}:{revision_id}" if manifest_ref != expected_manifest_ref: raise ArtifactError("sourceRevision.manifestRef 与 provider/gameId/revisionId 不匹配") if str(source_revision.get("sourceHash", "")) != source_hash: raise ArtifactError("本地源工程 hash 与 sourceRevision.sourceHash 不匹配") source_revision_record = { "provider": provider, "gameId": source_game_id, "revisionId": revision_id, "manifestRef": manifest_ref, "sourceHash": source_hash, "objects": source_objects, "bytes": source_bytes, } objects.sort(key=lambda item: (item["category"], item["path"])) if len(objects) > _MAX_OBJECTS: raise ArtifactError(f"对象数量超过 {_MAX_OBJECTS}") by_category = { category: [item for item in objects if item["category"] == category] for category in _CATEGORY_ORDER } bundle = _bundle_object(by_category["runtime"]) runtime_asset_bytes = sum(item["bytes"] for item in by_category["runtime"] + by_category["asset"]) if runtime_asset_bytes > _MAX_RUNTIME_ASSET_BYTES: raise ArtifactError(f"运行包与素材合计超过 {_MAX_RUNTIME_ASSET_BYTES} 字节") manifest = { "schemaVersion": SCHEMA_VERSION, "tenantId": tenant_id, "gameId": game_id, "versionId": version_id, "packageType": package_type, "keyPrefix": key_prefix, "sourceRevision": source_revision_record, "runtimeManifest": runtime_manifest, "objects": objects, "totals": { "objects": len(objects), "bytes": sum(item["bytes"] for item in objects), "assetBytes": sum(item["bytes"] for item in by_category["asset"]), "runtimeBytes": sum(item["bytes"] for item in by_category["runtime"]), "evidenceBytes": sum(item["bytes"] for item in by_category["evidence"]), }, "assetHash": _hash_objects(by_category["asset"]), "bundleHash": bundle["sha256"] if bundle else None, } manifest["manifestHash"] = _hash_bytes(_MANIFEST_HASH_DOMAIN + _canonical_json(manifest)) if len(_canonical_json(manifest)) > _MAX_MANIFEST_BYTES: raise ArtifactError(f"artifact manifest 超过 {_MAX_MANIFEST_BYTES} 字节") return manifest def verify_manifest(manifest: Mapping) -> None: """验证 manifest 自身 hash、对象路径和 totals，失败即拒绝消费。""" _validate_contract(manifest, "game-artifact-manifest.schema.json") if len(_canonical_json(manifest)) > _MAX_MANIFEST_BYTES: raise ArtifactError(f"artifact manifest 超过 {_MAX_MANIFEST_BYTES} 字节") if manifest.get("schemaVersion") != SCHEMA_VERSION: raise ArtifactError("manifest schemaVersion 不受支持") _validate_id(str(manifest.get("tenantId", "")), "tenantId") _validate_db_id(str(manifest.get("gameId", "")), "gameId") _validate_db_id(str(manifest.get("versionId", "")), "versionId") declared = manifest.get("manifestHash") if not isinstance(declared, str) or not _HASH_RE.fullmatch(declared): raise ArtifactError("manifestHash 非法") unsigned = dict(manifest) unsigned.pop("manifestHash", None) if _hash_bytes(_MANIFEST_HASH_DOMAIN + _canonical_json(unsigned)) != declared: raise ArtifactError("manifestHash 不匹配") if manifest.get("packageType") not in {"phaser", "littlejs", "canvas"}: raise ArtifactError("manifest packageType 不受支持") key_prefix = str(manifest.get("keyPrefix", "")) if key_prefix != f"tenants/{manifest['tenantId']}/games/{manifest['gameId']}/versions/{manifest['versionId']}": raise ArtifactError("manifest keyPrefix 与 tenant/game/version 不匹配") source_revision = manifest["sourceRevision"] provider = source_revision["provider"] if provider not in {"tier2_source_project", "game_source_archive"}: raise ArtifactError("sourceRevision.provider 不受支持") if provider == "tier2_source_project" and manifest.get("packageType") != "phaser": raise ArtifactError("tier2_source_project 只允许 packageType=phaser") if provider == "game_source_archive": _validate_db_id(str(source_revision["gameId"]), "sourceRevision.gameId") expected_source_ref = f"game-source-archive:{source_revision['gameId']}:{source_revision['revisionId']}" else: _validate_id(str(source_revision["gameId"]), "sourceRevision.gameId") expected_source_ref = f"tier2-source-project:{source_revision['gameId']}:{source_revision['revisionId']}" if source_revision["manifestRef"] != expected_source_ref: raise ArtifactError("sourceRevision.manifestRef 与身份字段不匹配") objects = manifest.get("objects") or [] totals = manifest.get("totals") or {} if len(objects) != totals.get("objects"): raise ArtifactError("manifest totals.objects 不匹配") if len(objects) > _MAX_OBJECTS: raise ArtifactError(f"对象数量超过 {_MAX_OBJECTS}") seen: set[str] = set() seen_sources: set[str] = set() sums = {category: 0 for category in _CATEGORY_ORDER} grouped: dict[str, list[Mapping]] = {category: [] for category in _CATEGORY_ORDER} for item in objects: raw_path = str(item.get("path", "")) raw_source_path = str(item.get("sourcePath", "")) path = _safe_rel(raw_path) source_path = _safe_rel(raw_source_path) if raw_path != path or raw_source_path != source_path: raise ArtifactError(f"manifest 路径必须使用 NFC 规范形式:{raw_source_path}") key = str(item.get("key", "")) if path in seen: raise ArtifactError(f"manifest 存在重复路径:{path}") if source_path in seen_sources: raise ArtifactError(f"manifest 存在重复源路径:{source_path}") seen.add(path) seen_sources.add(source_path) category = str(item.get("category", "")) if category not in _CATEGORY_DIR: raise ArtifactError(f"对象 category 非法:{path}") expected_path_prefix = f"{_CATEGORY_DIR[category]}/" if not path.startswith(expected_path_prefix): raise ArtifactError(f"对象 path 与 category 不匹配:{path}") if item.get("store") != _CATEGORY_STORE[category]: raise ArtifactError(f"对象 store 与 category 不匹配:{path}") expected_key = f"{key_prefix}/{path}" if key != expected_key: raise ArtifactError(f"对象 key 与 path 不匹配:{key}") if not isinstance(item.get("bytes"), int) or item["bytes"] < 0: raise ArtifactError(f"对象 bytes 非法:{path}") if item["bytes"] > _MAX_OBJECT_BYTES: raise ArtifactError(f"对象 bytes 超限:{path}") if not _HASH_RE.fullmatch(str(item.get("sha256", ""))): raise ArtifactError(f"对象 hash 非法:{path}") if not isinstance(item.get("mime"), str) or not item["mime"]: raise ArtifactError(f"对象 mime 非法:{path}") sums[category] += item["bytes"] grouped[category].append(item) runtime_manifest = manifest.get("runtimeManifest") if not isinstance(runtime_manifest, Mapping): raise ArtifactError("runtimeManifest 必须是对象") raw_runtime_source_path = str(runtime_manifest.get("sourcePath", "")) source_path = _safe_rel(raw_runtime_source_path) if raw_runtime_source_path != source_path: raise ArtifactError("runtimeManifest sourcePath 必须使用 NFC 规范形式") if str(runtime_manifest.get("key")) != f"{key_prefix}/manifest.json": raise ArtifactError("runtimeManifest key 不匹配") if runtime_manifest.get("store") != "artifact": raise ArtifactError("runtimeManifest store 必须为 artifact") if source_path in seen_sources: raise ArtifactError("runtimeManifest 与对象 sourcePath 重复") if not isinstance(runtime_manifest.get("bytes"), int) or runtime_manifest["bytes"] < 0: raise ArtifactError("runtimeManifest bytes 非法") if runtime_manifest["bytes"] > _MAX_OBJECT_BYTES: raise ArtifactError("runtimeManifest bytes 超限") if not _HASH_RE.fullmatch(str(runtime_manifest.get("sha256", ""))): raise ArtifactError("runtimeManifest hash 非法") if runtime_manifest.get("mime") != "application/json": raise ArtifactError("runtimeManifest mime 必须为 application/json") expected_totals = { "bytes": sum(sums.values()), "assetBytes": sums["asset"], "runtimeBytes": sums["runtime"], "evidenceBytes": sums["evidence"], } for name, expected in expected_totals.items(): if totals.get(name) != expected: raise ArtifactError(f"manifest totals.{name} 不匹配") if sums["runtime"] + sums["asset"] > _MAX_RUNTIME_ASSET_BYTES: raise ArtifactError("运行包与素材总字节超限") if manifest.get("assetHash") != _hash_objects(grouped["asset"]): raise ArtifactError("manifest assetHash 不匹配") bundle = _bundle_object(grouped["runtime"]) if manifest.get("bundleHash") != (bundle["sha256"] if bundle else None): raise ArtifactError("manifest bundleHash 不匹配") def _minio_client(config: Mapping): """惰性创建 MinIO 客户端；不在 import 期连接网络。""" try: from minio import Minio except Exception as exc: # pragma: no cover - 环境缺 SDK 时由调用方看到稳定错误 raise ArtifactError(f"缺少 minio Python SDK:{exc}") from exc endpoint = str(config.get("endpoint", "")).replace("http://", "").replace("https://", "") if not endpoint or not config.get("access_key") or not config.get("secret_key"): raise ArtifactError("缺少专用 game artifact 对象存储端点或凭据") return Minio(endpoint, access_key=config.get("access_key"), secret_key=config.get("secret_key"), secure=bool(config.get("secure", False))) def _read_remote(client, bucket: str, key: str, *, expected_bytes: int) -> bytes: """按调用方字节预算流式读取；发现第 expected+1 字节就立即拒绝。""" if not isinstance(expected_bytes, int) or expected_bytes < 0: raise ArtifactError(f"远端对象期望字节数非法:{key}") try: response = client.get_object(bucket, key) except Exception as exc: # noqa: BLE001 - 兼容 MinIO S3Error 和测试 fake if getattr(exc, "code", None) == "NoSuchBucket": raise ArtifactError(f"对象存储桶不存在:{bucket}") from exc raise try: chunks: list[bytes] = [] total = 0 try: while True: remaining = expected_bytes - total chunk = response.read(min(_REMOTE_READ_CHUNK_BYTES, remaining + 1)) if not chunk: break total += len(chunk) if total > expected_bytes: raise ArtifactError(f"远端对象超过期望字节数:{key}:{expected_bytes}") chunks.append(chunk) return b"".join(chunks) except ArtifactError: raise except Exception as exc: # noqa: BLE001 - SDK/网络读取统一失败语义 raise ArtifactError(f"读取远端对象失败:{bucket}/{key}:{type(exc).__name__}:{exc}") from exc finally: response.close() response.release_conn() def _bucket(config: Mapping, store: str) -> str: """把稳定 store 身份映射到环境桶名；manifest 不保存凭据或临时 URL。""" if store == "artifact": return str(config.get("artifact_bucket", config.get("bucket", "game-artifacts"))) if store == "evidence": return str(config.get("evidence_bucket", "game-evidence")) if store == "source": source_bucket = str(config.get("source_bucket", "")) if not source_bucket: raise ArtifactError("缺少显式 source_bucket；game-sources 尚未完成权限配置") if source_bucket in { str(config.get("artifact_bucket", "")), str(config.get("evidence_bucket", "")), }: raise ArtifactError("source_bucket 不能与 artifact/evidence 桶复用") return source_bucket raise ArtifactError(f"未知对象 store:{store}") def _existing_manifest(client, bucket: str, key: str) -> dict | None: """读取已存在的版本索引；仅把 S3 的对象不存在视为未上传。""" try: payload = _read_remote(client, bucket, key, expected_bytes=_MAX_MANIFEST_BYTES) except ArtifactError: raise except Exception as exc: # noqa: BLE001 - 兼容 MinIO S3Error 和测试 fake if getattr(exc, "code", None) in {"NoSuchKey", "NoSuchObject"}: return None raise ArtifactError(f"探测远端 artifact-manifest 失败:{type(exc).__name__}:{exc}") from exc try: existing = json.loads(payload.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise ArtifactError("远端 artifact-manifest 已存在但不是合法 JSON") from exc verify_manifest(existing) return existing def _verify_remote_object(client, bucket: str, key: str, expected_hash: str, expected_bytes: int) -> None: """用 GET 字节校验远端对象；不把 HEAD/ETag 当作内容完整性证据。""" try: data = _read_remote(client, bucket, key, expected_bytes=expected_bytes) except ArtifactError: raise except Exception as exc: # noqa: BLE001 - 缺对象/SDK 错误统一成稳定失败 raise ArtifactError(f"读取远端对象失败:{bucket}/{key}:{type(exc).__name__}:{exc}") from exc if len(data) != expected_bytes or _hash_bytes(data) != expected_hash: raise ArtifactError(f"上传后对象校验失败:{key}") def _put_remote(client, bucket: str, key: str, stream, *, length: int, content_type: str) -> None: """统一包装 S3 Put 异常，CLI 只暴露稳定 ArtifactError。""" try: client.put_object(bucket, key, stream, length=length, content_type=content_type) except ArtifactError: raise except Exception as exc: # noqa: BLE001 - 兼容 MinIO S3Error/网络异常 raise ArtifactError(f"对象上传失败:{bucket}/{key}:{type(exc).__name__}:{exc}") from exc def upload_manifest(root: Path, manifest: Mapping, config: Mapping, *, external_single_writer: bool = False) -> dict: """在外部单写锁内上传并校验对象；波次 1 只返回 verified_pending。""" verify_manifest(manifest) if external_single_writer is not True: raise ArtifactError("波次 1 上传必须由调用方持有外部单写锁") client = _minio_client(config) buckets = {store: _bucket(config, store) for store in ("artifact", "evidence")} root = Path(root).resolve() manifest_key = f"{manifest['keyPrefix']}/artifact-manifest.json" existing = _existing_manifest(client, buckets["artifact"], manifest_key) if existing is not None: if existing["manifestHash"] == manifest["manifestHash"]: for item in manifest["objects"]: _verify_remote_object(client, buckets[item["store"]], item["key"], item["sha256"], item["bytes"]) runtime_manifest = manifest["runtimeManifest"] _verify_remote_object(client, buckets["artifact"], runtime_manifest["key"], runtime_manifest["sha256"], runtime_manifest["bytes"]) return { "artifactBucket": buckets["artifact"], "artifactManifestKey": manifest_key, "manifestHash": manifest["manifestHash"], "idempotent": True, "status": "verified_pending", "committed": False, } raise ArtifactError("同一 gameId/versionId 已存在不同 artifact-manifest，拒绝覆盖") # 先把所有正文和 GamePackage 做完本地预检，再开始任何远端 Put，避免后一个文件漂移 # 时留下没有数据库 pending 记录的半上传对象。 local_objects: list[tuple[Mapping, Path, int]] = [] for item in manifest["objects"]: package_path = str(item["path"]) source = root / _safe_rel(str(item["sourcePath"])) if source.is_symlink() or not source.is_file(): raise ArtifactError(f"manifest 对应本地文件不存在:{package_path}") size, sha256 = _file_digest(source) if size != item["bytes"]: raise ArtifactError(f"本地对象字节数漂移:{package_path}") if sha256 != item["sha256"]: raise ArtifactError(f"本地对象 hash 漂移:{package_path}") local_objects.append((item, source, size)) runtime_manifest = manifest["runtimeManifest"] source = root / _safe_rel(str(runtime_manifest["sourcePath"])) if source.is_symlink() or not source.is_file(): raise ArtifactError("GamePackage manifest 本地文件不存在") payload = source.read_bytes() if len(payload) != runtime_manifest["bytes"] or _hash_bytes(payload) != runtime_manifest["sha256"]: raise ArtifactError("GamePackage manifest 本地 hash 漂移") _validate_game_package(payload, str(manifest["gameId"]), str(manifest["versionId"])) # 所有本地输入都已验证，以下才产生远端副作用。 for item, source, size in local_objects: with source.open("rb") as stream: _put_remote(client, buckets[item["store"]], item["key"], stream, length=size, content_type=item["mime"]) _put_remote(client, buckets["artifact"], runtime_manifest["key"], BytesIO(payload), length=len(payload), content_type="application/json") # 先对正文对象做 GET 全量复算，再写 marker；marker 仍不是数据库 CAS 提交记录。 for item in manifest["objects"]: _verify_remote_object(client, buckets[item["store"]], item["key"], item["sha256"], item["bytes"]) _verify_remote_object(client, buckets["artifact"], runtime_manifest["key"], runtime_manifest["sha256"], runtime_manifest["bytes"]) payload = _canonical_json(manifest) _put_remote(client, buckets["artifact"], manifest_key, BytesIO(payload), length=len(payload), content_type="application/json") remote_bytes = _read_remote( client, buckets["artifact"], manifest_key, expected_bytes=len(payload), ) if remote_bytes != payload: raise ArtifactError("上传后 manifest 下载 hash 不匹配") return { "artifactBucket": buckets["artifact"], "artifactManifestKey": manifest_key, "manifestHash": manifest["manifestHash"], "idempotent": False, "status": "verified_pending", "committed": False, } def fetch_manifest(config: Mapping, *, key: str, expected_manifest_hash: str, expected_tenant_id: str, expected_game_id: str, expected_version_id: str) -> dict: """从对象存储读取 artifact-manifest，不接受任意 URL，调用方需传入受信 key。""" client = _minio_client(config) bucket = _bucket(config, "artifact") if not _ARTIFACT_MANIFEST_KEY_RE.fullmatch(key): raise ArtifactError("artifact-manifest key 非法") if not _HASH_RE.fullmatch(expected_manifest_hash): raise ArtifactError("expectedManifestHash 非法") _validate_id(expected_tenant_id, "expectedTenantId") _validate_db_id(expected_game_id, "expectedGameId") _validate_db_id(expected_version_id, "expectedVersionId") payload = _read_remote(client, bucket, key, expected_bytes=_MAX_MANIFEST_BYTES) try: manifest = json.loads(payload.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise ArtifactError("远端 artifact-manifest 不是合法 JSON") from exc verify_manifest(manifest) if manifest["manifestHash"] != expected_manifest_hash: raise ArtifactError("artifact-manifest 与数据库期望 hash 不匹配") if (manifest["tenantId"], manifest["gameId"], manifest["versionId"]) != ( expected_tenant_id, expected_game_id, expected_version_id, ): raise ArtifactError("artifact-manifest 与数据库期望身份不匹配") if key != f"{manifest['keyPrefix']}/artifact-manifest.json": raise ArtifactError("artifact-manifest key 与内容身份不匹配") return manifest def download_manifest(manifest: Mapping, config: Mapping, target: Path, *, expected_manifest_hash: str) -> Path: """按可信 hash 物化到临时目录，全量校验成功后原子交付。""" verify_manifest(manifest) if manifest["manifestHash"] != expected_manifest_hash: raise ArtifactError("物化 manifest 与数据库期望 hash 不匹配") client = _minio_client(config) target = Path(target).resolve() if target.exists(): raise ArtifactError(f"物化目标已存在:{target}") target.parent.mkdir(parents=True, exist_ok=True) staging = Path(tempfile.mkdtemp(prefix=f".{target.name}.partial-", dir=target.parent)).resolve() try: for item in manifest["objects"]: rel = _safe_rel(str(item["path"])) destination = (staging / rel).resolve() if staging not in destination.parents: raise ArtifactError(f"下载路径越界:{rel}") destination.parent.mkdir(parents=True, exist_ok=True) data = _read_remote( client, _bucket(config, str(item["store"])), item["key"], expected_bytes=item["bytes"], ) if len(data) != item["bytes"] or _hash_bytes(data) != item["sha256"]: raise ArtifactError(f"下载对象 hash 不匹配:{rel}") destination.write_bytes(data) runtime_manifest = manifest["runtimeManifest"] destination = staging / "manifest.json" data = _read_remote( client, _bucket(config, "artifact"), runtime_manifest["key"], expected_bytes=runtime_manifest["bytes"], ) if len(data) != runtime_manifest["bytes"] or _hash_bytes(data) != runtime_manifest["sha256"]: raise ArtifactError("下载 GamePackage manifest hash 不匹配") _validate_game_package(data, str(manifest["gameId"]), str(manifest["versionId"])) destination.write_bytes(data) (staging / "artifact-manifest.json").write_bytes(_canonical_json(manifest)) staging.replace(target) except Exception: shutil.rmtree(staging, ignore_errors=True) raise return target