17727d08c3
冻结 GameArtifactManifest/GameSourceArchive 契约和 V34 pending/committed 状态记录，增加专用 writer 的全量回读物化校验，并让 Runtime API 在消费和发布前对账 committed、GamePackage checksum 与实际 engineBundle hash。隔离 staging 已验证真实消费拒绝未提交版本；源码桶权限仍保持 fail-closed，正式金标不切换。
749 lines
37 KiB
Python
749 lines
37 KiB
Python
"""游戏包对象存储工具。
|
||
|
||
本模块只处理平台侧的分类、哈希、manifest 和 MinIO/S3 读写；具体游戏目录只是一次性上传输入，
|
||
不是生产消费来源。实现使用标准库完成本地快照，只有真正执行对象存储读写时才惰性导入 minio，
|
||
让契约测试和离线 manifest 构建不依赖网络或第三方 SDK。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import mimetypes
|
||
import os
|
||
import posixpath
|
||
import re
|
||
import shutil
|
||
import tempfile
|
||
import unicodedata
|
||
from io import BytesIO
|
||
from pathlib import Path
|
||
from typing import Iterable, Mapping
|
||
|
||
SCHEMA_VERSION = "GameArtifactManifest/1"
|
||
_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
||
_DB_ID_RE = re.compile(r"^[1-9][0-9]*$")
|
||
_HASH_RE = re.compile(r"^[a-f0-9]{64}$")
|
||
_ARTIFACT_MANIFEST_KEY_RE = re.compile(
|
||
r"^tenants/[A-Za-z0-9][A-Za-z0-9._-]*/games/[1-9][0-9]*/"
|
||
r"versions/[1-9][0-9]*/artifact-manifest\.json$"
|
||
)
|
||
_CATEGORY_ORDER = ("asset", "runtime", "evidence")
|
||
_CATEGORY_DIR = {"asset": "assets", "runtime": "runtime", "evidence": "evidence"}
|
||
_CATEGORY_STORE = {"asset": "artifact", "runtime": "artifact", "evidence": "evidence"}
|
||
_TRANSIENT_PARTS = frozenset({
|
||
".git", ".idea", "node_modules", "__pycache__", ".pytest_cache", "target",
|
||
"coverage", ".DS_Store", "_store", "_cheap-sessions", "_tier2-gen",
|
||
})
|
||
_SECRET_NAMES = frozenset({
|
||
".env", ".env.local", ".env.production", "credentials.json", "service-account.json",
|
||
})
|
||
_MANIFEST_HASH_DOMAIN = b"GameArtifactManifest/1\0"
|
||
_TREE_HASH_DOMAIN = b"GameArtifactTreeHash/1\0"
|
||
_SOURCE_TREE_HASH_DOMAIN = b"GameSourceArchiveTreeHash/1\0"
|
||
_MAX_OBJECTS = 10_000
|
||
_MAX_PATH_BYTES = 1_024
|
||
_MAX_OBJECT_BYTES = 100 * 1024 * 1024
|
||
_MAX_RUNTIME_ASSET_BYTES = 10 * 1024 * 1024
|
||
_MAX_MANIFEST_BYTES = 8 * 1024 * 1024
|
||
_REMOTE_READ_CHUNK_BYTES = 1024 * 1024
|
||
|
||
|
||
class ArtifactError(ValueError):
|
||
"""对象包不满足安全或一致性约束时抛出的稳定异常。"""
|
||
|
||
|
||
def _validate_id(value: str, label: str) -> str:
|
||
"""校验 gameId/versionId，避免对象 key 被注入路径片段。"""
|
||
if not isinstance(value, str) or not _ID_RE.fullmatch(value):
|
||
raise ArtifactError(f"{label} 非法:{value!r}")
|
||
return value
|
||
|
||
|
||
def _validate_db_id(value: str, label: str) -> str:
|
||
"""生产 game/version 身份必须来自数据库正整数主键，禁止 Tier2 slug 冒充。"""
|
||
if not isinstance(value, str) or not _DB_ID_RE.fullmatch(value):
|
||
raise ArtifactError(f"{label} 必须是数据库正整数 ID:{value!r}")
|
||
return value
|
||
|
||
|
||
def _safe_rel(path: str) -> str:
|
||
"""把本地相对路径规范化为 POSIX，并拒绝绝对路径、父级跳转和空路径。"""
|
||
normalized = unicodedata.normalize("NFC", posixpath.normpath(path.replace(os.sep, "/")))
|
||
if not normalized or normalized in (".", "..") or normalized.startswith("../") or normalized.startswith("/"):
|
||
raise ArtifactError(f"对象相对路径非法:{path!r}")
|
||
if any(part in ("", ".", "..") for part in normalized.split("/")):
|
||
raise ArtifactError(f"对象相对路径含危险片段:{path!r}")
|
||
if len(normalized.encode("utf-8")) > _MAX_PATH_BYTES:
|
||
raise ArtifactError(f"对象相对路径超过 {_MAX_PATH_BYTES} 字节:{path!r}")
|
||
return normalized
|
||
|
||
|
||
def _canonical_json(value: Mapping) -> bytes:
|
||
"""使用无空格、排序键的 JSON 字节，保证跨语言 manifest hash 可复算。"""
|
||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||
|
||
|
||
def _hash_bytes(data: bytes) -> str:
|
||
return hashlib.sha256(data).hexdigest()
|
||
|
||
|
||
def _file_digest(path: Path) -> tuple[int, str]:
|
||
"""流式计算本地文件大小与 SHA-256，避免把大文件整体读入内存。"""
|
||
digest = hashlib.sha256()
|
||
size = 0
|
||
with path.open("rb") as stream:
|
||
while chunk := stream.read(1024 * 1024):
|
||
size += len(chunk)
|
||
if size > _MAX_OBJECT_BYTES:
|
||
raise ArtifactError(f"单文件超过 {_MAX_OBJECT_BYTES} 字节:{path.name}")
|
||
digest.update(chunk)
|
||
return size, digest.hexdigest()
|
||
|
||
|
||
def _source_project_hash(files: Iterable[tuple[str, Path]]) -> tuple[int, int, str]:
|
||
"""按 SourceProjectStore 的 path\0content\0 口径复算源工程 contentHash。"""
|
||
rows = sorted(files, key=lambda item: item[0])
|
||
digest = hashlib.sha256()
|
||
total_bytes = 0
|
||
for rel, path in rows:
|
||
data = path.read_bytes()
|
||
if len(data) > _MAX_OBJECT_BYTES:
|
||
raise ArtifactError(f"源文件超过 {_MAX_OBJECT_BYTES} 字节:{rel}")
|
||
try:
|
||
content = data.decode("utf-8")
|
||
except UnicodeDecodeError as exc:
|
||
raise ArtifactError(f"源工程文件必须是 UTF-8 文本:{rel}") from exc
|
||
digest.update((rel + "\0").encode("utf-8"))
|
||
digest.update((content + "\0").encode("utf-8"))
|
||
total_bytes += len(data)
|
||
if not rows:
|
||
raise ArtifactError("游戏包缺少源工程文件")
|
||
return len(rows), total_bytes, digest.hexdigest()
|
||
|
||
|
||
def _source_archive_rows(files: Iterable[tuple[str, Path]]) -> list[dict]:
|
||
"""按二进制内容构造引擎无关源归档的稳定文件行。"""
|
||
rows: list[dict] = []
|
||
seen: set[str] = set()
|
||
normalized_files = [(_safe_rel(rel), path) for rel, path in files]
|
||
for normalized, path in sorted(normalized_files, key=lambda item: item[0]):
|
||
if normalized in seen:
|
||
raise ArtifactError(f"源归档存在重复路径:{normalized}")
|
||
seen.add(normalized)
|
||
size, sha256 = _file_digest(path)
|
||
rows.append({"path": normalized, "bytes": size, "sha256": sha256})
|
||
if not rows:
|
||
raise ArtifactError("游戏包缺少源工程文件")
|
||
if len(rows) > _MAX_OBJECTS:
|
||
raise ArtifactError(f"源归档对象数量超过 {_MAX_OBJECTS}")
|
||
return rows
|
||
|
||
|
||
def _source_archive_hash(files: Iterable[tuple[str, Path]]) -> tuple[int, int, str]:
|
||
"""计算 GameSourceArchive/1 的二进制安全源工程 hash。"""
|
||
rows = _source_archive_rows(files)
|
||
total_bytes = sum(item["bytes"] for item in rows)
|
||
digest = _hash_bytes(_SOURCE_TREE_HASH_DOMAIN + _canonical_json(rows))
|
||
return len(rows), total_bytes, digest
|
||
|
||
|
||
def _contract_schema(name: str) -> dict:
|
||
"""从仓内契约 SoT 读取 JSON Schema。"""
|
||
path = Path(__file__).resolve().parents[3] / "contracts" / name
|
||
try:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError) as exc:
|
||
raise ArtifactError(f"读取契约失败:{name}:{exc}") from exc
|
||
|
||
|
||
def _validate_contract(instance: Mapping, schema_name: str) -> None:
|
||
"""执行 Draft 2020-12 契约校验，包含 additionalProperties 边界。"""
|
||
try:
|
||
from jsonschema import Draft202012Validator, FormatChecker
|
||
except Exception as exc: # pragma: no cover - 部署缺依赖时响亮失败
|
||
raise ArtifactError(f"缺少 jsonschema 依赖:{exc}") from exc
|
||
errors = sorted(
|
||
Draft202012Validator(
|
||
_contract_schema(schema_name), format_checker=FormatChecker(),
|
||
).iter_errors(instance),
|
||
key=lambda item: tuple(str(part) for part in item.absolute_path),
|
||
)
|
||
if errors:
|
||
first = errors[0]
|
||
location = "/".join(str(part) for part in first.absolute_path) or "#"
|
||
raise ArtifactError(f"{schema_name} 校验失败:{location}:{first.message}")
|
||
|
||
|
||
def _validate_game_package(data: bytes, game_id: str, version_id: str) -> dict:
|
||
"""验证运行时 manifest 是完整 GamePackage/1，且身份与对象版本一致。"""
|
||
try:
|
||
game_package = json.loads(data.decode("utf-8"))
|
||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||
raise ArtifactError("GamePackage manifest 不是合法 UTF-8 JSON") from exc
|
||
if not isinstance(game_package, Mapping):
|
||
raise ArtifactError("GamePackage manifest 顶层必须是对象")
|
||
_validate_contract(game_package, "game-package.schema.json")
|
||
if str(game_package.get("gameId")) != game_id or str(game_package.get("versionId")) != version_id:
|
||
raise ArtifactError("GamePackage gameId/versionId 与对象版本不匹配")
|
||
return dict(game_package)
|
||
|
||
|
||
def _hash_objects(objects: Iterable[Mapping]) -> str:
|
||
"""对对象清单的稳定摘要，不读取对象内容之外的机器路径和时间。"""
|
||
rows = [
|
||
{"category": item["category"], "path": item["path"], "bytes": item["bytes"], "sha256": item["sha256"]}
|
||
for item in objects
|
||
]
|
||
rows.sort(key=lambda item: (item["category"], item["path"]))
|
||
return _hash_bytes(_TREE_HASH_DOMAIN + _canonical_json(rows))
|
||
|
||
|
||
def _bundle_object(objects: Iterable[Mapping]) -> Mapping | None:
|
||
"""按固定优先级挑选 runtime bundle，避免排序变化导致身份漂移。"""
|
||
runtime_js = [item for item in objects if item.get("category") == "runtime" and str(item.get("path", "")).endswith(".js")]
|
||
if not runtime_js:
|
||
return None
|
||
runtime_js.sort(key=lambda item: (0 if str(item["path"]).endswith("bundle.iife.js") else 1,
|
||
0 if str(item["path"]).endswith("entry-bundle.js") else 1,
|
||
str(item["path"])))
|
||
return runtime_js[0]
|
||
|
||
|
||
def _classify(rel: str) -> tuple[str, str]:
|
||
"""按路径把游戏输入映射为 source/asset/runtime/evidence 和包内路径。"""
|
||
path = _safe_rel(rel)
|
||
parts = path.split("/")
|
||
name = parts[-1]
|
||
if parts[0] == "evidence":
|
||
return "evidence", path
|
||
if parts[0] == "assets":
|
||
return "asset", path
|
||
if parts[0] == "dist":
|
||
# dist 中只有可执行 bundle/meta 进入 runtime，其余报告留作 evidence。
|
||
if name.endswith(".js") or name.endswith(".js.meta.json") or name.endswith(".html"):
|
||
return "runtime", f"runtime/{path[5:]}"
|
||
return "evidence", f"evidence/{path}"
|
||
if name in {"bundle.iife.js", "entry-bundle.js"} or name.endswith(".bundle.js"):
|
||
return "runtime", f"runtime/{path}"
|
||
if parts[0] in {"screenshots", "reports"} or name.endswith((".png", ".jpg", ".jpeg", ".webp", ".mp3", ".wav")):
|
||
# 游戏根下孤立的二进制不是源码；按证据保存，资产应放在 assets/ 下。
|
||
return "evidence", f"evidence/{path}"
|
||
return "source", f"source/{path}"
|
||
|
||
|
||
def _is_transient(path: Path, rel: str) -> bool:
|
||
"""过滤缓存、凭据和临时批跑；过滤规则宁可少收，不把敏感文件上传。"""
|
||
if any(part in _TRANSIENT_PARTS for part in path.parts):
|
||
return True
|
||
if path.name in _SECRET_NAMES or path.name.startswith(".env.") or path.name.endswith((".pem", ".key", ".p12")):
|
||
return True
|
||
if "/raw/" in f"/{rel}/" or rel.startswith("runs/raw/"):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _iter_files(root: Path) -> Iterable[tuple[str, Path]]:
|
||
"""以排序顺序枚举真实文件，拒绝符号链接和越界路径。"""
|
||
root = root.resolve()
|
||
if not root.is_dir():
|
||
raise ArtifactError(f"游戏根目录不存在或不是目录:{root}")
|
||
for item in sorted(root.rglob("*"), key=lambda p: p.as_posix()):
|
||
rel = item.relative_to(root).as_posix()
|
||
if _is_transient(item, rel):
|
||
continue
|
||
if item.is_symlink():
|
||
raise ArtifactError(f"游戏包禁止符号链接:{rel}")
|
||
if item.is_file():
|
||
yield _safe_rel(rel), item
|
||
|
||
|
||
def build_manifest(root: Path, tenant_id: str, game_id: str, version_id: str, *, package_type: str,
|
||
runtime_manifest_path: str, source_revision: Mapping) -> dict:
|
||
"""扫描一款游戏目录，生成可上传的对象清单和分类 hash。"""
|
||
_validate_id(tenant_id, "tenantId")
|
||
_validate_db_id(game_id, "gameId")
|
||
_validate_db_id(version_id, "versionId")
|
||
if package_type not in {"phaser", "littlejs", "canvas"}:
|
||
raise ArtifactError(f"packageType 非法:{package_type}")
|
||
key_prefix = f"tenants/{tenant_id}/games/{game_id}/versions/{version_id}"
|
||
objects: list[dict] = []
|
||
root = Path(root).resolve()
|
||
runtime_manifest_rel = _safe_rel(runtime_manifest_path)
|
||
runtime_manifest = None
|
||
source_files: list[tuple[str, Path]] = []
|
||
for rel, file_path in _iter_files(root):
|
||
if rel == runtime_manifest_rel:
|
||
if file_path.stat().st_size > _MAX_OBJECT_BYTES:
|
||
raise ArtifactError(f"GamePackage manifest 超过 {_MAX_OBJECT_BYTES} 字节")
|
||
data = file_path.read_bytes()
|
||
_validate_game_package(data, game_id, version_id)
|
||
runtime_manifest = {
|
||
"store": "artifact",
|
||
"key": f"{key_prefix}/manifest.json",
|
||
"sourcePath": rel,
|
||
"bytes": len(data),
|
||
"sha256": _hash_bytes(data),
|
||
"mime": "application/json",
|
||
}
|
||
continue
|
||
category, package_path = _classify(rel)
|
||
if category == "source":
|
||
source_files.append((rel, file_path))
|
||
continue
|
||
size, sha256 = _file_digest(file_path)
|
||
mime = mimetypes.guess_type(file_path.name)[0] or "application/octet-stream"
|
||
key = f"{key_prefix}/{package_path}"
|
||
objects.append({
|
||
"category": category,
|
||
"store": _CATEGORY_STORE[category],
|
||
"path": package_path,
|
||
"sourcePath": rel,
|
||
"key": key,
|
||
"bytes": size,
|
||
"sha256": sha256,
|
||
"mime": mime,
|
||
})
|
||
if runtime_manifest is None:
|
||
raise ArtifactError(f"指定的 GamePackage manifest 不存在:{runtime_manifest_rel}")
|
||
provider = str(source_revision.get("provider", ""))
|
||
source_game_id = _validate_id(str(source_revision.get("gameId", "")), "sourceRevision.gameId")
|
||
revision_id = _validate_id(str(source_revision.get("revisionId", "")), "sourceRevision.revisionId")
|
||
manifest_ref = str(source_revision.get("manifestRef", ""))
|
||
if provider not in {"tier2_source_project", "game_source_archive"}:
|
||
raise ArtifactError(f"sourceRevision.provider 非法:{provider}")
|
||
if provider == "tier2_source_project" and package_type != "phaser":
|
||
raise ArtifactError("tier2_source_project 只允许 packageType=phaser")
|
||
if provider == "game_source_archive":
|
||
_validate_db_id(source_game_id, "sourceRevision.gameId")
|
||
source_objects, source_bytes, source_hash = _source_archive_hash(source_files)
|
||
expected_manifest_ref = f"game-source-archive:{source_game_id}:{revision_id}"
|
||
else:
|
||
source_objects, source_bytes, source_hash = _source_project_hash(source_files)
|
||
expected_manifest_ref = f"tier2-source-project:{source_game_id}:{revision_id}"
|
||
if manifest_ref != expected_manifest_ref:
|
||
raise ArtifactError("sourceRevision.manifestRef 与 provider/gameId/revisionId 不匹配")
|
||
if str(source_revision.get("sourceHash", "")) != source_hash:
|
||
raise ArtifactError("本地源工程 hash 与 sourceRevision.sourceHash 不匹配")
|
||
source_revision_record = {
|
||
"provider": provider,
|
||
"gameId": source_game_id,
|
||
"revisionId": revision_id,
|
||
"manifestRef": manifest_ref,
|
||
"sourceHash": source_hash,
|
||
"objects": source_objects,
|
||
"bytes": source_bytes,
|
||
}
|
||
objects.sort(key=lambda item: (item["category"], item["path"]))
|
||
if len(objects) > _MAX_OBJECTS:
|
||
raise ArtifactError(f"对象数量超过 {_MAX_OBJECTS}")
|
||
by_category = {
|
||
category: [item for item in objects if item["category"] == category]
|
||
for category in _CATEGORY_ORDER
|
||
}
|
||
bundle = _bundle_object(by_category["runtime"])
|
||
runtime_asset_bytes = sum(item["bytes"] for item in by_category["runtime"] + by_category["asset"])
|
||
if runtime_asset_bytes > _MAX_RUNTIME_ASSET_BYTES:
|
||
raise ArtifactError(f"运行包与素材合计超过 {_MAX_RUNTIME_ASSET_BYTES} 字节")
|
||
manifest = {
|
||
"schemaVersion": SCHEMA_VERSION,
|
||
"tenantId": tenant_id,
|
||
"gameId": game_id,
|
||
"versionId": version_id,
|
||
"packageType": package_type,
|
||
"keyPrefix": key_prefix,
|
||
"sourceRevision": source_revision_record,
|
||
"runtimeManifest": runtime_manifest,
|
||
"objects": objects,
|
||
"totals": {
|
||
"objects": len(objects),
|
||
"bytes": sum(item["bytes"] for item in objects),
|
||
"assetBytes": sum(item["bytes"] for item in by_category["asset"]),
|
||
"runtimeBytes": sum(item["bytes"] for item in by_category["runtime"]),
|
||
"evidenceBytes": sum(item["bytes"] for item in by_category["evidence"]),
|
||
},
|
||
"assetHash": _hash_objects(by_category["asset"]),
|
||
"bundleHash": bundle["sha256"] if bundle else None,
|
||
}
|
||
manifest["manifestHash"] = _hash_bytes(_MANIFEST_HASH_DOMAIN + _canonical_json(manifest))
|
||
if len(_canonical_json(manifest)) > _MAX_MANIFEST_BYTES:
|
||
raise ArtifactError(f"artifact manifest 超过 {_MAX_MANIFEST_BYTES} 字节")
|
||
return manifest
|
||
|
||
|
||
def verify_manifest(manifest: Mapping) -> None:
|
||
"""验证 manifest 自身 hash、对象路径和 totals，失败即拒绝消费。"""
|
||
_validate_contract(manifest, "game-artifact-manifest.schema.json")
|
||
if len(_canonical_json(manifest)) > _MAX_MANIFEST_BYTES:
|
||
raise ArtifactError(f"artifact manifest 超过 {_MAX_MANIFEST_BYTES} 字节")
|
||
if manifest.get("schemaVersion") != SCHEMA_VERSION:
|
||
raise ArtifactError("manifest schemaVersion 不受支持")
|
||
_validate_id(str(manifest.get("tenantId", "")), "tenantId")
|
||
_validate_db_id(str(manifest.get("gameId", "")), "gameId")
|
||
_validate_db_id(str(manifest.get("versionId", "")), "versionId")
|
||
declared = manifest.get("manifestHash")
|
||
if not isinstance(declared, str) or not _HASH_RE.fullmatch(declared):
|
||
raise ArtifactError("manifestHash 非法")
|
||
unsigned = dict(manifest)
|
||
unsigned.pop("manifestHash", None)
|
||
if _hash_bytes(_MANIFEST_HASH_DOMAIN + _canonical_json(unsigned)) != declared:
|
||
raise ArtifactError("manifestHash 不匹配")
|
||
if manifest.get("packageType") not in {"phaser", "littlejs", "canvas"}:
|
||
raise ArtifactError("manifest packageType 不受支持")
|
||
key_prefix = str(manifest.get("keyPrefix", ""))
|
||
if key_prefix != f"tenants/{manifest['tenantId']}/games/{manifest['gameId']}/versions/{manifest['versionId']}":
|
||
raise ArtifactError("manifest keyPrefix 与 tenant/game/version 不匹配")
|
||
source_revision = manifest["sourceRevision"]
|
||
provider = source_revision["provider"]
|
||
if provider not in {"tier2_source_project", "game_source_archive"}:
|
||
raise ArtifactError("sourceRevision.provider 不受支持")
|
||
if provider == "tier2_source_project" and manifest.get("packageType") != "phaser":
|
||
raise ArtifactError("tier2_source_project 只允许 packageType=phaser")
|
||
if provider == "game_source_archive":
|
||
_validate_db_id(str(source_revision["gameId"]), "sourceRevision.gameId")
|
||
expected_source_ref = f"game-source-archive:{source_revision['gameId']}:{source_revision['revisionId']}"
|
||
else:
|
||
_validate_id(str(source_revision["gameId"]), "sourceRevision.gameId")
|
||
expected_source_ref = f"tier2-source-project:{source_revision['gameId']}:{source_revision['revisionId']}"
|
||
if source_revision["manifestRef"] != expected_source_ref:
|
||
raise ArtifactError("sourceRevision.manifestRef 与身份字段不匹配")
|
||
objects = manifest.get("objects") or []
|
||
totals = manifest.get("totals") or {}
|
||
if len(objects) != totals.get("objects"):
|
||
raise ArtifactError("manifest totals.objects 不匹配")
|
||
if len(objects) > _MAX_OBJECTS:
|
||
raise ArtifactError(f"对象数量超过 {_MAX_OBJECTS}")
|
||
seen: set[str] = set()
|
||
seen_sources: set[str] = set()
|
||
sums = {category: 0 for category in _CATEGORY_ORDER}
|
||
grouped: dict[str, list[Mapping]] = {category: [] for category in _CATEGORY_ORDER}
|
||
for item in objects:
|
||
raw_path = str(item.get("path", ""))
|
||
raw_source_path = str(item.get("sourcePath", ""))
|
||
path = _safe_rel(raw_path)
|
||
source_path = _safe_rel(raw_source_path)
|
||
if raw_path != path or raw_source_path != source_path:
|
||
raise ArtifactError(f"manifest 路径必须使用 NFC 规范形式:{raw_source_path}")
|
||
key = str(item.get("key", ""))
|
||
if path in seen:
|
||
raise ArtifactError(f"manifest 存在重复路径:{path}")
|
||
if source_path in seen_sources:
|
||
raise ArtifactError(f"manifest 存在重复源路径:{source_path}")
|
||
seen.add(path)
|
||
seen_sources.add(source_path)
|
||
category = str(item.get("category", ""))
|
||
if category not in _CATEGORY_DIR:
|
||
raise ArtifactError(f"对象 category 非法:{path}")
|
||
expected_path_prefix = f"{_CATEGORY_DIR[category]}/"
|
||
if not path.startswith(expected_path_prefix):
|
||
raise ArtifactError(f"对象 path 与 category 不匹配:{path}")
|
||
if item.get("store") != _CATEGORY_STORE[category]:
|
||
raise ArtifactError(f"对象 store 与 category 不匹配:{path}")
|
||
expected_key = f"{key_prefix}/{path}"
|
||
if key != expected_key:
|
||
raise ArtifactError(f"对象 key 与 path 不匹配:{key}")
|
||
if not isinstance(item.get("bytes"), int) or item["bytes"] < 0:
|
||
raise ArtifactError(f"对象 bytes 非法:{path}")
|
||
if item["bytes"] > _MAX_OBJECT_BYTES:
|
||
raise ArtifactError(f"对象 bytes 超限:{path}")
|
||
if not _HASH_RE.fullmatch(str(item.get("sha256", ""))):
|
||
raise ArtifactError(f"对象 hash 非法:{path}")
|
||
if not isinstance(item.get("mime"), str) or not item["mime"]:
|
||
raise ArtifactError(f"对象 mime 非法:{path}")
|
||
sums[category] += item["bytes"]
|
||
grouped[category].append(item)
|
||
runtime_manifest = manifest.get("runtimeManifest")
|
||
if not isinstance(runtime_manifest, Mapping):
|
||
raise ArtifactError("runtimeManifest 必须是对象")
|
||
raw_runtime_source_path = str(runtime_manifest.get("sourcePath", ""))
|
||
source_path = _safe_rel(raw_runtime_source_path)
|
||
if raw_runtime_source_path != source_path:
|
||
raise ArtifactError("runtimeManifest sourcePath 必须使用 NFC 规范形式")
|
||
if str(runtime_manifest.get("key")) != f"{key_prefix}/manifest.json":
|
||
raise ArtifactError("runtimeManifest key 不匹配")
|
||
if runtime_manifest.get("store") != "artifact":
|
||
raise ArtifactError("runtimeManifest store 必须为 artifact")
|
||
if source_path in seen_sources:
|
||
raise ArtifactError("runtimeManifest 与对象 sourcePath 重复")
|
||
if not isinstance(runtime_manifest.get("bytes"), int) or runtime_manifest["bytes"] < 0:
|
||
raise ArtifactError("runtimeManifest bytes 非法")
|
||
if runtime_manifest["bytes"] > _MAX_OBJECT_BYTES:
|
||
raise ArtifactError("runtimeManifest bytes 超限")
|
||
if not _HASH_RE.fullmatch(str(runtime_manifest.get("sha256", ""))):
|
||
raise ArtifactError("runtimeManifest hash 非法")
|
||
if runtime_manifest.get("mime") != "application/json":
|
||
raise ArtifactError("runtimeManifest mime 必须为 application/json")
|
||
expected_totals = {
|
||
"bytes": sum(sums.values()),
|
||
"assetBytes": sums["asset"],
|
||
"runtimeBytes": sums["runtime"],
|
||
"evidenceBytes": sums["evidence"],
|
||
}
|
||
for name, expected in expected_totals.items():
|
||
if totals.get(name) != expected:
|
||
raise ArtifactError(f"manifest totals.{name} 不匹配")
|
||
if sums["runtime"] + sums["asset"] > _MAX_RUNTIME_ASSET_BYTES:
|
||
raise ArtifactError("运行包与素材总字节超限")
|
||
if manifest.get("assetHash") != _hash_objects(grouped["asset"]):
|
||
raise ArtifactError("manifest assetHash 不匹配")
|
||
bundle = _bundle_object(grouped["runtime"])
|
||
if manifest.get("bundleHash") != (bundle["sha256"] if bundle else None):
|
||
raise ArtifactError("manifest bundleHash 不匹配")
|
||
|
||
|
||
def _minio_client(config: Mapping):
|
||
"""惰性创建 MinIO 客户端；不在 import 期连接网络。"""
|
||
try:
|
||
from minio import Minio
|
||
except Exception as exc: # pragma: no cover - 环境缺 SDK 时由调用方看到稳定错误
|
||
raise ArtifactError(f"缺少 minio Python SDK:{exc}") from exc
|
||
endpoint = str(config.get("endpoint", "")).replace("http://", "").replace("https://", "")
|
||
if not endpoint or not config.get("access_key") or not config.get("secret_key"):
|
||
raise ArtifactError("缺少专用 game artifact 对象存储端点或凭据")
|
||
return Minio(endpoint, access_key=config.get("access_key"), secret_key=config.get("secret_key"),
|
||
secure=bool(config.get("secure", False)))
|
||
|
||
|
||
def _read_remote(client, bucket: str, key: str, *, expected_bytes: int) -> bytes:
|
||
"""按调用方字节预算流式读取；发现第 expected+1 字节就立即拒绝。"""
|
||
if not isinstance(expected_bytes, int) or expected_bytes < 0:
|
||
raise ArtifactError(f"远端对象期望字节数非法:{key}")
|
||
try:
|
||
response = client.get_object(bucket, key)
|
||
except Exception as exc: # noqa: BLE001 - 兼容 MinIO S3Error 和测试 fake
|
||
if getattr(exc, "code", None) == "NoSuchBucket":
|
||
raise ArtifactError(f"对象存储桶不存在:{bucket}") from exc
|
||
raise
|
||
try:
|
||
chunks: list[bytes] = []
|
||
total = 0
|
||
try:
|
||
while True:
|
||
remaining = expected_bytes - total
|
||
chunk = response.read(min(_REMOTE_READ_CHUNK_BYTES, remaining + 1))
|
||
if not chunk:
|
||
break
|
||
total += len(chunk)
|
||
if total > expected_bytes:
|
||
raise ArtifactError(f"远端对象超过期望字节数:{key}:{expected_bytes}")
|
||
chunks.append(chunk)
|
||
return b"".join(chunks)
|
||
except ArtifactError:
|
||
raise
|
||
except Exception as exc: # noqa: BLE001 - SDK/网络读取统一失败语义
|
||
raise ArtifactError(f"读取远端对象失败:{bucket}/{key}:{type(exc).__name__}:{exc}") from exc
|
||
finally:
|
||
response.close()
|
||
response.release_conn()
|
||
|
||
|
||
def _bucket(config: Mapping, store: str) -> str:
|
||
"""把稳定 store 身份映射到环境桶名；manifest 不保存凭据或临时 URL。"""
|
||
if store == "artifact":
|
||
return str(config.get("artifact_bucket", config.get("bucket", "game-artifacts")))
|
||
if store == "evidence":
|
||
return str(config.get("evidence_bucket", "game-evidence"))
|
||
if store == "source":
|
||
source_bucket = str(config.get("source_bucket", ""))
|
||
if not source_bucket:
|
||
raise ArtifactError("缺少显式 source_bucket；game-sources 尚未完成权限配置")
|
||
if source_bucket in {
|
||
str(config.get("artifact_bucket", "")),
|
||
str(config.get("evidence_bucket", "")),
|
||
}:
|
||
raise ArtifactError("source_bucket 不能与 artifact/evidence 桶复用")
|
||
return source_bucket
|
||
raise ArtifactError(f"未知对象 store:{store}")
|
||
|
||
|
||
def _existing_manifest(client, bucket: str, key: str) -> dict | None:
|
||
"""读取已存在的版本索引；仅把 S3 的对象不存在视为未上传。"""
|
||
try:
|
||
payload = _read_remote(client, bucket, key, expected_bytes=_MAX_MANIFEST_BYTES)
|
||
except ArtifactError:
|
||
raise
|
||
except Exception as exc: # noqa: BLE001 - 兼容 MinIO S3Error 和测试 fake
|
||
if getattr(exc, "code", None) in {"NoSuchKey", "NoSuchObject"}:
|
||
return None
|
||
raise ArtifactError(f"探测远端 artifact-manifest 失败:{type(exc).__name__}:{exc}") from exc
|
||
try:
|
||
existing = json.loads(payload.decode("utf-8"))
|
||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||
raise ArtifactError("远端 artifact-manifest 已存在但不是合法 JSON") from exc
|
||
verify_manifest(existing)
|
||
return existing
|
||
|
||
|
||
def _verify_remote_object(client, bucket: str, key: str, expected_hash: str, expected_bytes: int) -> None:
|
||
"""用 GET 字节校验远端对象；不把 HEAD/ETag 当作内容完整性证据。"""
|
||
try:
|
||
data = _read_remote(client, bucket, key, expected_bytes=expected_bytes)
|
||
except ArtifactError:
|
||
raise
|
||
except Exception as exc: # noqa: BLE001 - 缺对象/SDK 错误统一成稳定失败
|
||
raise ArtifactError(f"读取远端对象失败:{bucket}/{key}:{type(exc).__name__}:{exc}") from exc
|
||
if len(data) != expected_bytes or _hash_bytes(data) != expected_hash:
|
||
raise ArtifactError(f"上传后对象校验失败:{key}")
|
||
|
||
|
||
def _put_remote(client, bucket: str, key: str, stream, *, length: int, content_type: str) -> None:
|
||
"""统一包装 S3 Put 异常，CLI 只暴露稳定 ArtifactError。"""
|
||
try:
|
||
client.put_object(bucket, key, stream, length=length, content_type=content_type)
|
||
except ArtifactError:
|
||
raise
|
||
except Exception as exc: # noqa: BLE001 - 兼容 MinIO S3Error/网络异常
|
||
raise ArtifactError(f"对象上传失败:{bucket}/{key}:{type(exc).__name__}:{exc}") from exc
|
||
|
||
|
||
def upload_manifest(root: Path, manifest: Mapping, config: Mapping, *,
|
||
external_single_writer: bool = False) -> dict:
|
||
"""在外部单写锁内上传并校验对象；波次 1 只返回 verified_pending。"""
|
||
verify_manifest(manifest)
|
||
if external_single_writer is not True:
|
||
raise ArtifactError("波次 1 上传必须由调用方持有外部单写锁")
|
||
client = _minio_client(config)
|
||
buckets = {store: _bucket(config, store) for store in ("artifact", "evidence")}
|
||
root = Path(root).resolve()
|
||
manifest_key = f"{manifest['keyPrefix']}/artifact-manifest.json"
|
||
existing = _existing_manifest(client, buckets["artifact"], manifest_key)
|
||
if existing is not None:
|
||
if existing["manifestHash"] == manifest["manifestHash"]:
|
||
for item in manifest["objects"]:
|
||
_verify_remote_object(client, buckets[item["store"]], item["key"],
|
||
item["sha256"], item["bytes"])
|
||
runtime_manifest = manifest["runtimeManifest"]
|
||
_verify_remote_object(client, buckets["artifact"], runtime_manifest["key"],
|
||
runtime_manifest["sha256"], runtime_manifest["bytes"])
|
||
return {
|
||
"artifactBucket": buckets["artifact"],
|
||
"artifactManifestKey": manifest_key,
|
||
"manifestHash": manifest["manifestHash"],
|
||
"idempotent": True,
|
||
"status": "verified_pending",
|
||
"committed": False,
|
||
}
|
||
raise ArtifactError("同一 gameId/versionId 已存在不同 artifact-manifest，拒绝覆盖")
|
||
# 先把所有正文和 GamePackage 做完本地预检，再开始任何远端 Put，避免后一个文件漂移
|
||
# 时留下没有数据库 pending 记录的半上传对象。
|
||
local_objects: list[tuple[Mapping, Path, int]] = []
|
||
for item in manifest["objects"]:
|
||
package_path = str(item["path"])
|
||
source = root / _safe_rel(str(item["sourcePath"]))
|
||
if source.is_symlink() or not source.is_file():
|
||
raise ArtifactError(f"manifest 对应本地文件不存在:{package_path}")
|
||
size, sha256 = _file_digest(source)
|
||
if size != item["bytes"]:
|
||
raise ArtifactError(f"本地对象字节数漂移:{package_path}")
|
||
if sha256 != item["sha256"]:
|
||
raise ArtifactError(f"本地对象 hash 漂移:{package_path}")
|
||
local_objects.append((item, source, size))
|
||
runtime_manifest = manifest["runtimeManifest"]
|
||
source = root / _safe_rel(str(runtime_manifest["sourcePath"]))
|
||
if source.is_symlink() or not source.is_file():
|
||
raise ArtifactError("GamePackage manifest 本地文件不存在")
|
||
payload = source.read_bytes()
|
||
if len(payload) != runtime_manifest["bytes"] or _hash_bytes(payload) != runtime_manifest["sha256"]:
|
||
raise ArtifactError("GamePackage manifest 本地 hash 漂移")
|
||
_validate_game_package(payload, str(manifest["gameId"]), str(manifest["versionId"]))
|
||
# 所有本地输入都已验证，以下才产生远端副作用。
|
||
for item, source, size in local_objects:
|
||
with source.open("rb") as stream:
|
||
_put_remote(client, buckets[item["store"]], item["key"], stream, length=size,
|
||
content_type=item["mime"])
|
||
_put_remote(client, buckets["artifact"], runtime_manifest["key"], BytesIO(payload), length=len(payload),
|
||
content_type="application/json")
|
||
# 先对正文对象做 GET 全量复算，再写 marker；marker 仍不是数据库 CAS 提交记录。
|
||
for item in manifest["objects"]:
|
||
_verify_remote_object(client, buckets[item["store"]], item["key"], item["sha256"], item["bytes"])
|
||
_verify_remote_object(client, buckets["artifact"], runtime_manifest["key"],
|
||
runtime_manifest["sha256"], runtime_manifest["bytes"])
|
||
payload = _canonical_json(manifest)
|
||
_put_remote(client, buckets["artifact"], manifest_key, BytesIO(payload), length=len(payload),
|
||
content_type="application/json")
|
||
remote_bytes = _read_remote(
|
||
client, buckets["artifact"], manifest_key, expected_bytes=len(payload),
|
||
)
|
||
if remote_bytes != payload:
|
||
raise ArtifactError("上传后 manifest 下载 hash 不匹配")
|
||
return {
|
||
"artifactBucket": buckets["artifact"],
|
||
"artifactManifestKey": manifest_key,
|
||
"manifestHash": manifest["manifestHash"],
|
||
"idempotent": False,
|
||
"status": "verified_pending",
|
||
"committed": False,
|
||
}
|
||
|
||
|
||
def fetch_manifest(config: Mapping, *, key: str, expected_manifest_hash: str,
|
||
expected_tenant_id: str, expected_game_id: str, expected_version_id: str) -> dict:
|
||
"""从对象存储读取 artifact-manifest，不接受任意 URL，调用方需传入受信 key。"""
|
||
client = _minio_client(config)
|
||
bucket = _bucket(config, "artifact")
|
||
if not _ARTIFACT_MANIFEST_KEY_RE.fullmatch(key):
|
||
raise ArtifactError("artifact-manifest key 非法")
|
||
if not _HASH_RE.fullmatch(expected_manifest_hash):
|
||
raise ArtifactError("expectedManifestHash 非法")
|
||
_validate_id(expected_tenant_id, "expectedTenantId")
|
||
_validate_db_id(expected_game_id, "expectedGameId")
|
||
_validate_db_id(expected_version_id, "expectedVersionId")
|
||
payload = _read_remote(client, bucket, key, expected_bytes=_MAX_MANIFEST_BYTES)
|
||
try:
|
||
manifest = json.loads(payload.decode("utf-8"))
|
||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||
raise ArtifactError("远端 artifact-manifest 不是合法 JSON") from exc
|
||
verify_manifest(manifest)
|
||
if manifest["manifestHash"] != expected_manifest_hash:
|
||
raise ArtifactError("artifact-manifest 与数据库期望 hash 不匹配")
|
||
if (manifest["tenantId"], manifest["gameId"], manifest["versionId"]) != (
|
||
expected_tenant_id, expected_game_id, expected_version_id,
|
||
):
|
||
raise ArtifactError("artifact-manifest 与数据库期望身份不匹配")
|
||
if key != f"{manifest['keyPrefix']}/artifact-manifest.json":
|
||
raise ArtifactError("artifact-manifest key 与内容身份不匹配")
|
||
return manifest
|
||
|
||
|
||
def download_manifest(manifest: Mapping, config: Mapping, target: Path, *, expected_manifest_hash: str) -> Path:
|
||
"""按可信 hash 物化到临时目录，全量校验成功后原子交付。"""
|
||
verify_manifest(manifest)
|
||
if manifest["manifestHash"] != expected_manifest_hash:
|
||
raise ArtifactError("物化 manifest 与数据库期望 hash 不匹配")
|
||
client = _minio_client(config)
|
||
target = Path(target).resolve()
|
||
if target.exists():
|
||
raise ArtifactError(f"物化目标已存在:{target}")
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
staging = Path(tempfile.mkdtemp(prefix=f".{target.name}.partial-", dir=target.parent)).resolve()
|
||
try:
|
||
for item in manifest["objects"]:
|
||
rel = _safe_rel(str(item["path"]))
|
||
destination = (staging / rel).resolve()
|
||
if staging not in destination.parents:
|
||
raise ArtifactError(f"下载路径越界:{rel}")
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
data = _read_remote(
|
||
client, _bucket(config, str(item["store"])), item["key"],
|
||
expected_bytes=item["bytes"],
|
||
)
|
||
if len(data) != item["bytes"] or _hash_bytes(data) != item["sha256"]:
|
||
raise ArtifactError(f"下载对象 hash 不匹配:{rel}")
|
||
destination.write_bytes(data)
|
||
runtime_manifest = manifest["runtimeManifest"]
|
||
destination = staging / "manifest.json"
|
||
data = _read_remote(
|
||
client, _bucket(config, "artifact"), runtime_manifest["key"],
|
||
expected_bytes=runtime_manifest["bytes"],
|
||
)
|
||
if len(data) != runtime_manifest["bytes"] or _hash_bytes(data) != runtime_manifest["sha256"]:
|
||
raise ArtifactError("下载 GamePackage manifest hash 不匹配")
|
||
_validate_game_package(data, str(manifest["gameId"]), str(manifest["versionId"]))
|
||
destination.write_bytes(data)
|
||
(staging / "artifact-manifest.json").write_bytes(_canonical_json(manifest))
|
||
staging.replace(target)
|
||
except Exception:
|
||
shutil.rmtree(staging, ignore_errors=True)
|
||
raise
|
||
return target
|