diff --git a/.gitignore b/.gitignore index 8de90754..97c2001c 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,10 @@ coverage/ .DS_Store .gstack/ +# ── 本地个人环境配置：仅本机生效，绝不入库（机器画像 / 工具链 / 本地端口属个人配置）── +# 与 AGENTS.md §10 配套：AGENTS.md 用 @localagents.md 可选引用，文件缺失不影响共享文档。 +localagents.md + # ── 例外 1：源码树（src/）下名为 build 的目录是合法源码，绝非构建产物 ── # 第 5 行笼统的 build/（为构建产物设计）会误吞 Java/前端源码中名为 build 的包/目录，例如： # 后端 com.wanxiang.huijing.game.module.runtime.{dal.dataobject,dal.mysql,service}.build（编译任务域，4 文件）、 @@ -62,38 +66,36 @@ __pycache__/ # __pycache__/ 与 node_modules/ 已由上方全局规则拦截；新 spike 原始 json/png 默认 untracked、勿 git add 批量产物。 # 注意（遵 gitignore-swallows-frontend-source 教训）：此处不设 *.json/*.png 笼统忽略，以免误吞 contracts/ 等业务目录下的合法同类文件——分层靠规范 + 收口第 8 步执行，不靠笼统 glob。 -# ── AI 生成 / e2e 批跑游戏产物：可重生成、本地跑出、不入库（2026-06-25）── -# game-runtime/games/_wg1-gen/ 下「精选参考游戏」(angrybirds/breakout/... 无前缀，含 _shared/_goldenpath-*)已跟踪、须保留； -# 下列是 agent 生成线与 e2e 批跑的输出（每次跑批产生上百个时间戳目录），纯产物、可重生成，长期污染 git diff，故忽略。 -# 仍遵「专一不笼统」：只忽略已知的生成/批跑前缀，绝不忽略整个 _wg1-gen（会吞掉已跟踪的参考游戏）。 -game-runtime/games/_wg1-gen/saa-saa-e2e-*/ -game-runtime/games/_wg1-gen/gd-*/ -game-runtime/games/_spike-local-verify/ -game-runtime/games/_amodel-spike/ +# ── 游戏内容存储边界：Git 只放平台代码，具体游戏全部进入对象存储（2026-07-29）── +# 具体游戏的源码、素材、bundle、证据和参照游戏都不再作为平台源码进入 Git。 +# 下列豁免只包含通用生成壳、平台模板和共享工具；它们不绑定任何一款具体游戏。 +# 已跟踪的历史游戏不会因 ignore 自动消失，必须在对象存储消费切换和下载对账后独立删除。 +/game-runtime/games/* +!/game-runtime/games/_generic/ +!/game-runtime/games/_generic/** +!/game-runtime/games/_template/ +!/game-runtime/games/_template/** +!/game-runtime/games/_template-feiyi/ +!/game-runtime/games/_template-feiyi/** +!/game-runtime/games/_template-puzzle/ +!/game-runtime/games/_template-puzzle/** +!/game-runtime/games/_template-shop/ +!/game-runtime/games/_template-shop/** +!/game-runtime/games/_template-story/ +!/game-runtime/games/_template-story/** +!/game-runtime/games/_template-trpg/ +!/game-runtime/games/_template-trpg/** +!/game-runtime/games/_wg1-gen/ +/game-runtime/games/_wg1-gen/* +!/game-runtime/games/_wg1-gen/_shared/ +!/game-runtime/games/_wg1-gen/_shared/** +/game-runtime/games/_wg1-gen/_shared/**/__pycache__/ +!/game-runtime/games/_wg1-gen/.agent +/tier2/games/* -# ── cheap-worker 便宜档 Python spike:venv 与生成/对照产物（可重生成、不入库，2026-06-26）── -# 遵「专一不笼统」：只忽略 cheap-worker 自己的 venv 与本 spike 的 cheap-*/node-* 产物前缀，不碰已跟踪的参考游戏。 +# cheap-worker 本地虚拟环境与批跑结果不入库。 cheap-worker/.venv/ cheap-worker/results/ -game-runtime/games/amgen-cheap-*/ -game-runtime/games/amgen-node-*/ -game-runtime/games/_wg1-gen/cheap-*/ -game-runtime/games/_wg1-gen/node-*/ - -# ── M1 达标门 bake-off + 代表样本/校验重生成产物（可重生成、不入库,2026-06-27）── -# 遵「专一不笼统」：只忽略 bake-off/代表样本/校验 的已知生成前缀(bake-/base[0-9]/vf-/v2-/smoke)。 -# 已跟踪参考游戏(angrybirds/breakout/t2048/whack/...)与 _shared/_goldenpath-* 均不以这些前缀开头、不受影响。 -game-runtime/games/amgen-bake-*/ -game-runtime/games/_wg1-gen/bake-*/ -game-runtime/games/amgen-base[0-9]*/ -game-runtime/games/_wg1-gen/base[0-9]*/ -game-runtime/games/amgen-vf-*/ -game-runtime/games/_wg1-gen/vf-*/ -game-runtime/games/amgen-v2-*/ -game-runtime/games/amgen-smoke*/ -# M2 auto-vs-golden 同款双驱动批跑产物(avg- 前缀,可重生成、不入库) -game-runtime/games/amgen-avg-*/ -game-runtime/games/_wg1-gen/avg-*/ # ── 本机个人配置:localagents.md 自声明"绝不提交、已在 .gitignore 忽略",但此前无对应忽略行(声明未被机器执行);2026-07-06 补真 ── /localagents.md diff --git a/contracts/README.md b/contracts/README.md index 4ec70c8f..88f5bada 100644 --- a/contracts/README.md +++ b/contracts/README.md @@ -13,13 +13,14 @@ | 1 | API OpenAPI | `api-schemas/*.yaml` | WS1 lead（全员 review） | 前端 mock / 各模块互调 | | 2 | DB 迁移 | `db-schemas/V*.sql` | WS1 | 后端各模块 | | 3 | SDK 接口 + postMessage 协议 | `sdk-interface.d.ts` | WS3 SDK 负责人 | game-studio 宿主 / 游戏侧 | -| 4 | GamePackage 清单 | `game-package.schema.json` | WS3 + WS2 | 生成→编译→预览→发布→运行全链路 | +| 4 | GamePackage 运行清单 + 对象存储索引 | `game-package.schema.json` + `game-artifact-manifest.schema.json` + `game-source-archive.schema.json` | WS3 + WS2 | 生成→编译→源归档→制品存储→预览→发布→运行全链路 | | 5 | telemetry 事件 | `events.schema.json` | WS5 | 前端埋点 / SDK 上报 / 看板 | | 6 | ~~Dify workflow I/O~~ ⚠️ 降级远期未部署 | `dify-workflow-io.json`（见 [`DEPRECATED-dify-workflow-io.md`](./DEPRECATED-dify-workflow-io.md)） | WS2 | ~~aigc Java 壳 ↔ Dify~~（现行=new-api 网关，C2/HJ-GEN-001；JSON 本体保留作历史/远期） | | 7 | 广告位配置 | `ad-slot.schema.json` | WS5 | ad 模块 / SDK Plugin.Ad | | 8 | Prompt Registry | `prompts/`（独立目录） | 各 prompt owner 工位 | aigc / 生成链路（见 [`prompt-governance`](../.agents/skills/prompt-governance.md)） | > **M0 八类契约已锁**（JSON/YAML/Schema 语法校验通过）：#1 project.yaml · #2 V1.0.0__…sql · #3 sdk-interface.d.ts · #4 game-package · #5 events · #6 dify-workflow-io · #7 ad-slot · #8 prompts/。 +> **#4 对象存储伴随契约（2026-07-29）**：`game-artifact-manifest.schema.json` 只描述素材、运行包、证据对象及源修订身份，不是宿主读取的 GamePackage；`game-source-archive.schema.json` 单独描述引擎无关源码归档。生产 `gameId/versionId` 只接受 project 数据库正整数 ID；Tier2 字符串版本只进入 Tier2 `sourceRevision.revisionId`，LittleJS/Canvas 等游戏使用 `game_source_archive` provider，不得伪装成 Phaser Tier2。发布状态继续由 `game_runtime_package.status` 管理，不写入不可变对象清单。 > **Wave1 脊柱 4 模块契约已补并锁**（aigc/runtime/feed/telemetry，2026-06-08）：经契约先行起草 + 一致性评审 + 收口（§2.1）+ 主 agent 校验（YAML 语法/裸 select* 0/错误码段 101-104 独占/Flyway V2-5 唯一）。 > **Wave2 变现契约已补并锁**（ad/trade，2026-06-08）：`ad.yaml`+`trade.yaml`（API #1）+ `V6.0.0`/`V7.0.0`（DB #2），消费既有 #7 ad-slot / #5 events 不重造；经主 agent 校验（YAML 语法/错误码 111·106 独占/Flyway V6-7 唯一/金额用「分」/资金幂等键+状态机/ad↔trade Feign seam 一致）；ad 计费 `uk_trace` 由主 agent 收紧为 `(trace_id,event_type,tenant_id)` 防 reward 被 impression 静默吞掉。**pay=复用 huijing-pay 后置**（收单 P1）。 > **Wave3 跨主线收口（进行中）**：新增 `studio`（创作主链路最小版编排，错误码 112 / Flyway V8）+ `compliance`（锁风门 Gate，发布前合规扫描）两模块契约。Phase A 已收口 project 侧：publish 唯一化（返回 admitted+gates 门禁聚合）、ProjectApi seam 定稿（getCurrentVersionId GAP-3 / createProject 写类 RPC）、GAP-4 把 `events.schema.json` 信封 envelope/user 由 snake_case 统一为 camelCase（与 telemetry.yaml/前端 SDK 对齐）。compliance gate 在 project.submitPublish 内仅留 Phase C 接入 seam（TODO 注释），尚未接 ComplianceGateApi。其余模块（community/ip/biz）待后续波次补。 @@ -52,8 +53,11 @@ contracts/ │ ├── V5.0.0__create_game_telemetry.sql │ ├── V6.0.0__create_game_ad.sql # ad（广告位 + 收入台账，Wave2） │ ├── V7.0.0__create_game_trade.sql # trade（账户 + 流水 + 提现，Wave2） -│ └── V8.0.0__create_game_studio.sql # studio（创作主链路编排状态，Wave3） +│ ├── V8.0.0__create_game_studio.sql # studio（创作主链路编排状态，Wave3） +│ └── V34.0.0__create_game_artifact_storage.sql # runtime/storage（对象清单 pending/committed 状态） ├── game-package.schema.json # #4 GamePackage 清单（已落地） +├── game-artifact-manifest.schema.json # #4 运行包/素材/证据对象索引（不替代 GamePackage） +├── game-source-archive.schema.json # #4 引擎无关源工程归档索引（独立 source 桶） ├── events.schema.json # #5 telemetry 事件 v1（已落地） ├── sdk-interface.d.ts # #3 已锁（SDK API + postMessage 协议） ├── dify-workflow-io.json # #6 ⚠️ 降级远期未部署（见 DEPRECATED-dify-workflow-io.md；JSON 保留作历史，现行=new-api 网关） diff --git a/contracts/db-schemas/V34.0.0__create_game_artifact_storage.sql b/contracts/db-schemas/V34.0.0__create_game_artifact_storage.sql new file mode 100644 index 00000000..8ba0a998 --- /dev/null +++ b/contracts/db-schemas/V34.0.0__create_game_artifact_storage.sql @@ -0,0 +1,43 @@ +-- ============================================================================= +-- 契约 #2 DB 迁移 | 模块：runtime/storage（对象存储版本状态）| owner：WS3 + WS2 +-- 文件：V34.0.0__create_game_artifact_storage.sql +-- 目的：保存对象存储清单的桶、key、hash、源修订身份和 pending/committed 状态。 +-- 边界：本表只负责对象存储提交状态；运行包自身的取包状态仍由 runtime 表管理。 +-- ============================================================================= + +CREATE TABLE `game_artifact_storage` ( + `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT '对象存储记录 ID', + `tenant_id` BIGINT NOT NULL DEFAULT 0 COMMENT '租户 ID', + `game_id` BIGINT NOT NULL COMMENT '游戏项目 ID（project.game_project.id）', + `version_id` BIGINT NOT NULL COMMENT '游戏版本 ID（project.game_version.id）', + `artifact_bucket` VARCHAR(128) NOT NULL DEFAULT '' COMMENT '运行包/素材/证据清单所在桶', + `artifact_manifest_key` VARCHAR(512) NOT NULL DEFAULT '' COMMENT 'artifact-manifest.json 对象 key', + `artifact_manifest_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT 'artifact-manifest.json 原始字节 SHA-256', + `artifact_manifest_bytes` BIGINT NOT NULL DEFAULT 0 COMMENT 'artifact-manifest.json 字节数', + `runtime_manifest_key` VARCHAR(512) NOT NULL DEFAULT '' COMMENT 'GamePackage manifest.json 对象 key', + `runtime_manifest_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT 'GamePackage manifest.json 原始字节 SHA-256；与 runtime checksum 对账', + `source_bucket` VARCHAR(128) NOT NULL DEFAULT '' COMMENT '引擎无关源归档所在桶；Tier2 外部源修订可为空', + `source_provider` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '源修订 provider：tier2_source_project/game_source_archive', + `source_game_id` VARCHAR(128) NOT NULL DEFAULT '' COMMENT '源修订所属稳定标识', + `source_revision_id` VARCHAR(128) NOT NULL DEFAULT '' COMMENT '源修订不可变标识', + `source_manifest_key` VARCHAR(512) NOT NULL DEFAULT '' COMMENT 'GameSourceArchive source-manifest.json key', + `source_manifest_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT 'source-manifest.json 原始字节 SHA-256', + `source_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT '源文件树 SHA-256', + `package_type` VARCHAR(16) NOT NULL DEFAULT '' COMMENT '运行包形态：phaser/littlejs/canvas', + `object_count` INT NOT NULL DEFAULT 0 COMMENT 'artifact manifest 正文对象数量', + `total_bytes` BIGINT NOT NULL DEFAULT 0 COMMENT '运行包、素材和证据对象总字节数', + `asset_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT '素材 tree hash', + `bundle_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT '运行 bundle SHA-256；无 bundle 时为空', + `status` VARCHAR(16) NOT NULL DEFAULT 'pending' COMMENT '对象状态：pending=已上传并回读校验；committed=数据库 CAS 提交；abandoned=失败回收', + `failure_reason` VARCHAR(500) NOT NULL DEFAULT '' COMMENT '失败或回收原因，正常状态为空', + `committed_at` DATETIME NULL COMMENT '数据库 CAS 成功时间', + `creator` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '创建者', + `create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `updater` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '更新者', + `update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `deleted` BIT(1) NOT NULL DEFAULT b'0' COMMENT '逻辑删除标志；不可用它替换同一 game/version 记录', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_tenant_game_version` (`tenant_id`, `game_id`, `version_id`) COMMENT '同一租户游戏版本不可绑定第二份对象清单', + KEY `idx_status_created` (`status`, `create_time`) COMMENT '扫描 pending/abandoned 记录', + KEY `idx_manifest_hash` (`artifact_manifest_hash`) COMMENT '按清单 hash 查询幂等记录' +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT = '游戏对象存储版本状态记录（与运行包状态分离）'; diff --git a/contracts/game-artifact-manifest.schema.json b/contracts/game-artifact-manifest.schema.json new file mode 100644 index 00000000..c3f0a654 --- /dev/null +++ b/contracts/game-artifact-manifest.schema.json @@ -0,0 +1,107 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://wanxiang.ai/contracts/game-artifact-manifest.schema.json", + "title": "GameArtifactManifest", + "description": "具体游戏版本的对象存储索引。它不是 GamePackage/1 运行时 manifest；运行时仍读取同一版本前缀下的 manifest.json，状态由 game_runtime_package.status 管理。实际游戏内容只存 MinIO/S3。", + "type": "object", + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "sourceRevision": { + "properties": { "provider": { "const": "tier2_source_project" } } + } + } + }, + "then": { + "properties": { "packageType": { "const": "phaser" } } + } + } + ], + "required": [ + "schemaVersion", + "tenantId", + "gameId", + "versionId", + "packageType", + "keyPrefix", + "sourceRevision", + "runtimeManifest", + "objects", + "totals", + "assetHash", + "bundleHash", + "manifestHash" + ], + "properties": { + "schemaVersion": { "const": "GameArtifactManifest/1", "type": "string" }, + "tenantId": { "type": "string", "minLength": 1, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" }, + "gameId": { "description": "project.game_project.id 的十进制字符串", "type": "string", "pattern": "^[1-9][0-9]*$" }, + "versionId": { "description": "project.game_version.id 的十进制字符串", "type": "string", "pattern": "^[1-9][0-9]*$" }, + "packageType": { "type": "string", "enum": ["phaser", "littlejs", "canvas"] }, + "keyPrefix": { "type": "string", "pattern": "^tenants/[A-Za-z0-9][A-Za-z0-9._-]*/games/[1-9][0-9]*/versions/[1-9][0-9]*$" }, + "sourceRevision": { + "description": "源工程由既有 Tier2 SourceProjectStore 或引擎无关 GameSourceArchive 持有；运行制品只绑定其不可变身份，不复制源文件。", + "type": "object", + "additionalProperties": false, + "required": ["provider", "gameId", "revisionId", "manifestRef", "sourceHash", "objects", "bytes"], + "properties": { + "provider": { "type": "string", "enum": ["tier2_source_project", "game_source_archive"] }, + "gameId": { "type": "string", "minLength": 1, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" }, + "revisionId": { "type": "string", "minLength": 1, "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" }, + "manifestRef": { "type": "string", "minLength": 1, "maxLength": 1024, "pattern": "^(tier2-source-project|game-source-archive):[^:]+:[^:]+$" }, + "sourceHash": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "objects": { "type": "integer", "minimum": 1, "maximum": 10000 }, + "bytes": { "type": "integer", "minimum": 1 } + } + }, + "runtimeManifest": { + "description": "现有 GamePackage/1 manifest.json 的稳定对象身份；不能把 artifact-manifest.json 当作该运行时清单。", + "type": "object", + "additionalProperties": false, + "required": ["store", "key", "sourcePath", "bytes", "sha256", "mime"], + "properties": { + "store": { "const": "artifact", "type": "string" }, + "key": { "type": "string", "minLength": 1 }, + "sourcePath": { "type": "string", "minLength": 1, "pattern": "^(?!/)(?!.*(?:^|/)\\.{1,2}(?:/|$))(?!.*\\\\).+$" }, + "bytes": { "type": "integer", "minimum": 0, "maximum": 104857600 }, + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "mime": { "type": "string", "minLength": 1 } + } + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["category", "store", "path", "sourcePath", "key", "bytes", "sha256", "mime"], + "properties": { + "category": { "type": "string", "enum": ["asset", "runtime", "evidence"] }, + "store": { "type": "string", "enum": ["artifact", "evidence"] }, + "path": { "type": "string", "minLength": 1, "pattern": "^(assets|runtime|evidence)/[^/].*" }, + "sourcePath": { "type": "string", "minLength": 1, "pattern": "^(?!/)(?!.*(?:^|/)\\.{1,2}(?:/|$))(?!.*\\\\).+$" }, + "key": { "type": "string", "minLength": 1, "pattern": "^tenants/.+/games/[1-9][0-9]*/versions/[1-9][0-9]*/(assets|runtime|evidence)/[^/].*" }, + "bytes": { "type": "integer", "minimum": 0, "maximum": 104857600 }, + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "mime": { "type": "string", "minLength": 1 } + } + } + }, + "totals": { + "type": "object", + "additionalProperties": false, + "required": ["objects", "bytes", "assetBytes", "runtimeBytes", "evidenceBytes"], + "properties": { + "objects": { "type": "integer", "minimum": 0 }, + "bytes": { "type": "integer", "minimum": 0 }, + "assetBytes": { "type": "integer", "minimum": 0 }, + "runtimeBytes": { "type": "integer", "minimum": 0 }, + "evidenceBytes": { "type": "integer", "minimum": 0 } + } + }, + "assetHash": { "description": "SHA-256(GameArtifactTreeHash/1\\0 + canonical JSON entries)", "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "bundleHash": { "type": ["string", "null"], "pattern": "^[a-f0-9]{64}$" }, + "manifestHash": { "description": "SHA-256(GameArtifactManifest/1\\0 + 去除 manifestHash 后的 UTF-8、NFC 路径、键排序紧凑 JSON)", "type": "string", "pattern": "^[a-f0-9]{64}$" } + } +} diff --git a/contracts/game-source-archive.schema.json b/contracts/game-source-archive.schema.json new file mode 100644 index 00000000..11dd57a2 --- /dev/null +++ b/contracts/game-source-archive.schema.json @@ -0,0 +1,103 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://wanxiang.ai/contracts/game-source-archive.schema.json", + "title": "GameSourceArchive", + "description": "引擎无关的游戏源工程对象清单。源代码和源配置进入独立 game-sources 桶；运行包、发布素材和验收证据仍由 GameArtifactManifest/1 管理。", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "tenantId", + "gameId", + "revisionId", + "engine", + "keyPrefix", + "objects", + "totals", + "sourceHash", + "manifestHash" + ], + "properties": { + "schemaVersion": { "const": "GameSourceArchive/1", "type": "string" }, + "tenantId": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "gameId": { + "description": "project.game_project.id 的十进制字符串。", + "type": "string", + "pattern": "^[1-9][0-9]*$" + }, + "revisionId": { + "description": "源工程不可变修订标识，不承载路径分隔符。", + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "engine": { + "description": "生产源工程使用的引擎或宿主描述，只作元数据，不把非 Phaser 游戏伪装成 Tier2。", + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._+-]*$" + }, + "keyPrefix": { + "type": "string", + "pattern": "^tenants/[A-Za-z0-9][A-Za-z0-9._-]*/games/[1-9][0-9]*/source-revisions/[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "objects": { + "type": "array", + "minItems": 1, + "maxItems": 10000, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sourcePath", "key", "bytes", "sha256", "mime"], + "properties": { + "path": { + "type": "string", + "minLength": 1, + "pattern": "^source/[^/].*" + }, + "sourcePath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?!.*(?:^|/)\\.{1,2}(?:/|$))(?!.*\\\\).+$" + }, + "key": { + "type": "string", + "minLength": 1, + "pattern": "^tenants/.+/games/[1-9][0-9]*/source-revisions/[A-Za-z0-9][A-Za-z0-9._-]*/source/[^/].*" + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 104857600 + }, + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "mime": { "type": "string", "minLength": 1 } + } + } + }, + "totals": { + "type": "object", + "additionalProperties": false, + "required": ["objects", "bytes"], + "properties": { + "objects": { "type": "integer", "minimum": 0, "maximum": 10000 }, + "bytes": { "type": "integer", "minimum": 1 } + } + }, + "sourceHash": { + "description": "SHA-256(GameSourceArchiveTreeHash/1\\0 + 按 NFC 路径排序的 path/bytes/sha256 规范 JSON)。", + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "manifestHash": { + "description": "SHA-256(GameSourceArchive/1\\0 + 去除 manifestHash 后的规范 JSON)。", + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + } +} diff --git a/docs/architecture/架构/契约总览.md b/docs/architecture/架构/契约总览.md index 3ccb6eb2..e6f744f1 100644 --- a/docs/architecture/架构/契约总览.md +++ b/docs/architecture/架构/契约总览.md @@ -43,7 +43,7 @@ flowchart TB | **跨仓 #1 API** | `contracts/api-schemas/*.yaml`(12 个:aigc/ad/biz/community/compliance/feed/passport/project/runtime/studio/telemetry/trade) | WS1 lead | 后端定义 → 前端 mock / 各模块互调 | 已建 | | **跨仓 #2 DB** | `contracts/db-schemas/V*.sql` | WS1 | 后端各模块 | 已建(镜像已漂移,见下节) | | **跨仓 #3 SDK** | `contracts/sdk-interface.d.ts` | WS3 | game-studio 宿主 ↔ 游戏侧 postMessage | 已建 | -| **跨仓 #4 GamePackage** | `contracts/game-package.schema.json` | WS3 + WS2 | 生成 → 编译 → 预览 → 发布 → 运行全链路 | 已建 | +| **跨仓 #4 GamePackage** | `contracts/game-package.schema.json` + `contracts/game-artifact-manifest.schema.json` + `contracts/game-source-archive.schema.json` | WS3 + WS2 | 生成 → 编译 → 源归档/对象存储 → 预览 → 发布 → 运行全链路 | GamePackage 已建；制品索引和引擎无关源归档契约已建、生产接线分波迁移 | | **跨仓 #5 events** | `contracts/events.schema.json` | WS5 | 前端埋点 / SDK 上报 → 看板 | 已建 | | **跨仓 #6 Dify I/O** | `contracts/dify-workflow-io.json` | WS2 | ~~aigc 壳 ↔ Dify~~ | **废**(降远期 / 从未部署;现行=new-api,见 `DEPRECATED-dify-workflow-io.md`) | | **跨仓 #7 ad-slot** | `contracts/ad-slot.schema.json` | WS5 | ad 模块 / SDK Plugin.Ad | 已建 | @@ -56,6 +56,8 @@ flowchart TB `templates/generic.schema.json` 有个反直觉的约束:P3 走引擎 bundle 路后,后端对 GameConfig 玩法字段的校验已退场,但这 13 个 schema 文件仍须就位 —— 缺一个,`PromptResourceLoader.isReady()` 就因 `getAllTemplateSchemaTexts` 链加载不到资源而卡在 false([generic.schema.json:5](../../../contracts/templates/generic.schema.json))。退场不等于可删。 +#4 内部有三份用途互斥的 JSON。`game-package.schema.json` 是宿主真正解析的运行清单，版本前缀下文件名仍为 `manifest.json`；`game-artifact-manifest.schema.json` 是平台上传、下载和逐文件校验使用的运行包/素材/证据对象索引，文件名固定为 `artifact-manifest.json`；`game-source-archive.schema.json` 是引擎无关源码归档索引，文件名固定为 `source-manifest.json`，存放在独立 `game-sources` 桶。对象索引不能作为 `manifestUrl` 返回给宿主，不能携带 candidate/published/retired 状态，也不保存签名 URL。生产版本身份来自 project 数据库，Tier2 源工程继续绑定现有 SourceProjectStore；LittleJS/Canvas 等游戏通过 `game_source_archive` provider 绑定引擎无关源归档。对象存储提交记录由 V34 `game_artifact_storage` 保存，不能把 MinIO marker 当作 committed。 + `agent-loop/` 和 `templates/` 这两个目录在 `contracts/README.md` 的目录结构图里没有 —— 那张图只画了跨仓 8 类。新人 `ls` 到它们时无从对应,这是 README 自身的覆盖缺口,在此补登:`agent-loop/` = 生成线的源项目 keystone 加 QA 闭环工件契约族,`templates/` = 13 个品类/形态 schema。 ## "第几类"口径收口 diff --git a/docs/plans/2026-07-29-游戏包与资产对象存储迁移-plan.md b/docs/plans/2026-07-29-游戏包与资产对象存储迁移-plan.md new file mode 100644 index 00000000..7700ed66 --- /dev/null +++ b/docs/plans/2026-07-29-游戏包与资产对象存储迁移-plan.md @@ -0,0 +1,168 @@ +--- +status: 修订待复核 +sot-impact: 修订 GamePackage、运行时分发、参照资产消费和游戏内容存储边界；本档只记录迁移方案，不复制架构 SoT +canonical: false +上级: docs/architecture/架构/契约总览.md +--- + +# 游戏包与资产对象存储迁移 + +> **执行约束**：按波次实施，每一波独立验证。任何具体游戏从 Git 删除前，必须先完成对象存储上传、下载逐文件对账、真实消费者切换和回滚观察。 + +**目标**：Git 只保存平台代码、契约和最小合成测试夹具；每款具体游戏的源工程、素材、运行包和验收证据进入对象存储，并能被授权下载、预览、发布和可信消费。 + +**架构**：对象存储保存不可变内容，数据库保存租户归属、发布状态、对象 key 和可信 hash。运行时清单继续使用现有 `GamePackage/1`；`GameArtifactManifest/1` 只索引运行包、素材和证据；引擎无关 `GameSourceArchive/1` 单独索引源码。三者不共用 schema、不共用 URL。对象 marker 只代表回读校验完成，V34 `game_artifact_storage` 的数据库 CAS 才代表 committed。 + +**现有技术**：MinIO/S3、MySQL、Spring Runtime API、Python 生成 worker、Vue 游戏宿主、ReferenceAsset 消费门。 + +## 边界 + +| 类别 | Git | 对象存储 | 生产消费方式 | +|---|---|---|---| +| 平台代码 | 保留 | 可选发布镜像 | 正常构建、部署 | +| 合成测试夹具 | 保留最小量 | 可选 | 只验证平台算法，不呈现具体游戏内容 | +| 游戏源工程 | 不保留 | 引擎无关源码进入私有 `game-sources`；Tier2 历史源工程仍在 `tier2-src` | 生成、修复服务按 source manifest 或 Tier2 manifest 物化 | +| 游戏素材 | 不保留 | `game-artifacts` | 运行时授权后代理或签发精确短期 URL | +| 游戏运行包 | 不保留 | `game-artifacts` | 宿主读取现有 `GamePackage/1` 和版本 bundle | +| 验收证据 | 不保留 | 私有 `game-evidence` | 审核服务按 record/key/hash 读取 | +| 临时批跑 | 不保留 | 默认不保存 | 需要审计时按保留期进入证据桶 | +| 设计和签认结论 | 保留摘要 | 原始媒体可外置 | Git 保存决策、签认身份和对象 hash，不保存游戏内容 | + +平台共享引擎、SDK、生成器、构建器、上传下载器、契约和门脚本仍属于项目代码。绑定某一款游戏的玩法代码、配置、素材、bundle、截图、轨迹和测试不属于平台代码。 + +## 三份清单 + +运行版本前缀下保留运行清单和制品清单；源码修订使用独立清单和桶： + +```text +tenants//games//versions// +├── manifest.json # 现有 GamePackage/1，游戏宿主读取 +├── artifact-manifest.json # GameArtifactManifest/1，运行包/素材/证据索引 +├── assets/<相对路径> +└── runtime/ + +game-sources 桶（GameSourceArchive/1；权限和桶预建完成前禁止联网写）: +└── tenants//games//source-revisions// + ├── source-manifest.json + └── source/<源工程相对路径> + +tier2-src 桶（沿用现有 Tier2 SourceProjectStore，仅 Tier2）： +└── //<源工程相对路径> + +game-evidence 桶: +└── tenants//games//versions//evidence/<验收证据路径> +``` + +`manifest.json` 的原始字节 hash 继续写入 `game_runtime_package.checksum`，宿主按现有 `GamePackage/1` 解析。`artifact-manifest.json` 记录素材、运行包和证据对象的 `category/store/path/sourcePath/key/bytes/sha256/mime`、分类 tree hash、bundle hash 和自身 hash。引擎无关源工程由 `GameSourceArchive/1` 记录源码路径、字节 hash、sourceHash 和 `source-manifest.json` 自身 hash；`game_source_archive` provider 绑定这份清单，Tier2 继续使用 `tier2_source_project` 和现有 `tier2-src`。Tier2 provider 只允许 Phaser package，LittleJS/Canvas 必须走引擎无关 provider。清单不包含发布状态、时间或执行者，也不能作为 `manifestUrl` 返回给宿主。创建时间、操作者和 trace 属于数据库审计记录，不能让相同内容重复构建出不同 manifest hash。 + +对象 key 永久稳定，签名 URL 只在授权请求时生成，不写入清单。路径先做 NFC 规范化并按 POSIX 相对路径排序；tree hash 和 manifest hash 使用不同域标签，防止不同类型摘要被混用。 + +## 数据权威 + +| 数据 | 权威位置 | 说明 | +|---|---|---| +| 游戏版本身份 | `project.game_version.id` | 运行包对象前缀的 `versionId`，契约只接受正整数十进制字符串；Tier2 字符串版本只能写入 `sourceRevision.revisionId` | +| 发布生命周期 | `game_runtime_package.status` | 保持 preview/published/retired 的 CAS 门，不写入不可变对象清单 | +| GamePackage 原始 hash | `game_runtime_package.checksum` | 对 `manifest.json` 原始字节计算 | +| 对象索引身份 | 新的版本存储记录 | 保存 bucket、artifact manifest key/hash、状态 pending/committed、字节数和创建者 | +| 引擎无关源工程 | `GameSourceArchive/1` + `game-sources` | 保存 source manifest key/hash、source hash、引擎元数据；V34 对象记录保存其与运行版本的绑定 | +| Tier0/1 结构化源项目 | 迁移后由对象存储持有正文 | `game_source_project` 只保存版本、对象 key、source hash 和元数据 | +| Tier2 源工程 | 现有 MySQL manifest + `tier2-src` | 保留已上线的 manifest-first 路径；生产制品通过 `tier2-source-project::` 和 sourceHash 绑定，不复制源文件 | +| 金标消费身份 | 新版 ReferenceAsset Registry/Release | 保存可信对象 key/hash；消费前物化到临时可信根并逐文件校验 | + +对象存储版本记录将在波次 2 使用状态 `pending → committed`。所有对象上传并 GET 全量复算后，还必须由数据库条件创建/CAS 把 pending 变为 committed；发布 CAS 只接受 committed 版本。同一 `tenantId/gameId/versionId` 已存在不同 manifest hash 时拒绝覆盖，相同 hash 视为幂等命中。失败的 pending 记录由定时回收任务按保留期清理，published/retired 版本都不能被覆盖。 + +波次 1 使用的 MinIO Python SDK 没有公开的条件 Put 接口。离线上传器因此只允许调用方先持有 `tenantId/gameId/versionId` 级外部单写锁，并返回 `verified_pending`，明确 `committed=false`。`artifact-manifest.json` 和 `source-manifest.json` 已存在时，不同 hash 会在任何 Put 前拒绝，相同 hash 会逐对象回读校验；marker 尚不存在时，残留 partial 对象可在单写锁内被当前内容覆盖。源码桶必须是显式独立桶，不能与 artifact/evidence 桶复用。这个行为不是并发不可覆盖保证：两个无锁写者仍可能在“读取 marker”和“写 marker”之间相互覆盖，波次 1 的 marker 也不能作为发布提交凭据。 + +对象存储提交状态与运行包发布状态是两台独立状态机，只通过“发布只接受 committed 对象”这道门连接。 + +```mermaid +stateDiagram-v2 + state "对象存储版本" as storage { + [*] --> pending: 创建存储记录 + pending --> committed: 对象全量 GET 对账 + 数据库 CAS + pending --> abandoned: 超时或上传失败 + committed --> abandoned: 未发布且明确废弃 + } + state "运行包发布" as release { + [*] --> preview + preview --> published: 存储记录必须 committed + published --> retired: 下架 + published --> published: 禁止覆盖 + retired --> retired: 只读或法务冻结 + } +``` + +## 读写接口 + +平台只接受数据库中已登记的对象 key，不接受调用方传入任意 URL。 + +| 接口 | 输入 | 输出 | 失败方式 | +|---|---|---|---| +| 构建对象索引 | 本地临时游戏目录、数据库 tenant/game/version、完整 GamePackage、已保存的 Tier2 sourceRevision | `GameArtifactManifest/1` | GamePackage schema/身份不一致、源修订 hash 漂移、路径越界、符号链接、凭据文件或重复路径即拒绝 | +| 构建源归档 | 本地临时游戏目录、数据库 tenant/game、source revision、引擎描述 | `GameSourceArchive/1` | 空源树、二进制/路径 hash 漂移、运行 manifest 混入、路径越界、符号链接或重复路径即拒绝 | +| 波次 1 上传验证 | manifest、可信本地根、服务端身份、外部单写锁 | `verified_pending`、`committed=false` 回执 | 未声明外部单写、同版本 marker 不同 hash、上传/回读不一致即拒绝 | +| 生产上传提交（波次 2） | pending 数据库记录、manifest、可信本地根、服务端身份 | 数据库 CAS 后的 committed 回执 | 并发条件创建失败、同版本不同 hash、配额超限、上传/回读不一致即拒绝 | +| 物化版本 | 数据库登记的 tenant/game/version、manifest key/hash、用途、目标临时根 | 已逐文件校验并原子交付的本地快照 | 期望身份/hash 缺失、生产消费未 committed、越权或逐文件 hash 不一致即拒绝且清理临时目录；波次 1 只允许诊断回验，不接生产消费 | +| 运行时取包 | versionId、scene、当前用户 | 同源 API 响应或精确短期 URL | 保持 owner/status/tenant 门；生产失败显式报错，不回退 demo | +| 金标取参照 | releaseId、policyId、recordId | 验证回执与只读物化根 | registry/release/manifest/hash 任一不一致即拒绝 | + +前端只向同源 Runtime API 发送平台 Token。访问对象存储时使用后端签发的精确短期 URL，不能附带平台 Authorization 或 tenant-id；重定向到非允许域时拒绝。source 和 evidence 永不直接暴露给浏览器，运行包域需要独立 CORS、CSP、MIME 和缓存策略。写入只允许专用 artifact-writer 身份，源码桶必须单独授权且不能和 artifact/evidence 同桶；对象 key 由数据库 tenant/game/version 关系派生，调用方不能提交桶或前缀。读者无 List/Delete，写者无建桶、删桶、策略管理或已提交版本 Delete 权限。现有 `ragflow` MinIO root 凭据不能复用；专用账号、源桶和权限未就位时上传器必须拒绝联网。生产启用 TLS、静态加密、版本化和 90 天凭据轮换。 + +## 四条迁移线 + +### 运行时与素材 + +Runtime 编译完成后同时写现有 GamePackage 和对象存储版本记录。影子读阶段仍由 DB 返回 GamePackage，但后台从 OSS 下载并对账；切换后 Runtime API 在完成 tenant/owner/status 门禁后代理返回 GamePackage 或签发对象 URL。宿主的生产模式遇到网络、schema 或 hash 错误必须显示加载失败，demo 兜底只留在显式 mock/development 模式。 + +当前真实消费链已核对为：`GET /app-api/runtime/package/{versionId}` 与原始 manifest 端点先进入 `RuntimePackageServiceImpl.getPackageManifest`，再由 `PackageStore` 读取 `game_runtime_package.package_json`；`publishPackage` 是同一运行包发布入口。新增的 `ArtifactStorageGate` 挂在取包和发布之前，配置 `game.artifact-storage.enforce-committed=false` 时保持旧 DB 路径，打开后只放行 V34 `game_artifact_storage.status=committed` 的版本，并对账 `runtime_manifest_hash == game_runtime_package.checksum`。当 GamePackage 有 `engineBundle` 时，门禁还对实际 UTF-8 bundle 文本计算 SHA-256 并与 `bundle_hash` 比较；没有可选 bundle 时两边都必须为空。manifest 缺失、解析失败、hash 漂移或对象记录未提交均拒绝，不回退 demo。这个门禁目前是生产切换前的影子实现，尚未代表 OSS `PackageStore` 已切换；V34 迁移、对象上传和观察窗完成前不得打开开关。 + +参照资产旧版 v2 测试仍有一条 worktree 断言假定具体游戏文件未被 Git 忽略；当前仓库已按本计划把具体游戏列入忽略并准备迁移对象存储，因此该红灯是测试口径陈旧，不是生产 `clean-archive/frozen_preflight` 消费失败。修该测试应单独纳入参照资产测试治理，不通过恢复 Git 游戏内容来消红。 + +### 引擎无关源工程 + +对 LittleJS、Canvas 和其它非 Tier2 工程，先用 `GameSourceArchive/1` 固定 NFC 路径、二进制安全 sourceHash 和 source manifest hash，再由 V34 对象记录绑定 game/version。`game-sources` 未完成专用权限和预建桶前只允许离线构建与 fake/合成测试；完成上传、逐文件回读和 CAS 后才允许生产物化。 + +### Tier0/1 结构化源工程 + +先确定 Tier0/1 SourceProjectStore 的 canonical hash 和不可变修订身份，再修订 artifact schema 增加对应 provider。随后给 `game_source_project` 增加对象 key、source hash、manifest hash 和迁移状态，写入时双写 DB JSON 与对象存储；完成历史 backfill 和全量对账后，读取改为对象存储，最后停止写正文 JSON。对外 fetch DTO 返回稳定对象身份，不返回永久公开 URL。 + +### Tier2 源工程 + +保留现有 `BackendStore` 的 MySQL manifest-first 与 `tier2-src` MinIO 实现。发布运行包时，artifact manifest 的 `versionId` 只写 `project.game_version.id`，Tier2 返回的 `{id, versionId, sourceHash}` 原样写入 `sourceRevision`；构建器必须复算本地 SourceProjectStore 口径的 `path\0content\0` hash 并与 sourceHash 相等。两者的映射随 artifact manifest hash 一起进入对象存储版本记录，不新建第二套 Tier2 源权威。 + +### 金标和参照资产 + +冻结现有 ReferenceAsset `/1`、`/2`，新增对象存储消费版本，不就地放宽相对路径。新 Registry 保存对象身份与 artifact manifest hash；Release 继续绑定 Registry 和 policy 原始字节 hash。`frozen_preflight` 先把对象下载到临时可信根，再沿现有 fd/逐文件 hash 逻辑校验。只有新消费路径和回归测试全绿后，才把《山海行纪》及其它参照内容从 Git 删除。 + +## 实施波次 + +- [x] **波次 1：契约与离线工具。** 已冻结 `GamePackage/1`、`GameArtifactManifest/1`、`GameSourceArchive/1` 三份清单，收紧 Tier2/Phaser provider 边界、引擎无关源 hash、外部单写前提下的 marker 幂等、`verified_pending` 回执、下载逐文件验证和安全负测；未产出 committed，未改生产消费，未删除 Git 内容。 +- [ ] **波次 2：对象存储状态记录。** 增加 pending/committed 记录、数据库条件创建/CAS、并发不可覆盖门、孤儿回收、审计日志和配额；完成至少一款候选的上传、committed 和下载回验。 +- [ ] **波次 3：运行时影子读。** GamePackage 双写 OSS，后端影子下载对账；修正绝对 URL Token 泄露和生产 demo 回退。 +- [ ] **波次 4：源工程迁移。** 分别迁移 Tier0/1 和 Tier2，完成 backfill、影子读、全量对账和回滚观察。 +- [ ] **波次 5：金标消费迁移。** 新 Registry/Release/物化 preflight 通过后，将 active 金标绑定对象身份。 +- [ ] **波次 6：Git 清理。** 上传所有具体游戏、切换全部消费者、确认观察窗无回退后，独立提交删除 tracked 游戏内容并扩大 ignore；平台合成 fixture 留在 Git。 + +## 验收 + +### 当前执行证据（2026-07-29） + +离线工具、V34 状态记录、运行时提交门和正式《山海行纪》验收均已在本地完成验证：Python worker `181 passed`，Java runtime 定向测试 `28 passed`，Shanhai Node 测试 `54/54`，三场景 headless smoke 全绿，quick/formal simulation gate 分别满足 c1–c4，`test_reference_asset_and_acceptance_v3.py` 为 `43/43`，docs-gate 与 `git diff --check` 全绿。运行时门的默认开关仍为 `false`；它会同时校验三元业务身份、V34 `committed`、runtime manifest hash 和实际 UTF-8 bundle hash。 + +波次 2 仍未宣称完成，但隔离 staging 已完成一轮真实身份和对象链路验证：通过正式 App API 创建项目 `gameId=80034`、生成版本 `versionId=93156`，并在 `game-staging-mysql/ruoyi-vue-pro` 应用 V34。原始 GamePackage 由实际 Runtime API 取回，`runtimeManifestHash=1191b84776703ce009d361c920da0c3dbb73f328f1383e25580915b10937e4ee`；为满足 generic 宿主的 `__GameBundle` 入口，staging 适配包的 `bundleHash=ef2fff2764227b773e4f4dc092941ddbef7f7062fae2dae80160992de637066d`，不改正式签认包的 `17b9073c767faf7990e0bf4563a86d55ffa81121f11e7c8ad37c8b8ce72e8cbd`。artifact manifest 已上传 `game-artifacts` 并由专用 writer 全量 GET 回读，得到 `manifestHash=815f8918c0355ebda4ca7feb9e9e012e9f76fa597ef4dab05df06b5a302f347e`；V34 存储记录为 `pending/committed=false`。打开 `game.artifact-storage.enforce-committed=true` 的隔离 Runtime 后，真实 `GET /app-api/runtime/package/93156?scene=preview` 返回 `1102001001`，日志明确记录 `[artifactStorageGate] 对象记录未 committed，拒绝运行时消费`；同一 artifact manifest 又从对象存储物化并逐文件校验 263 个对象，下载后的 bundle 与上述 hash 一致。 + +源码归档清单已离线构建（57 个对象，`sourceHash=26dcb489bd147c393dfae2b567090f787c2be1219525321e40d9d619af8eddbc`，`manifestHash=761dd48aef911bbfff9a5a03d9710ccec6cce1b8360382b397a233291f7d5358`），但联网上传仍按设计 fail-closed：`game-sources` 尚未预建并授权给专用写者，CLI 返回 `GAME-SOURCE-ARCHIVE-ERROR`。因此尚不能把这轮 staging 证据记为 committed、不能打开生产运行时开关，也不能据此改写正式金标 Registry；生产切换和波次 2 完成仍以源码桶权限、pending→committed CAS 及观察窗为前置条件。 + +- 干净 clone 不含任何具体游戏源、素材、bundle、dist 或 evidence；只保留平台代码和合成 fixture。 +- 每款迁移游戏都有 committed 回执，能从对象存储下载并逐文件复算 hash。 +- 浏览器真实加载对象存储提供的 GamePackage、bundle 和素材；控制台无阻断错误，平台 Token 不发往对象域。 +- 运行包加素材总量不超过 10MB、首屏不超过 2MB；单文件不超过 100MB、单版本不超过 10000 个对象、相对路径不超过 1024 字节，租户总配额由服务端记录强制。 +- preview、play、retired、跨租户、过期签名、重定向 SSRF、并发覆写、错误 MIME/CORS/CSP、配额和孤儿回收负测均被拦截。 +- ReferenceAsset 的 Registry、Release、消费清单和物化快照形成完整 hash 链，干净 clone 下仍能完成 `frozen_preflight`。 +- 同一版本重复上传相同 hash 为幂等命中，不同 hash 拒绝；published 后不可覆盖，retired 后不可再签发新 URL。 +- `.agents/tools/docs-gate.sh`、相关契约/后端/前端测试、对象存储 smoke、浏览器真玩和 `git diff --check` 全绿。 + +## 删除门 + +现有 `game-runtime/games/*`、`tier2/games/*` 和 `_wg1-gen` 中仍有 tracked 具体游戏。增加 `.gitignore` 不能移除已跟踪文件，也不能替代消费迁移。只有对应游戏同时满足“已上传、已下载对账、消费者已切换、观察窗结束、回滚包可取”五项条件，才允许从 Git 删除；任何一项缺失都继续保留，避免发布链和金标门被切断。 diff --git a/docs/内网凭据与端点.md b/docs/内网凭据与端点.md index 7df1ae6e..1a4860c6 100644 --- a/docs/内网凭据与端点.md +++ b/docs/内网凭据与端点.md @@ -62,6 +62,26 @@ canonical: true > Redis 需 AUTH(NOAUTH 已实证);MinIO root 即 ragflow(与 ragflow 共用实例,tier2 另开 bucket 如 `tier2-src` 隔离)。这些是内网 Tailscale 凭据、按铁律入仓。 +### 游戏内容对象存储专用身份（2026-07-29） + +具体游戏的素材、运行包和验收证据不复用 MinIO root。专用账号当前只允许对两个预建桶的 `tenants/` 前缀执行对象 Put/Get、前缀内 List 与分片上传恢复，并允许 Python SDK 读取这两个桶的区域；不能访问未授权桶，也没有 DeleteObject、建桶、删桶或策略管理权限。引擎无关源归档的 `game-sources` 桶尚未预建、未加入该账号策略；在契约和权限独立复核前，源归档 CLI 的联网写入必须失败。当前 MinIO 的根列桶接口会返回调用者有权访问的两个桶名，不会返回其它桶；这项行为不能表述为根列桶请求被拒绝。 + +| 项 | 值 | +|---|---| +| S3 端点 | `100.64.0.8:9000`（内网 HTTP；公网/生产必须切 TLS） | +| access key | `game-artifact-writer` | +| secret key | `4aa0a3f0036aac13cded3942c056df2c270b99cb7962624b` | +| 策略 | `game-artifact-writer-policy-v3` | +| 运行包与素材桶 | `game-artifacts` | +| 验收证据桶 | `game-evidence` | +| 允许资源 | `game-artifacts/tenants/*`、`game-evidence/tenants/*` | + +2026-07-29 权限实测：两个允许前缀 Put/Get 和前缀内 List 成功，Python SDK `GetBucketLocation` 成功且不存在对象稳定返回 `NoSuchKey`；根列桶只返回 `game-artifacts`、`game-evidence`，列/写 `tier2-src`、删除已写对象均返回 Access Denied。随后在单一测试进程控制的单写条件下，用真实 Python SDK 完成合成包首次上传、全量 GET 校验、可信 hash 下载、原子物化和重复上传幂等命中，回执 `manifestHash=3343b3e3563098ae6adb55c17151c75aec69efbb6722b70d3052858b84da25da`；测试前缀最终由 MinIO root 全量清理并确认对象不存在。这份实测只证明单写流程可用，不能证明并发条件写或数据库 committed；波次 1 工具统一返回 `verified_pending`，必须等波次 2 数据库条件创建/CAS 后才能形成 committed 回执。运行器读取 `tier2/config/infra.yaml` 的 `game_artifact_minio` 分区；该分区缺失时必须拒绝联网，禁止回落到 `minio` 分区的 root 凭据。 + +同日生产身份盘点纠偏：mini-infra 的 `game_cloud` 空库没有接入现行 game-cloud，不是游戏身份权威。内网 staging 的真实消费库是 mini-desktop `game-staging-mysql/ruoyi-vue-pro`；2026-07-29 在隔离服务进程中应用 V34 后，库中共有 34 条成功 Flyway 历史、110 个项目、53 个版本和 54 个运行包，并通过正式 App API 创建了对象存储验收项目 `gameId=80034`、版本 `versionId=93156`。V34 存储记录已写入 `pending/committed=false`，源码归档 `sourceRevision=gold-20260729` 只完成离线清单；`game-sources` 尚未预建授权，联网上传仍必须 fail-closed。该 staging 身份只用于本轮隔离验收，不能当作生产发布或正式金标身份，也禁止在空 `game_cloud` 建无人消费的临时记录。 + +运行时对象提交门已接入 game-cloud 的取包、原始 manifest 和发布入口，配置键为 `game.artifact-storage.enforce-committed`，默认 `false` 仅用于 V34 尚未提交的兼容窗口。隔离 Runtime 打开 `true` 后，真实 `GET /app-api/runtime/package/93156?scene=preview` 返回 `1102001001`，日志记录 `[artifactStorageGate] 对象记录未 committed，拒绝运行时消费`；同一版本的 artifact manifest 已由专用 writer 全量 GET 回读并物化 263 个对象，`runtimeManifestHash=1191b84776703ce009d361c920da0c3dbb73f328f1383e25580915b10937e4ee`，staging generic 适配 bundle 的 `bundleHash=ef2fff2764227b773e4f4dc092941ddbef7f7062fae2dae80160992de637066d`，下载后逐字节一致。生产切换仍需 `pending→committed` CAS、源码桶权限和观察窗；正式金标继续使用已签认 bundle `17b9073c767faf7990e0bf4563a86d55ffa81121f11e7c8ad37c8b8ce72e8cbd`，未切换 OSS `PackageStore`。 + --- ## Nacos 配置中心 + 服务发现（mini-infra `100.64.0.8` · 2026-07-01 自托管落档 · 切片一 阶段〇 B1） diff --git a/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/dal/dataobject/storage/ArtifactStorageDO.java b/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/dal/dataobject/storage/ArtifactStorageDO.java new file mode 100644 index 00000000..c16c13bc --- /dev/null +++ b/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/dal/dataobject/storage/ArtifactStorageDO.java @@ -0,0 +1,69 @@ +package com.wanxiang.huijing.game.module.runtime.dal.dataobject.storage; + +import com.baomidou.mybatisplus.annotation.KeySequence; +import com.baomidou.mybatisplus.annotation.TableName; +import com.wanxiang.huijing.framework.tenant.core.db.TenantBaseDO; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * 游戏对象存储版本状态 DO。 + * + *

该表只记录对象上传校验与数据库提交状态，不替代 {@code game_runtime_package.status}。 + * 运行时消费门只读取 status=committed 的记录，并用 runtime manifest hash 对账运行包 checksum。

+ */ +@TableName("game_artifact_storage") +@KeySequence("game_artifact_storage_seq") +@Data +@EqualsAndHashCode(callSuper = true) +public class ArtifactStorageDO extends TenantBaseDO { + + /** 对象存储状态记录 ID。 */ + private Long id; + /** 游戏项目 ID。 */ + private Long gameId; + /** 游戏版本 ID。 */ + private Long versionId; + /** 运行包、素材和证据对象所在桶。 */ + private String artifactBucket; + /** artifact-manifest.json 对象 key。 */ + private String artifactManifestKey; + /** artifact-manifest.json 原始字节 SHA-256。 */ + private String artifactManifestHash; + /** artifact-manifest.json 字节数。 */ + private Long artifactManifestBytes; + /** GamePackage manifest.json 对象 key。 */ + private String runtimeManifestKey; + /** GamePackage manifest.json 原始字节 SHA-256，与 runtime checksum 对账。 */ + private String runtimeManifestHash; + /** 引擎无关源归档所在桶；Tier2 外部源修订可为空。 */ + private String sourceBucket; + /** 源修订 provider。 */ + private String sourceProvider; + /** 源修订所属稳定标识。 */ + private String sourceGameId; + /** 源修订不可变标识。 */ + private String sourceRevisionId; + /** source-manifest.json 对象 key。 */ + private String sourceManifestKey; + /** source-manifest.json 原始字节 SHA-256。 */ + private String sourceManifestHash; + /** 源文件树 SHA-256。 */ + private String sourceHash; + /** 运行包形态：phaser/littlejs/canvas。 */ + private String packageType; + /** artifact manifest 正文对象数量。 */ + private Integer objectCount; + /** 运行包、素材和证据对象总字节数。 */ + private Long totalBytes; + /** 素材 tree hash。 */ + private String assetHash; + /** 运行 bundle SHA-256。 */ + private String bundleHash; + /** pending/committed/abandoned。 */ + private String status; + /** 失败或回收原因。 */ + private String failureReason; + /** 数据库 CAS 成功时间。 */ + private java.time.LocalDateTime committedAt; +} diff --git a/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/dal/mysql/storage/ArtifactStorageMapper.java b/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/dal/mysql/storage/ArtifactStorageMapper.java new file mode 100644 index 00000000..2b51f2ca --- /dev/null +++ b/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/dal/mysql/storage/ArtifactStorageMapper.java @@ -0,0 +1,33 @@ +package com.wanxiang.huijing.game.module.runtime.dal.mysql.storage; + +import com.wanxiang.huijing.framework.mybatis.core.mapper.BaseMapperX; +import com.wanxiang.huijing.framework.mybatis.core.query.LambdaQueryWrapperX; +import com.wanxiang.huijing.game.module.runtime.dal.dataobject.storage.ArtifactStorageDO; +import org.apache.ibatis.annotations.Mapper; + +/** + * 游戏对象存储版本状态 Mapper。 + * + *

消费侧只允许按版本读取 committed 记录，pending/abandoned 不得进入运行时。

+ */ +@Mapper +public interface ArtifactStorageMapper extends BaseMapperX { + + /** + * 按租户、游戏和版本读取唯一已提交对象记录。 + * + *

三元身份必须完整进入查询条件，不能只信 versionId；这是对象记录跨租户隔离的可信边界。

+ * + * @param tenantId 租户 ID + * @param gameId 游戏项目 ID + * @param versionId 游戏版本 ID + * @return committed 记录；不存在时返回 null + */ + default ArtifactStorageDO selectCommittedByIdentity(Long tenantId, Long gameId, Long versionId) { + return selectOne(new LambdaQueryWrapperX() + .eq(ArtifactStorageDO::getTenantId, tenantId) + .eq(ArtifactStorageDO::getGameId, gameId) + .eq(ArtifactStorageDO::getVersionId, versionId) + .eq(ArtifactStorageDO::getStatus, "committed")); + } +} diff --git a/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/service/pkg/ArtifactStorageGate.java b/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/service/pkg/ArtifactStorageGate.java new file mode 100644 index 00000000..ef2b9d9d --- /dev/null +++ b/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/service/pkg/ArtifactStorageGate.java @@ -0,0 +1,134 @@ +package com.wanxiang.huijing.game.module.runtime.service.pkg; + +import com.wanxiang.huijing.game.module.runtime.dal.dataobject.pkg.RuntimePackageDO; +import com.wanxiang.huijing.game.module.runtime.dal.dataobject.storage.ArtifactStorageDO; +import com.wanxiang.huijing.game.module.runtime.dal.mysql.storage.ArtifactStorageMapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Objects; + +import static com.wanxiang.huijing.game.module.runtime.enums.ErrorCodeConstants.RUNTIME_PACKAGE_ARTIFACT_MISMATCH; +import static com.wanxiang.huijing.game.module.runtime.enums.ErrorCodeConstants.RUNTIME_PACKAGE_NOT_READY; +import static com.wanxiang.huijing.framework.common.exception.util.ServiceExceptionUtil.exception; + +/** + * 对象存储提交状态消费门。 + * + *

开关默认关闭，允许 V34 尚未部署的环境继续使用旧 DB manifest 路径；打开后，运行时必须 + * 找到 committed 记录，且记录中的 GamePackage manifest hash 必须与 runtime package checksum + * 一致；若 manifest 含 engineBundle，还必须把实际 bundle 字节 hash 与对象记录对账。

+ */ +@Slf4j +@Component +public class ArtifactStorageGate { + + /** 只读解析原始 GamePackage，不对外重新序列化，避免破坏 checksum 字节语义。 */ + private static final ObjectMapper PACKAGE_MAPPER = new ObjectMapper(); + + @Resource + private ArtifactStorageMapper artifactStorageMapper; + + /** + * 生产切换开关。只有 V34 已落库且上传器完成 pending→committed 后才允许打开。 + */ + @Value("${game.artifact-storage.enforce-committed:false}") + private boolean enforceCommitted; + + /** + * 校验运行包是否已经绑定到已提交对象记录。 + * + * @param runtimePackage 当前运行包记录 + */ + public void requireCommitted(RuntimePackageDO runtimePackage) { + if (!enforceCommitted) { + return; + } + ArtifactStorageDO storage = artifactStorageMapper.selectCommittedByIdentity( + runtimePackage.getTenantId(), runtimePackage.getGameId(), runtimePackage.getVersionId()); + if (storage == null) { + log.warn("[artifactStorageGate] 对象记录未 committed，拒绝运行时消费 versionId={}", + runtimePackage.getVersionId()); + throw exception(RUNTIME_PACKAGE_NOT_READY); + } + BundleCheck bundleCheck = inspectBundle(runtimePackage); + boolean bundleMatches = bundleCheck.readable() + && (bundleCheck.present() + ? StringUtils.hasText(storage.getBundleHash()) + && Objects.equals(storage.getBundleHash(), bundleCheck.sha256()) + : !StringUtils.hasText(storage.getBundleHash())); + if (!Objects.equals(storage.getRuntimeManifestHash(), runtimePackage.getChecksum()) + || !bundleMatches) { + log.error("[artifactStorageGate] 对象记录与运行包摘要不一致，拒绝消费 versionId={}, " + + "runtimeManifestHash={}, runtimeChecksum={}, bundleHash={}, actualBundleHash={}", + runtimePackage.getVersionId(), storage.getRuntimeManifestHash(), runtimePackage.getChecksum(), + storage.getBundleHash(), bundleCheck.sha256()); + throw exception(RUNTIME_PACKAGE_ARTIFACT_MISMATCH); + } + } + + /** + * 从当前实际消费的 DB manifest 提取 engineBundle 并计算 UTF-8 SHA-256。 + * + *

对象上传器已经对远端 bundle 做逐字节回读；消费侧再对 DB manifest 中即将交给宿主的 + * engineBundle 做一次对账，避免只检查数据库里的 bundle_hash 非空而放过陈旧内容。

+ */ + private BundleCheck inspectBundle(RuntimePackageDO runtimePackage) { + if (!StringUtils.hasText(runtimePackage.getPackageJson())) { + log.error("[artifactStorageGate] 运行包缺少 DB manifest，无法校验实际 bundle versionId={}", + runtimePackage.getVersionId()); + return BundleCheck.unreadable(); + } + try { + JsonNode root = PACKAGE_MAPPER.readTree(runtimePackage.getPackageJson()); + JsonNode bundle = root == null ? null : root.get("engineBundle"); + if (bundle == null || bundle.isNull()) { + // GamePackage.engineBundle 是可选字段；缺失时必须同时没有 bundle_hash。 + return BundleCheck.noBundle(); + } + if (!bundle.isTextual() || !StringUtils.hasText(bundle.asText())) { + log.error("[artifactStorageGate] DB manifest 的 engineBundle 类型或内容非法 versionId={}", + runtimePackage.getVersionId()); + return BundleCheck.unreadable(); + } + return BundleCheck.present(sha256Hex(bundle.asText())); + } catch (Exception ex) { + log.error("[artifactStorageGate] DB manifest 无法解析，拒绝 bundle 校验 versionId={}", + runtimePackage.getVersionId(), ex); + return BundleCheck.unreadable(); + } + } + + /** 当前 manifest 是否可读、是否包含 bundle，以及 bundle 的真实摘要。 */ + private record BundleCheck(boolean readable, boolean present, String sha256) { + private static BundleCheck unreadable() { + return new BundleCheck(false, false, null); + } + + private static BundleCheck noBundle() { + return new BundleCheck(true, false, null); + } + + private static BundleCheck present(String sha256) { + return new BundleCheck(true, true, sha256); + } + } + + /** 计算 bundle 文本 UTF-8 字节摘要，与对象归档及生成链路使用同一 sha256 口径。 */ + private static String sha256Hex(String value) { + try { + byte[] digest = MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8)); + return java.util.HexFormat.of().formatHex(digest); + } catch (Exception ex) { + throw new IllegalStateException("JDK 缺少 SHA-256 算法", ex); + } + } +} diff --git a/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/service/pkg/RuntimePackageServiceImpl.java b/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/service/pkg/RuntimePackageServiceImpl.java index 887b3e35..6c1c160d 100644 --- a/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/service/pkg/RuntimePackageServiceImpl.java +++ b/game-cloud/game-module-runtime/game-module-runtime-server/src/main/java/com/wanxiang/huijing/game/module/runtime/service/pkg/RuntimePackageServiceImpl.java @@ -57,6 +57,10 @@ public class RuntimePackageServiceImpl implements RuntimePackageService { @Resource private ProjectApi projectApi; + /** 对象存储提交状态门；开关关闭时保持现有 DB manifest 兼容路径。 */ + @Resource + private ArtifactStorageGate artifactStorageGate; + @Override public RuntimePackageDO getPackageManifest(Long versionId, String scene, Long userId) { // 取就绪运行包（uk_version 唯一） @@ -65,6 +69,8 @@ public class RuntimePackageServiceImpl implements RuntimePackageService { // 无对应就绪运行包（编译未完成或版本不存在），对齐契约 #1：返回 1-102-001-001 throw exception(RUNTIME_PACKAGE_NOT_READY); } + // 生产切换后只允许消费已经完成对象上传、回读校验和数据库 CAS 的版本。 + artifactStorageGate.requireCommitted(pkg); // 取包门禁（决策1，以 game_runtime_package.status 为权威判定字段，不与 project.game_version.status 混用） if (RuntimeSceneEnum.isPreview(scene)) { // 预览：放行 status∈{0 预览就绪,1 已发布}，且校验调用者为项目 owner 或服务端已认证管理员。 @@ -89,6 +95,8 @@ public class RuntimePackageServiceImpl implements RuntimePackageService { if (pkg == null) { throw exception(RUNTIME_PACKAGE_NOT_READY); } + // 发布前同样校验 committed 记录，避免未完成对象上传的版本进入已发布状态。 + artifactStorageGate.requireCommitted(pkg); if (!Objects.equals(pkg.getChecksum(), expectedArtifactHash)) { log.warn("[publishPackage] 产物摘要不一致，拒绝发布 versionId={}", versionId); throw exception(RUNTIME_PACKAGE_ARTIFACT_MISMATCH); diff --git a/game-cloud/game-module-runtime/game-module-runtime-server/src/test/java/com/wanxiang/huijing/game/module/runtime/service/pkg/ArtifactStorageGateTest.java b/game-cloud/game-module-runtime/game-module-runtime-server/src/test/java/com/wanxiang/huijing/game/module/runtime/service/pkg/ArtifactStorageGateTest.java new file mode 100644 index 00000000..d625bde6 --- /dev/null +++ b/game-cloud/game-module-runtime/game-module-runtime-server/src/test/java/com/wanxiang/huijing/game/module/runtime/service/pkg/ArtifactStorageGateTest.java @@ -0,0 +1,110 @@ +package com.wanxiang.huijing.game.module.runtime.service.pkg; + +import com.wanxiang.huijing.framework.common.exception.ServiceException; +import com.wanxiang.huijing.game.module.runtime.dal.dataobject.pkg.RuntimePackageDO; +import com.wanxiang.huijing.game.module.runtime.dal.dataobject.storage.ArtifactStorageDO; +import com.wanxiang.huijing.game.module.runtime.dal.mysql.storage.ArtifactStorageMapper; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import static com.wanxiang.huijing.game.module.runtime.enums.ErrorCodeConstants.RUNTIME_PACKAGE_ARTIFACT_MISMATCH; +import static com.wanxiang.huijing.game.module.runtime.enums.ErrorCodeConstants.RUNTIME_PACKAGE_NOT_READY; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** committed 对象记录与运行包 checksum/bundle hash 的消费门测试。 */ +class ArtifactStorageGateTest { + + private static final String BUNDLE_HASH = "1e6ed65d77d6364eeaed5a745ba5c4985ae2b700dd85d7cf7f027bdf294a33fc"; + + @Test + void disabledGateKeepsLegacyReadPath() { + ArtifactStorageMapper mapper = mock(ArtifactStorageMapper.class); + ArtifactStorageGate gate = new ArtifactStorageGate(); + ReflectionTestUtils.setField(gate, "artifactStorageMapper", mapper); + ReflectionTestUtils.setField(gate, "enforceCommitted", false); + + assertDoesNotThrow(() -> gate.requireCommitted(pkg("a".repeat(64)))); + } + + @Test + void enabledGateRejectsMissingCommittedRecord() { + ArtifactStorageMapper mapper = mock(ArtifactStorageMapper.class); + when(mapper.selectCommittedByIdentity(1L, 1024L, 2048L)).thenReturn(null); + ArtifactStorageGate gate = configuredGate(mapper); + + ServiceException ex = assertThrows(ServiceException.class, + () -> gate.requireCommitted(pkg("a".repeat(64)))); + assertEquals(RUNTIME_PACKAGE_NOT_READY.getCode(), ex.getCode()); + } + + @Test + void enabledGateRejectsStaleRuntimeManifestOrMissingBundleHash() { + ArtifactStorageMapper mapper = mock(ArtifactStorageMapper.class); + ArtifactStorageDO stale = committed("b".repeat(64), "c".repeat(64)); + when(mapper.selectCommittedByIdentity(1L, 1024L, 2048L)).thenReturn(stale); + ArtifactStorageGate gate = configuredGate(mapper); + + ServiceException ex = assertThrows(ServiceException.class, + () -> gate.requireCommitted(pkg("a".repeat(64)))); + assertEquals(RUNTIME_PACKAGE_ARTIFACT_MISMATCH.getCode(), ex.getCode()); + + stale.setRuntimeManifestHash("a".repeat(64)); + stale.setBundleHash(""); + ex = assertThrows(ServiceException.class, () -> gate.requireCommitted(pkg("a".repeat(64)))); + assertEquals(RUNTIME_PACKAGE_ARTIFACT_MISMATCH.getCode(), ex.getCode()); + } + + @Test + void enabledGateAllowsCommittedHashes() { + ArtifactStorageMapper mapper = mock(ArtifactStorageMapper.class); + when(mapper.selectCommittedByIdentity(1L, 1024L, 2048L)) + .thenReturn(committed("a".repeat(64), BUNDLE_HASH)); + ArtifactStorageGate gate = configuredGate(mapper); + + assertDoesNotThrow(() -> gate.requireCommitted(pkg("a".repeat(64)))); + } + + @Test + void enabledGateAllowsManifestWithoutOptionalBundle() { + ArtifactStorageMapper mapper = mock(ArtifactStorageMapper.class); + when(mapper.selectCommittedByIdentity(1L, 1024L, 2048L)) + .thenReturn(committed("a".repeat(64), "")); + ArtifactStorageGate gate = configuredGate(mapper); + RuntimePackageDO runtimePackage = pkg("a".repeat(64)); + runtimePackage.setPackageJson("{\"schemaVersion\":\"1.0\"}"); + + assertDoesNotThrow(() -> gate.requireCommitted(runtimePackage)); + verify(mapper).selectCommittedByIdentity(1L, 1024L, 2048L); + } + + private static ArtifactStorageGate configuredGate(ArtifactStorageMapper mapper) { + ArtifactStorageGate gate = new ArtifactStorageGate(); + ReflectionTestUtils.setField(gate, "artifactStorageMapper", mapper); + ReflectionTestUtils.setField(gate, "enforceCommitted", true); + return gate; + } + + private static RuntimePackageDO pkg(String checksum) { + RuntimePackageDO pkg = new RuntimePackageDO(); + pkg.setTenantId(1L); + pkg.setGameId(1024L); + pkg.setVersionId(2048L); + pkg.setChecksum(checksum); + pkg.setPackageJson("{\"engineBundle\":\"bundle\"}"); + return pkg; + } + + private static ArtifactStorageDO committed(String runtimeManifestHash, String bundleHash) { + ArtifactStorageDO storage = new ArtifactStorageDO(); + storage.setVersionId(2048L); + storage.setStatus("committed"); + storage.setRuntimeManifestHash(runtimeManifestHash); + storage.setBundleHash(bundleHash); + return storage; + } +} diff --git a/game-cloud/game-module-runtime/game-module-runtime-server/src/test/java/com/wanxiang/huijing/game/module/runtime/service/pkg/RuntimePackageServiceImplTest.java b/game-cloud/game-module-runtime/game-module-runtime-server/src/test/java/com/wanxiang/huijing/game/module/runtime/service/pkg/RuntimePackageServiceImplTest.java index ccd9e37a..d2c055cd 100644 --- a/game-cloud/game-module-runtime/game-module-runtime-server/src/test/java/com/wanxiang/huijing/game/module/runtime/service/pkg/RuntimePackageServiceImplTest.java +++ b/game-cloud/game-module-runtime/game-module-runtime-server/src/test/java/com/wanxiang/huijing/game/module/runtime/service/pkg/RuntimePackageServiceImplTest.java @@ -49,6 +49,9 @@ class RuntimePackageServiceImplTest extends BaseMockitoUnitTest { @Mock private ProjectApi projectApi; + @Mock + private ArtifactStorageGate artifactStorageGate; + @AfterEach void clearSecurityContext() { org.springframework.security.core.context.SecurityContextHolder.clearContext(); diff --git a/game-cloud/huijing-server/src/main/resources/db/migration/V34.0.0__create_game_artifact_storage.sql b/game-cloud/huijing-server/src/main/resources/db/migration/V34.0.0__create_game_artifact_storage.sql new file mode 100644 index 00000000..8ba0a998 --- /dev/null +++ b/game-cloud/huijing-server/src/main/resources/db/migration/V34.0.0__create_game_artifact_storage.sql @@ -0,0 +1,43 @@ +-- ============================================================================= +-- 契约 #2 DB 迁移 | 模块：runtime/storage（对象存储版本状态）| owner：WS3 + WS2 +-- 文件：V34.0.0__create_game_artifact_storage.sql +-- 目的：保存对象存储清单的桶、key、hash、源修订身份和 pending/committed 状态。 +-- 边界：本表只负责对象存储提交状态；运行包自身的取包状态仍由 runtime 表管理。 +-- ============================================================================= + +CREATE TABLE `game_artifact_storage` ( + `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT '对象存储记录 ID', + `tenant_id` BIGINT NOT NULL DEFAULT 0 COMMENT '租户 ID', + `game_id` BIGINT NOT NULL COMMENT '游戏项目 ID（project.game_project.id）', + `version_id` BIGINT NOT NULL COMMENT '游戏版本 ID（project.game_version.id）', + `artifact_bucket` VARCHAR(128) NOT NULL DEFAULT '' COMMENT '运行包/素材/证据清单所在桶', + `artifact_manifest_key` VARCHAR(512) NOT NULL DEFAULT '' COMMENT 'artifact-manifest.json 对象 key', + `artifact_manifest_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT 'artifact-manifest.json 原始字节 SHA-256', + `artifact_manifest_bytes` BIGINT NOT NULL DEFAULT 0 COMMENT 'artifact-manifest.json 字节数', + `runtime_manifest_key` VARCHAR(512) NOT NULL DEFAULT '' COMMENT 'GamePackage manifest.json 对象 key', + `runtime_manifest_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT 'GamePackage manifest.json 原始字节 SHA-256；与 runtime checksum 对账', + `source_bucket` VARCHAR(128) NOT NULL DEFAULT '' COMMENT '引擎无关源归档所在桶；Tier2 外部源修订可为空', + `source_provider` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '源修订 provider：tier2_source_project/game_source_archive', + `source_game_id` VARCHAR(128) NOT NULL DEFAULT '' COMMENT '源修订所属稳定标识', + `source_revision_id` VARCHAR(128) NOT NULL DEFAULT '' COMMENT '源修订不可变标识', + `source_manifest_key` VARCHAR(512) NOT NULL DEFAULT '' COMMENT 'GameSourceArchive source-manifest.json key', + `source_manifest_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT 'source-manifest.json 原始字节 SHA-256', + `source_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT '源文件树 SHA-256', + `package_type` VARCHAR(16) NOT NULL DEFAULT '' COMMENT '运行包形态：phaser/littlejs/canvas', + `object_count` INT NOT NULL DEFAULT 0 COMMENT 'artifact manifest 正文对象数量', + `total_bytes` BIGINT NOT NULL DEFAULT 0 COMMENT '运行包、素材和证据对象总字节数', + `asset_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT '素材 tree hash', + `bundle_hash` CHAR(64) NOT NULL DEFAULT '' COMMENT '运行 bundle SHA-256；无 bundle 时为空', + `status` VARCHAR(16) NOT NULL DEFAULT 'pending' COMMENT '对象状态：pending=已上传并回读校验；committed=数据库 CAS 提交；abandoned=失败回收', + `failure_reason` VARCHAR(500) NOT NULL DEFAULT '' COMMENT '失败或回收原因，正常状态为空', + `committed_at` DATETIME NULL COMMENT '数据库 CAS 成功时间', + `creator` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '创建者', + `create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `updater` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '更新者', + `update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `deleted` BIT(1) NOT NULL DEFAULT b'0' COMMENT '逻辑删除标志；不可用它替换同一 game/version 记录', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_tenant_game_version` (`tenant_id`, `game_id`, `version_id`) COMMENT '同一租户游戏版本不可绑定第二份对象清单', + KEY `idx_status_created` (`status`, `create_time`) COMMENT '扫描 pending/abandoned 记录', + KEY `idx_manifest_hash` (`artifact_manifest_hash`) COMMENT '按清单 hash 查询幂等记录' +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COMMENT = '游戏对象存储版本状态记录（与运行包状态分离）'; diff --git a/game-runtime/.gitignore b/game-runtime/.gitignore index 3b006008..0e9ed5ae 100644 --- a/game-runtime/.gitignore +++ b/game-runtime/.gitignore @@ -1,17 +1,5 @@ -# ── 游戏生成线运行时 / 生成产物:可重生成、本地跑出、不入库(2026-07-23)── -# 以下三处均为 a-model 生成线与 Actor 校准的批量产物,tracked=0(已用 git ls-files 逐一复核), -# 纯产物、可重生成,长期污染 git status,故整目录忽略。 -# 遵「专一不笼统」红线: -# - amgen-* = a-model 生成的游戏目录(bundle.iife.js/entry-bundle.js/trace.jsonl/turns.jsonl/evidence/src/...), -# 其中 src/ 是该「生成游戏」自身的产出源码,非本仓手写工程源码;全部 93 个 amgen-* 均 tracked=0,故整前缀忽略。 -# - 绝不忽略 _wg1-gen(B 组 tracked=197,含 _shared 真实新源码与精选参考游戏)与各 _template*/_tier2-gen 等目录。 -# - _amgen-archive 以 _ 开头,不被 amgen-* 匹配,单独列出。 +# ── 游戏运行目录只作临时物化根；具体游戏由仓根 .gitignore 统一排除（2026-07-29）── +# 本文件只保留运行时局部缓存规则；通用模板/共享工具的豁免也由仓根维护，避免两套口径漂移。 # Actor 校准 attempt 产物(attempt-*/ + calibration-summary.json + runtime-log.jsonl,4787 个,tracked=0) evidence/match3-actor-calibration/ - -# a-model 生成的游戏目录(93 个,tracked=0)——覆盖根 .gitignore 已忽略的具体前缀之外的全部 amgen-* -games/amgen-*/ - -# a-model 生成归档(tracked=0) -games/_amgen-archive/ diff --git a/tier2/config/infra.yaml b/tier2/config/infra.yaml index d8166a62..d96501b5 100644 --- a/tier2/config/infra.yaml +++ b/tier2/config/infra.yaml @@ -57,3 +57,14 @@ minio: bucket: "tier2-src" # tier2 源文件全文桶(U2 落库前先 mb 建桶) kb_bucket: "cheap-kb" # 便宜档知识件外置知识包桶(W-CFG-KB;与 tier2-src 隔离、另立) secure: false # 内网 http(非 https) + +# ── 游戏内容制品:专用最小权限写入身份（不复用上面的 MinIO root）── +# 只允许 Put/Get 和 List 两个桶的 tenants/ 前缀，并允许读取桶区域；无 ListAll/Delete/建桶/策略管理权。 +# 上传器只读本分区，缺失即拒绝联网。凭据权威副本见 docs/内网凭据与端点.md。 +game_artifact_minio: + endpoint: "100.64.0.8:9000" + access_key: "game-artifact-writer" + secret_key: "4aa0a3f0036aac13cded3942c056df2c270b99cb7962624b" + artifact_bucket: "game-artifacts" + evidence_bucket: "game-evidence" + secure: false # 当前 Tailscale 内网；公网/生产必须切 TLS diff --git a/tier2/gen-worker/requirements.txt b/tier2/gen-worker/requirements.txt index 57b60e98..a878c769 100644 --- a/tier2/gen-worker/requirements.txt +++ b/tier2/gen-worker/requirements.txt @@ -12,6 +12,7 @@ agentscope[service,storage]==2.0.3 openai>=2.41.1 # 便宜档（deepseek-v4 等）走 OpenAI 兼容端点（new-api） json-repair # 脏 JSON 兜底（承袭裸路） pyyaml # 模型路由 models.yaml + infra.yaml 基建配置 +jsonschema>=4.23,<5 # GameArtifactManifest 与 GamePackage Draft 2020-12 契约校验 # U2 第二装载落库（BackendStore，仅 TIER2_STORE=backend 启用；worker/store.py 内惰性 import， # 6c6g 不装这俩也能 py_compile / import store.py；真跑/部署在 mini-desktop 时装）。 diff --git a/tier2/gen-worker/scripts/game_artifact_sync.py b/tier2/gen-worker/scripts/game_artifact_sync.py new file mode 100644 index 00000000..57a539b7 --- /dev/null +++ b/tier2/gen-worker/scripts/game_artifact_sync.py @@ -0,0 +1,144 @@ +"""游戏包对象存储同步 CLI。 + +本地合成夹具示例（只构建 manifest，绝不上传）： + PYTHONPATH=tier2/gen-worker python3 tier2/gen-worker/scripts/game_artifact_sync.py \ + --root /tmp/synthetic-game-artifact-fixture \ + --tenant-id fixture --game-id 9000001 --version-id 9000002 --package-type phaser \ + --runtime-manifest game-package.json --source-provider tier2_source_project \ + --source-game-id synthetic-fixture --source-revision-id fixture-r1 \ + --source-manifest-ref tier2-source-project:synthetic-fixture:fixture-r1 --source-hash + +默认只构建本地 manifest。波次 1 上传还必须显式传 --external-single-writer，表示调用方已在本工具外 +持有该 tenant/game/version 的单写锁；上传结果只到 verified_pending，不代表数据库 committed。 +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + +from worker.game_artifact_store import ( + ArtifactError, + build_manifest, + download_manifest, + fetch_manifest, + upload_manifest, +) + + +def _config(bucket: str) -> dict: + """读取专用 artifact-writer 配置；禁止回落复用现有 MinIO root 凭据。""" + try: + from service import infra_config + config = { + "endpoint": os.getenv("GAME_ARTIFACT_ENDPOINT", infra_config.get("game_artifact_minio", "endpoint", "")), + "access_key": os.getenv("GAME_ARTIFACT_ACCESS_KEY", infra_config.get("game_artifact_minio", "access_key", "")), + "secret_key": os.getenv("GAME_ARTIFACT_SECRET_KEY", infra_config.get("game_artifact_minio", "secret_key", "")), + "secure": os.getenv("GAME_ARTIFACT_SECURE", str(infra_config.get("game_artifact_minio", "secure", True))).lower() in {"1", "true", "yes"}, + "artifact_bucket": os.getenv("GAME_ARTIFACT_BUCKET", infra_config.get("game_artifact_minio", "artifact_bucket", bucket)), + "evidence_bucket": os.getenv("GAME_EVIDENCE_BUCKET", infra_config.get("game_artifact_minio", "evidence_bucket", "game-evidence")), + } + if not config["endpoint"] or not config["access_key"] or not config["secret_key"]: + raise ArtifactError("专用 game_artifact_minio 配置未就位，拒绝复用现有 MinIO root 凭据") + return config + except Exception as exc: # noqa: BLE001 - CLI 在缺 tier2 配置时给出稳定错误 + raise ArtifactError(f"读取 MinIO 配置失败:{exc}") from exc + + +def main() -> int: + parser = argparse.ArgumentParser(description="构建、上传或下载不可变游戏内容版本") + parser.add_argument("--root", type=Path, help="具体游戏目录（只作为本次上传输入）") + parser.add_argument("--tenant-id") + parser.add_argument("--game-id") + parser.add_argument("--version-id") + parser.add_argument("--package-type", choices=["phaser", "littlejs", "canvas"]) + parser.add_argument("--runtime-manifest", help="游戏目录内现有 GamePackage/1 JSON 相对路径") + parser.add_argument("--source-provider", choices=["tier2_source_project", "game_source_archive"]) + parser.add_argument("--source-game-id") + parser.add_argument("--source-revision-id") + parser.add_argument("--source-manifest-ref") + parser.add_argument("--source-hash") + parser.add_argument("--bucket", default="game-artifacts") + parser.add_argument("--manifest-out", type=Path) + parser.add_argument("--upload", action="store_true", help="显式访问 MinIO 并上传") + parser.add_argument( + "--external-single-writer", action="store_true", + help="确认调用方已持有外部单写锁；波次 1 不提供对象存储条件写", + ) + parser.add_argument("--download-key", help="受信 artifact-manifest 对象 key；不接受任意 URL") + parser.add_argument("--expected-manifest-hash", help="数据库登记的 artifact manifest SHA-256") + parser.add_argument("--target", type=Path, help="下载物化目录") + args = parser.parse_args() + try: + if args.download_key: + if args.upload or args.external_single_writer or not all((args.target, args.expected_manifest_hash, + args.tenant_id, args.game_id, args.version_id)): + raise ArtifactError("下载模式必须提供 --target、期望 hash 和 tenant/game/version，且不能带上传参数") + manifest = fetch_manifest( + _config(args.bucket), key=args.download_key, + expected_manifest_hash=args.expected_manifest_hash, + expected_tenant_id=args.tenant_id, + expected_game_id=args.game_id, + expected_version_id=args.version_id, + ) + result = download_manifest( + manifest, _config(args.bucket), args.target, + expected_manifest_hash=args.expected_manifest_hash, + ) + print(json.dumps({ + "gameId": manifest["gameId"], + "versionId": manifest["versionId"], + "manifestHash": manifest["manifestHash"], + "target": str(result), + }, ensure_ascii=False)) + return 0 + required = ( + args.root, args.tenant_id, args.game_id, args.version_id, args.package_type, + args.runtime_manifest, args.source_provider, args.source_game_id, + args.source_revision_id, args.source_manifest_ref, args.source_hash, + ) + if not all(required): + raise ArtifactError("构建/上传模式缺少生产版本、GamePackage 或 sourceRevision 必填参数") + manifest = build_manifest(args.root, args.tenant_id, args.game_id, args.version_id, + package_type=args.package_type, + runtime_manifest_path=args.runtime_manifest, + source_revision={ + "provider": args.source_provider, + "gameId": args.source_game_id, + "revisionId": args.source_revision_id, + "manifestRef": args.source_manifest_ref, + "sourceHash": args.source_hash, + }) + payload = json.dumps(manifest, ensure_ascii=False, indent=2) + "\n" + if args.manifest_out: + args.manifest_out.parent.mkdir(parents=True, exist_ok=True) + args.manifest_out.write_text(payload, encoding="utf-8") + print(json.dumps({ + "tenantId": manifest["tenantId"], + "gameId": manifest["gameId"], + "versionId": manifest["versionId"], + "objects": manifest["totals"]["objects"], + "bytes": manifest["totals"]["bytes"], + "sourceHash": manifest["sourceRevision"]["sourceHash"], + "assetHash": manifest["assetHash"], + "bundleHash": manifest["bundleHash"], + "manifestHash": manifest["manifestHash"], + "upload": args.upload, + }, ensure_ascii=False)) + if args.upload: + if not args.external_single_writer: + raise ArtifactError("波次 1 上传必须显式确认 --external-single-writer") + result = upload_manifest( + args.root, manifest, _config(args.bucket), external_single_writer=True, + ) + print(json.dumps(result, ensure_ascii=False)) + return 0 + except ArtifactError as exc: + print(f"GAME-ARTIFACT-ERROR:{exc}") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tier2/gen-worker/scripts/game_source_archive_sync.py b/tier2/gen-worker/scripts/game_source_archive_sync.py new file mode 100644 index 00000000..4c88eccd --- /dev/null +++ b/tier2/gen-worker/scripts/game_source_archive_sync.py @@ -0,0 +1,133 @@ +"""引擎无关游戏源归档同步 CLI。 + +默认只构建本地 GameSourceArchive/1 清单。波次 1 上传必须显式传 +``--external-single-writer``，并且只返回 ``verified_pending``，不能把对象 +marker 当成数据库 committed 状态。 +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + +from worker.game_source_archive_store import ( + ArtifactError, + build_source_archive, + download_source_archive, + fetch_source_archive, + upload_source_archive, +) + + +def _config() -> dict: + """读取专用 game_artifact_minio 配置，禁止回落到 MinIO root。""" + try: + from service import infra_config + + section = "game_artifact_minio" + config = { + "endpoint": os.getenv("GAME_ARTIFACT_ENDPOINT", infra_config.get(section, "endpoint", "")), + "access_key": os.getenv("GAME_ARTIFACT_ACCESS_KEY", infra_config.get(section, "access_key", "")), + "secret_key": os.getenv("GAME_ARTIFACT_SECRET_KEY", infra_config.get(section, "secret_key", "")), + "secure": os.getenv("GAME_ARTIFACT_SECURE", str(infra_config.get(section, "secure", True))).lower() + in {"1", "true", "yes"}, + "artifact_bucket": os.getenv("GAME_ARTIFACT_BUCKET", infra_config.get(section, "artifact_bucket", "")), + "evidence_bucket": os.getenv("GAME_EVIDENCE_BUCKET", infra_config.get(section, "evidence_bucket", "")), + "source_bucket": os.getenv("GAME_SOURCE_BUCKET", infra_config.get(section, "source_bucket", "")), + } + if not config["endpoint"] or not config["access_key"] or not config["secret_key"]: + raise ArtifactError("专用 game_artifact_minio 配置未就位，拒绝复用现有 MinIO root 凭据") + if not config["source_bucket"]: + raise ArtifactError("game-sources 尚未预建或加入专用身份策略，拒绝联网上传") + if config["source_bucket"] in {config["artifact_bucket"], config["evidence_bucket"]}: + raise ArtifactError("source_bucket 不能与 artifact/evidence 桶复用") + return config + except Exception as exc: # noqa: BLE001 - CLI 给出稳定错误 + raise ArtifactError(f"读取 MinIO 配置失败:{exc}") from exc + + +def main() -> int: + parser = argparse.ArgumentParser(description="构建、上传或下载引擎无关游戏源归档") + parser.add_argument("--root", type=Path, help="具体游戏源目录（只作为本次上传输入）") + parser.add_argument("--tenant-id") + parser.add_argument("--game-id") + parser.add_argument("--revision-id") + parser.add_argument("--engine") + parser.add_argument("--runtime-manifest", default="game-package.json", + help="与 GameArtifactManifest 同一输入根中的 GamePackage/1 相对路径") + parser.add_argument("--exclude-path", action="append", default=[], help="不进入源归档的目录内相对路径") + parser.add_argument("--manifest-out", type=Path) + parser.add_argument("--upload", action="store_true", help="显式访问 MinIO 并上传") + parser.add_argument( + "--external-single-writer", action="store_true", + help="确认调用方已持有 tenant/game/revision 外部单写锁", + ) + parser.add_argument("--download-key", help="受信 source-manifest 对象 key；不接受任意 URL") + parser.add_argument("--expected-manifest-hash", help="数据库登记的 source manifest SHA-256") + parser.add_argument("--target", type=Path, help="下载物化目录") + args = parser.parse_args() + try: + if args.download_key: + if args.upload or args.external_single_writer or not all(( + args.target, args.expected_manifest_hash, args.tenant_id, + args.game_id, args.revision_id, + )): + raise ArtifactError("下载模式必须提供 target、期望 hash 和 tenant/game/revision，且不能带上传参数") + config = _config() + manifest = fetch_source_archive( + config, key=args.download_key, + expected_manifest_hash=args.expected_manifest_hash, + expected_tenant_id=args.tenant_id, + expected_game_id=args.game_id, + expected_revision_id=args.revision_id, + ) + target = download_source_archive( + manifest, config, args.target, expected_manifest_hash=args.expected_manifest_hash, + ) + print(json.dumps({ + "gameId": manifest["gameId"], + "revisionId": manifest["revisionId"], + "sourceHash": manifest["sourceHash"], + "manifestHash": manifest["manifestHash"], + "target": str(target), + }, ensure_ascii=False)) + return 0 + + if not all((args.root, args.tenant_id, args.game_id, args.revision_id, args.engine)): + raise ArtifactError("构建/上传模式缺少 root、tenant/game/revision 或 engine") + manifest = build_source_archive( + args.root, args.tenant_id, args.game_id, args.revision_id, + engine=args.engine, exclude_paths=args.exclude_path, + runtime_manifest_path=args.runtime_manifest, + ) + payload = json.dumps(manifest, ensure_ascii=False, indent=2) + "\n" + if args.manifest_out: + args.manifest_out.parent.mkdir(parents=True, exist_ok=True) + args.manifest_out.write_text(payload, encoding="utf-8") + print(json.dumps({ + "tenantId": manifest["tenantId"], + "gameId": manifest["gameId"], + "revisionId": manifest["revisionId"], + "objects": manifest["totals"]["objects"], + "bytes": manifest["totals"]["bytes"], + "sourceHash": manifest["sourceHash"], + "manifestHash": manifest["manifestHash"], + "upload": args.upload, + }, ensure_ascii=False)) + if args.upload: + if not args.external_single_writer: + raise ArtifactError("波次 1 上传必须显式确认 --external-single-writer") + result = upload_source_archive( + args.root, manifest, _config(), external_single_writer=True, + ) + print(json.dumps(result, ensure_ascii=False)) + return 0 + except ArtifactError as exc: + print(f"GAME-SOURCE-ARCHIVE-ERROR:{exc}") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tier2/gen-worker/tests/test_game_artifact_storage_ddl.py b/tier2/gen-worker/tests/test_game_artifact_storage_ddl.py new file mode 100644 index 00000000..2a734782 --- /dev/null +++ b/tier2/gen-worker/tests/test_game_artifact_storage_ddl.py @@ -0,0 +1,47 @@ +"""对象存储状态记录的 DB 契约与 Flyway 执行副本对账门。""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +CONTRACT = ROOT / "contracts/db-schemas/V34.0.0__create_game_artifact_storage.sql" +EXECUTION = ROOT / "game-cloud/huijing-server/src/main/resources/db/migration/V34.0.0__create_game_artifact_storage.sql" + + +def _norm_sql(text: str) -> str: + """去注释、分号和空白，比较 SQL token 而不是排版。""" + lines = [line for line in text.splitlines() if not line.strip().startswith("--")] + return " ".join(" ".join(lines).replace(";", " ").split()).strip() + + +def test_storage_record_contract_has_flyway_execution_copy(): + assert CONTRACT.exists(), f"缺 DB 契约:{CONTRACT}" + assert EXECUTION.exists(), f"缺 Flyway 执行副本:{EXECUTION}" + assert _norm_sql(CONTRACT.read_text(encoding="utf-8")) == _norm_sql( + EXECUTION.read_text(encoding="utf-8"), + ) + + +def test_storage_record_is_immutable_per_tenant_game_version(): + sql = CONTRACT.read_text(encoding="utf-8") + assert "CREATE TABLE `game_artifact_storage`" in sql + assert "`artifact_bucket`" in sql + assert "`artifact_manifest_key`" in sql + assert "`artifact_manifest_hash`" in sql + assert "`runtime_manifest_key`" in sql + assert "`runtime_manifest_hash`" in sql + assert "`source_provider`" in sql + assert "`source_manifest_key`" in sql + assert "`source_manifest_hash`" in sql + assert "`source_hash`" in sql + assert "`status`" in sql + assert "pending" in sql and "committed" in sql and "abandoned" in sql + assert "UNIQUE KEY `uk_tenant_game_version` (`tenant_id`, `game_id`, `version_id`)" in sql + assert "KEY `idx_status_created` (`status`, `create_time`)" in sql + + +def test_storage_record_cas_columns_are_not_runtime_release_state(): + """对象提交状态必须独立于 game_runtime_package.status。""" + sql = CONTRACT.read_text(encoding="utf-8") + assert "game_runtime_package" not in sql + assert "release" not in sql.lower() diff --git a/tier2/gen-worker/tests/test_game_artifact_storage_store.py b/tier2/gen-worker/tests/test_game_artifact_storage_store.py new file mode 100644 index 00000000..3edab785 --- /dev/null +++ b/tier2/gen-worker/tests/test_game_artifact_storage_store.py @@ -0,0 +1,213 @@ +"""game_artifact_storage writer 的条件创建/CAS 行为测试。""" + +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from test_game_artifact_store import _make_game +from worker import game_artifact_store as artifact_store +from worker import game_source_archive_store as source_store +from worker import game_artifact_storage_store as store + + +def _record(*, artifact_hash: str = "a" * 64, source_hash: str = "b" * 64) -> dict: + """构造已通过上游 manifest 校验的存储状态记录。""" + return { + "tenant_id": 1, + "game_id": 1024, + "version_id": 2048, + "artifact_bucket": "game-artifacts", + "artifact_manifest_key": "tenants/1/games/1024/versions/2048/artifact-manifest.json", + "artifact_manifest_hash": artifact_hash, + "artifact_manifest_bytes": 512, + "runtime_manifest_key": "tenants/1/games/1024/versions/2048/manifest.json", + "runtime_manifest_hash": "6" * 64, + "source_bucket": "game-sources", + "source_provider": "game_source_archive", + "source_game_id": "1024", + "source_revision_id": "shanhai-r1", + "source_manifest_key": "tenants/1/games/1024/source-revisions/shanhai-r1/source-manifest.json", + "source_manifest_hash": "c" * 64, + "source_hash": source_hash, + "package_type": "littlejs", + "object_count": 3, + "total_bytes": 4096, + "asset_hash": "d" * 64, + "bundle_hash": "e" * 64, + "status": "pending", + "failure_reason": "", + "creator": "test", + "updater": "test", + } + + +class _Duplicate(Exception): + errno = 1062 + + +class _Cursor: + def __init__(self, rows: list[dict]): + self.rows = rows + self.last = None + self.rowcount = 0 + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def execute(self, sql: str, params=()): + upper = " ".join(sql.split()).upper() + if upper.startswith("INSERT INTO"): + ( + tenant_id, game_id, version_id, artifact_bucket, artifact_key, artifact_hash, + artifact_bytes, runtime_key, runtime_hash, source_bucket, source_provider, source_game_id, source_revision_id, + source_key, source_manifest_hash, source_hash, package_type, object_count, + total_bytes, asset_hash, bundle_hash, status, failure_reason, creator, updater, + ) = params + if any( + row["tenant_id"] == tenant_id and row["game_id"] == game_id and row["version_id"] == version_id + for row in self.rows + ): + raise _Duplicate("duplicate version") + row = _record(artifact_hash=artifact_hash, source_hash=source_hash) + row.update({ + "tenant_id": tenant_id, "game_id": game_id, "version_id": version_id, + "artifact_bucket": artifact_bucket, "artifact_manifest_key": artifact_key, + "artifact_manifest_bytes": artifact_bytes, "source_bucket": source_bucket, + "runtime_manifest_key": runtime_key, "runtime_manifest_hash": runtime_hash, + "source_provider": source_provider, "source_game_id": source_game_id, + "source_revision_id": source_revision_id, "source_manifest_key": source_key, + "source_manifest_hash": source_manifest_hash, "package_type": package_type, + "object_count": object_count, "total_bytes": total_bytes, "asset_hash": asset_hash, + "bundle_hash": bundle_hash, "status": status, "failure_reason": failure_reason, + "creator": creator, "updater": updater, "id": len(self.rows) + 1, + }) + self.rows.append(row) + self.rowcount = 1 + return self.rowcount + if upper.startswith("SELECT"): + tenant_id, game_id, version_id = params[:3] + self.last = next(( + dict(row) for row in self.rows + if row["tenant_id"] == tenant_id and row["game_id"] == game_id and row["version_id"] == version_id + ), None) + self.rowcount = 0 + return 0 + if upper.startswith("UPDATE"): + ( + committed_at, updater, tenant_id, game_id, version_id, + artifact_hash, source_hash, source_manifest_hash, + ) = params + self.rowcount = 0 + for row in self.rows: + if ( + row["tenant_id"], row["game_id"], row["version_id"], row["status"], + row["artifact_manifest_hash"], row["source_hash"], row["source_manifest_hash"], + ) == ( + tenant_id, game_id, version_id, "pending", artifact_hash, source_hash, + source_manifest_hash, + ): + row["status"] = "committed" + row["committed_at"] = committed_at + row["updater"] = updater + self.rowcount = 1 + return self.rowcount + raise AssertionError(f"未覆盖 SQL:{sql}") + + def fetchone(self): + return self.last + + +class _Connection: + def __init__(self): + self.rows: list[dict] = [] + self.commits = 0 + self.rollbacks = 0 + + def cursor(self): + return _Cursor(self.rows) + + def commit(self): + self.commits += 1 + + def rollback(self): + self.rollbacks += 1 + + def close(self): + return None + + +def test_create_pending_is_idempotent_and_immutable(): + connection = _Connection() + writer = store.MySqlArtifactStorageStore(lambda: connection) + record = _record() + + first = writer.create_pending(record) + second = writer.create_pending(record) + assert first["status"] == "pending" + assert first["idempotent"] is False + assert second["idempotent"] is True + assert second["id"] == first["id"] + assert len(connection.rows) == 1 + + changed = _record(artifact_hash="f" * 64) + with pytest.raises(store.StorageRecordError, match="不可覆盖"): + writer.create_pending(changed) + assert len(connection.rows) == 1 + + +def test_commit_is_hash_conditioned_and_repeat_is_idempotent(): + connection = _Connection() + writer = store.MySqlArtifactStorageStore(lambda: connection) + record = _record() + writer.create_pending(record) + committed_at = datetime(2026, 7, 29, tzinfo=timezone.utc) + + first = writer.commit_pending(record, committed_at=committed_at) + second = writer.commit_pending(record, committed_at=committed_at) + assert first["status"] == "committed" + assert first["idempotent"] is False + assert second["status"] == "committed" + assert second["idempotent"] is True + assert connection.rows[0]["committed_at"] == committed_at + + wrong = _record(source_hash="9" * 64) + with pytest.raises(store.StorageRecordError, match="CAS"): + writer.commit_pending(wrong, committed_at=committed_at) + + +def test_build_storage_record_binds_real_source_and_artifact_manifests(tmp_path: Path): + _make_game(tmp_path) + source_manifest = source_store.build_source_archive( + tmp_path, "1", "1024", "shanhai-r1", engine="custom-canvas-littlejs", + ) + artifact_manifest = artifact_store.build_manifest( + tmp_path, "1", "1024", "2048", package_type="littlejs", + runtime_manifest_path="game-package.json", + source_revision={ + "provider": "game_source_archive", + "gameId": "1024", + "revisionId": "shanhai-r1", + "manifestRef": "game-source-archive:1024:shanhai-r1", + "sourceHash": source_manifest["sourceHash"], + }, + ) + record = store.build_storage_record( + artifact_manifest, artifact_bucket="game-artifacts", + source_bucket="game-sources", source_archive=source_manifest, + ) + assert record["artifact_manifest_hash"] == artifact_manifest["manifestHash"] + assert record["source_manifest_hash"] == source_manifest["manifestHash"] + assert record["status"] == "pending" + + +def test_build_storage_record_requires_generic_source_manifest(): + record = _record() + record["source_bucket"] = "" + record["source_manifest_key"] = "" + record["source_manifest_hash"] = "" + with pytest.raises(store.StorageRecordError, match="source_bucket"): + store._validate_record(record) diff --git a/tier2/gen-worker/tests/test_game_artifact_store.py b/tier2/gen-worker/tests/test_game_artifact_store.py new file mode 100644 index 00000000..7c7917c4 --- /dev/null +++ b/tier2/gen-worker/tests/test_game_artifact_store.py @@ -0,0 +1,477 @@ +"""游戏内容对象存储契约的离线测试。""" + +import hashlib +import json +import subprocess +import sys +from io import BytesIO +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from worker import game_artifact_store as store +from worker import game_source_archive_store as source_store + + +TENANT_ID = "1" +GAME_ID = "1024" +VERSION_ID = "2048" + + +def _make_game(root: Path, *, package_game_id: str = GAME_ID, package_version_id: str = VERSION_ID) -> None: + """创建含完整 GamePackage、源工程、素材、runtime 和证据的最小游戏包。""" + (root / "src").mkdir(parents=True) + (root / "assets").mkdir(parents=True) + (root / "dist").mkdir(parents=True) + (root / "evidence").mkdir(parents=True) + (root / "src" / "main.js").write_text("export const GAME = 1;\n", encoding="utf-8") + (root / "assets" / "hero.webp").write_bytes(b"asset") + (root / "dist" / "bundle.iife.js").write_bytes(b"bundle") + (root / "dist" / "scan.json").write_text("{}", encoding="utf-8") + (root / "evidence" / "qa.md").write_text("verified", encoding="utf-8") + game_package = { + "schemaVersion": "1.0", + "gameId": package_game_id, + "versionId": package_version_id, + "templateId": "runner", + "gameConfig": {}, + "assets": [], + "manifest": { + "runtimeVersion": "1.0.0", + "entry": "index.html", + "preloadPolicy": "eager", + "bundleSize": len(b"bundle"), + "checksum": "0" * 64, + }, + "meta": { + "title": "契约测试游戏", + "summary": "只用于平台契约测试", + "cover": "https://assets.invalid/cover.webp", + "ageRating": "all", + }, + } + (root / "game-package.json").write_text( + json.dumps(game_package, ensure_ascii=False, separators=(",", ":")), encoding="utf-8", + ) + (root / "src" / "__pycache__").mkdir() + (root / "src" / "__pycache__" / "ignored.pyc").write_bytes(b"cache") + + +def _source_revision(root: Path, *, source_hash: str | None = None) -> dict: + """构造指向现有 Tier2 SourceProjectStore 的不可变源修订身份。""" + if source_hash is None: + digest = hashlib.sha256() + digest.update(b"src/main.js\0") + digest.update("export const GAME = 1;\n\0".encode()) + source_hash = digest.hexdigest() + return { + "provider": "tier2_source_project", + "gameId": "night-market", + "revisionId": "m4-r1", + "manifestRef": "tier2-source-project:night-market:m4-r1", + "sourceHash": source_hash, + } + + +def _build(root: Path) -> dict: + return store.build_manifest( + root, TENANT_ID, GAME_ID, VERSION_ID, + package_type="phaser", + runtime_manifest_path="game-package.json", + source_revision=_source_revision(root), + ) + + +def test_build_and_verify_manifest(tmp_path: Path): + _make_game(tmp_path) + manifest = _build(tmp_path) + store.verify_manifest(manifest) + assert manifest["totals"]["objects"] == 4 + assert manifest["totals"]["assetBytes"] == len(b"asset") + assert manifest["bundleHash"] == store._hash_bytes(b"bundle") + assert manifest["runtimeManifest"]["key"].endswith("/manifest.json") + assert manifest["sourceRevision"]["revisionId"] == "m4-r1" + assert not any(item["category"] == "source" for item in manifest["objects"]) + assert not any("ignored.pyc" in item["path"] for item in manifest["objects"]) + + schema_path = Path(__file__).resolve().parents[3] / "contracts/game-artifact-manifest.schema.json" + schema = json.loads(schema_path.read_text(encoding="utf-8")) + errors = sorted(Draft202012Validator(schema).iter_errors(manifest), key=lambda item: list(item.path)) + assert not errors, [error.message for error in errors] + assert _build(tmp_path)["manifestHash"] == manifest["manifestHash"] + + +def test_manifest_hash_and_schema_detect_mutation(tmp_path: Path): + _make_game(tmp_path) + manifest = _build(tmp_path) + manifest["objects"][0]["bytes"] += 1 + with pytest.raises(store.ArtifactError, match="manifestHash"): + store.verify_manifest(manifest) + + manifest = _build(tmp_path) + manifest["status"] = "published" + unsigned = dict(manifest) + unsigned.pop("manifestHash") + manifest["manifestHash"] = store._hash_bytes(store._MANIFEST_HASH_DOMAIN + store._canonical_json(unsigned)) + with pytest.raises(store.ArtifactError, match="game-artifact-manifest.schema.json"): + store.verify_manifest(manifest) + + +def test_rejects_symlink_and_source_hash_drift(tmp_path: Path): + _make_game(tmp_path) + outside = tmp_path.parent / "outside-game-secret.txt" + outside.write_text("secret", encoding="utf-8") + (tmp_path / "src" / "linked.txt").symlink_to(outside) + with pytest.raises(store.ArtifactError, match="符号链接"): + _build(tmp_path) + (tmp_path / "src" / "linked.txt").unlink() + with pytest.raises(store.ArtifactError, match="源工程 hash"): + store.build_manifest( + tmp_path, TENANT_ID, GAME_ID, VERSION_ID, package_type="phaser", + runtime_manifest_path="game-package.json", + source_revision=_source_revision(tmp_path, source_hash="0" * 64), + ) + + +def test_game_package_contract_and_identity_are_enforced(tmp_path: Path): + _make_game(tmp_path, package_version_id="9999") + with pytest.raises(store.ArtifactError, match="对象版本不匹配"): + _build(tmp_path) + with pytest.raises(store.ArtifactError, match="数据库正整数"): + store.build_manifest( + tmp_path, TENANT_ID, "night-market", "m4-r1", package_type="phaser", + runtime_manifest_path="game-package.json", source_revision=_source_revision(tmp_path), + ) + + +class _Response: + """模拟 MinIO HTTPResponse，覆盖连接释放协议。""" + + def __init__(self, data: bytes): + self._data = data + self._offset = 0 + self.read_sizes: list[int] = [] + + def read(self, size: int = -1) -> bytes: + self.read_sizes.append(size) + if size < 0: + size = len(self._data) - self._offset + chunk = self._data[self._offset:self._offset + size] + self._offset += len(chunk) + return chunk + + def close(self) -> None: + return None + + def release_conn(self) -> None: + return None + + +class _NoSuchKey(Exception): + """模拟 MinIO S3Error 的稳定 code。""" + + code = "NoSuchKey" + + +class _NoSuchBucket(Exception): + """模拟桶缺失，必须与对象缺失采用不同失败语义。""" + + code = "NoSuchBucket" + + +class _FakeMinio: + """内存对象存储，用于验证上传、幂等、可信下载和原子物化。""" + + def __init__(self): + self.objects: dict[tuple[str, str], bytes] = {} + self.bucket_exists_calls = 0 + self.make_bucket_calls = 0 + self.put_calls: list[tuple[str, str]] = [] + self.last_response: _Response | None = None + + def bucket_exists(self, _bucket: str) -> bool: + self.bucket_exists_calls += 1 + raise AssertionError("上传器禁止探测/列出桶") + + def make_bucket(self, _bucket: str) -> None: + self.make_bucket_calls += 1 + + def put_object(self, bucket: str, key: str, stream: BytesIO, *, length: int, content_type: str): + data = stream.read() + assert len(data) == length + assert content_type + self.put_calls.append((bucket, key)) + self.objects[(bucket, key)] = data + + def get_object(self, bucket: str, key: str) -> _Response: + identity = (bucket, key) + if identity not in self.objects: + raise _NoSuchKey(key) + self.last_response = _Response(self.objects[identity]) + return self.last_response + + +def _trusted_fetch(store_config: dict, key: str, manifest: dict) -> dict: + return store.fetch_manifest( + store_config, key=key, + expected_manifest_hash=manifest["manifestHash"], + expected_tenant_id=TENANT_ID, + expected_game_id=GAME_ID, + expected_version_id=VERSION_ID, + ) + + +def test_upload_download_and_idempotency(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + game = tmp_path / "game" + _make_game(game) + manifest = _build(game) + client = _FakeMinio() + monkeypatch.setattr(store, "_minio_client", lambda _config: client) + config = {"artifact_bucket": "game-artifacts", "evidence_bucket": "game-evidence"} + first = store.upload_manifest(game, manifest, config, external_single_writer=True) + assert first["idempotent"] is False + assert first["status"] == "verified_pending" + assert first["committed"] is False + second = store.upload_manifest(game, manifest, config, external_single_writer=True) + assert second["idempotent"] is True + assert second["status"] == "verified_pending" + assert second["committed"] is False + prefix = f"tenants/{TENANT_ID}/games/{GAME_ID}/versions/{VERSION_ID}" + assert ("game-artifacts", f"{prefix}/assets/hero.webp") in client.objects + assert ("game-evidence", f"{prefix}/evidence/qa.md") in client.objects + assert not any("source/src/main.js" in key for _, key in client.objects) + + remote = _trusted_fetch(config, first["artifactManifestKey"], manifest) + target = tmp_path / "materialized" + store.download_manifest(remote, config, target, expected_manifest_hash=manifest["manifestHash"]) + assert not (target / "source").exists() + assert (target / "assets/hero.webp").read_bytes() == b"asset" + assert json.loads((target / "manifest.json").read_text(encoding="utf-8"))["schemaVersion"] == "1.0" + + changed = dict(manifest) + changed["sourceRevision"] = dict(manifest["sourceRevision"]) + changed["sourceRevision"]["revisionId"] = "m4-r2" + changed["sourceRevision"]["manifestRef"] = "tier2-source-project:night-market:m4-r2" + unsigned = dict(changed) + unsigned.pop("manifestHash") + changed["manifestHash"] = store._hash_bytes(store._MANIFEST_HASH_DOMAIN + store._canonical_json(unsigned)) + puts_before = list(client.put_calls) + marker_before = client.objects[("game-artifacts", first["artifactManifestKey"])] + with pytest.raises(store.ArtifactError, match="拒绝覆盖"): + store.upload_manifest(game, changed, config, external_single_writer=True) + assert client.put_calls == puts_before + assert client.objects[("game-artifacts", first["artifactManifestKey"])] == marker_before + + +def test_partial_objects_require_external_single_writer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """波次 1 可覆盖无 marker 的残留 partial，但必须由调用方持有外部单写锁。""" + _make_game(tmp_path) + manifest = _build(tmp_path) + client = _FakeMinio() + asset = next(item for item in manifest["objects"] if item["category"] == "asset") + client.objects[("game-artifacts", asset["key"])] = b"stale-partial" + monkeypatch.setattr(store, "_minio_client", lambda _config: client) + config = {"artifact_bucket": "game-artifacts", "evidence_bucket": "game-evidence"} + + with pytest.raises(store.ArtifactError, match="外部单写"): + store.upload_manifest(tmp_path, manifest, config) + assert client.objects[("game-artifacts", asset["key"])] == b"stale-partial" + + receipt = store.upload_manifest(tmp_path, manifest, config, external_single_writer=True) + assert receipt["status"] == "verified_pending" + assert receipt["committed"] is False + assert client.objects[("game-artifacts", asset["key"])] == b"asset" + + +def test_artifact_upload_preflights_runtime_manifest_before_put(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + _make_game(tmp_path) + manifest = _build(tmp_path) + (tmp_path / "game-package.json").write_text("drift", encoding="utf-8") + client = _FakeMinio() + monkeypatch.setattr(store, "_minio_client", lambda _config: client) + with pytest.raises(store.ArtifactError, match="GamePackage manifest 本地 hash 漂移"): + store.upload_manifest(tmp_path, manifest, { + "artifact_bucket": "game-artifacts", "evidence_bucket": "game-evidence", + }, external_single_writer=True) + assert client.put_calls == [] + + +def test_trusted_anchor_and_atomic_materialization(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + game = tmp_path / "game" + _make_game(game) + manifest = _build(game) + client = _FakeMinio() + monkeypatch.setattr(store, "_minio_client", lambda _config: client) + config = {"artifact_bucket": "game-artifacts", "evidence_bucket": "game-evidence"} + receipt = store.upload_manifest(game, manifest, config, external_single_writer=True) + with pytest.raises(store.ArtifactError, match="数据库期望 hash"): + store.fetch_manifest( + config, key=receipt["artifactManifestKey"], expected_manifest_hash="0" * 64, + expected_tenant_id=TENANT_ID, expected_game_id=GAME_ID, expected_version_id=VERSION_ID, + ) + + remote = _trusted_fetch(config, receipt["artifactManifestKey"], manifest) + asset = next(item for item in manifest["objects"] if item["category"] == "asset") + client.objects[("game-artifacts", asset["key"])] = b"wrong" + target = tmp_path / "materialized" + with pytest.raises(store.ArtifactError, match="下载对象 hash"): + store.download_manifest(remote, config, target, expected_manifest_hash=manifest["manifestHash"]) + assert not target.exists() + assert not list(tmp_path.glob(".materialized.partial-*")) + + +def test_fetch_rejects_untrusted_key(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(store, "_minio_client", lambda _config: _FakeMinio()) + with pytest.raises(store.ArtifactError, match="key 非法"): + store.fetch_manifest( + {}, key="https://attacker.invalid/artifact-manifest.json", + expected_manifest_hash="0" * 64, expected_tenant_id=TENANT_ID, + expected_game_id=GAME_ID, expected_version_id=VERSION_ID, + ) + + +def test_upload_never_probes_or_creates_bucket(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + _make_game(tmp_path) + client = _FakeMinio() + monkeypatch.setattr(store, "_minio_client", lambda _config: client) + store.upload_manifest(tmp_path, _build(tmp_path), {}, external_single_writer=True) + assert client.bucket_exists_calls == 0 + assert client.make_bucket_calls == 0 + + +def test_missing_runtime_manifest_is_rejected(tmp_path: Path): + _make_game(tmp_path) + with pytest.raises(store.ArtifactError, match="GamePackage manifest 不存在"): + store.build_manifest( + tmp_path, TENANT_ID, GAME_ID, VERSION_ID, package_type="phaser", + runtime_manifest_path="missing.json", source_revision=_source_revision(tmp_path), + ) + + +def test_cli_build_reports_source_revision_hash(tmp_path: Path): + """CLI 构建结果必须从 sourceRevision 输出源 hash，不能读取不存在的顶层字段。""" + _make_game(tmp_path) + source_revision = _source_revision(tmp_path) + script = Path(__file__).resolve().parents[1] / "scripts/game_artifact_sync.py" + result = subprocess.run( + [ + sys.executable, str(script), + "--root", str(tmp_path), + "--tenant-id", TENANT_ID, + "--game-id", GAME_ID, + "--version-id", VERSION_ID, + "--package-type", "phaser", + "--runtime-manifest", "game-package.json", + "--source-provider", source_revision["provider"], + "--source-game-id", source_revision["gameId"], + "--source-revision-id", source_revision["revisionId"], + "--source-manifest-ref", source_revision["manifestRef"], + "--source-hash", source_revision["sourceHash"], + ], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + receipt = json.loads(result.stdout) + assert receipt["sourceHash"] == source_revision["sourceHash"] + + +def test_only_tier2_source_provider_is_supported(tmp_path: Path): + """Tier0/1 尚无统一 SourceProjectStore hash 口径，禁止提前声明 provider。""" + _make_game(tmp_path) + revision = _source_revision(tmp_path) + revision.update({ + "provider": "studio_source_project", + "manifestRef": "studio-source-project:night-market:m4-r1", + }) + with pytest.raises(store.ArtifactError, match="sourceRevision.provider"): + store.build_manifest( + tmp_path, TENANT_ID, GAME_ID, VERSION_ID, + package_type="phaser", runtime_manifest_path="game-package.json", + source_revision=revision, + ) + + +def test_engine_agnostic_source_provider_binds_binary_safe_hash(tmp_path: Path): + """LittleJS/Canvas 等游戏可绑定独立源归档，但不能复用 Tier2 文本 hash。""" + _make_game(tmp_path) + source_hash = source_store.source_hash_from_files([ + ("src/main.js", tmp_path / "src/main.js"), + ]) + manifest = store.build_manifest( + tmp_path, TENANT_ID, GAME_ID, VERSION_ID, + package_type="littlejs", + runtime_manifest_path="game-package.json", + source_revision={ + "provider": "game_source_archive", + "gameId": GAME_ID, + "revisionId": "shanhai-r1", + "manifestRef": f"game-source-archive:{GAME_ID}:shanhai-r1", + "sourceHash": source_hash, + }, + ) + store.verify_manifest(manifest) + assert manifest["sourceRevision"]["provider"] == "game_source_archive" + assert manifest["sourceRevision"]["sourceHash"] == source_hash + + +def test_tier2_source_provider_cannot_bind_littlejs_package(tmp_path: Path): + _make_game(tmp_path) + with pytest.raises(store.ArtifactError, match="packageType=phaser"): + store.build_manifest( + tmp_path, TENANT_ID, GAME_ID, VERSION_ID, + package_type="littlejs", runtime_manifest_path="game-package.json", + source_revision=_source_revision(tmp_path), + ) + + +def test_schema_rejects_tier2_source_with_littlejs_package(tmp_path: Path): + _make_game(tmp_path) + manifest = _build(tmp_path) + manifest["packageType"] = "littlejs" + schema_path = Path(__file__).resolve().parents[3] / "contracts/game-artifact-manifest.schema.json" + schema = json.loads(schema_path.read_text(encoding="utf-8")) + errors = list(Draft202012Validator(schema).iter_errors(manifest)) + assert any("phaser" in error.message for error in errors) + + +def test_remote_read_stops_at_expected_bytes_plus_one(): + """远端响应超限时只多读一个字节，不把任意大对象完整载入内存。""" + client = _FakeMinio() + key = "tenants/1/games/1024/versions/2048/runtime/bundle.js" + client.objects[("game-artifacts", key)] = b"0123456789" + with pytest.raises(store.ArtifactError, match="超过期望字节数"): + store._read_remote(client, "game-artifacts", key, expected_bytes=4) + assert client.last_response is not None + assert client.last_response._offset == 5 + assert client.last_response.read_sizes == [5] + + +def test_fetch_manifest_rejects_oversized_remote_payload(monkeypatch: pytest.MonkeyPatch): + """artifact manifest 也必须有固定读取上限，不能因伪造 marker 耗尽内存。""" + client = _FakeMinio() + key = "tenants/1/games/1024/versions/2048/artifact-manifest.json" + client.objects[("game-artifacts", key)] = b"x" * (8 * 1024 * 1024 + 1) + monkeypatch.setattr(store, "_minio_client", lambda _config: client) + with pytest.raises(store.ArtifactError, match="超过期望字节数"): + store.fetch_manifest( + {}, key=key, expected_manifest_hash="0" * 64, + expected_tenant_id=TENANT_ID, + expected_game_id=GAME_ID, + expected_version_id=VERSION_ID, + ) + + +def test_missing_bucket_is_not_treated_as_missing_marker(): + """桶缺失表示部署错误，不能降级成尚未上传。""" + class MissingBucketClient: + def get_object(self, _bucket: str, _key: str): + raise _NoSuchBucket("game-artifacts") + + with pytest.raises(store.ArtifactError, match="对象存储桶不存在"): + store._existing_manifest( + MissingBucketClient(), "game-artifacts", + "tenants/1/games/1024/versions/2048/artifact-manifest.json", + ) diff --git a/tier2/gen-worker/tests/test_game_source_archive.py b/tier2/gen-worker/tests/test_game_source_archive.py new file mode 100644 index 00000000..fe39da4d --- /dev/null +++ b/tier2/gen-worker/tests/test_game_source_archive.py @@ -0,0 +1,301 @@ +"""引擎无关游戏源归档的契约与对象存储离线测试。""" + +import json +import os +import subprocess +import sys +import unicodedata +from io import BytesIO +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from worker import game_artifact_store as artifact_store +from worker import game_source_archive_store as store + + +TENANT_ID = "1" +GAME_ID = "1024" +REVISION_ID = "shanhai-r1" + + +def _make_source(root: Path) -> None: + """创建包含源码、二进制源码资产、制品和证据的最小归档输入。""" + (root / "src").mkdir(parents=True) + (root / "assets").mkdir(parents=True) + (root / "dist").mkdir(parents=True) + (root / "evidence").mkdir(parents=True) + (root / "src" / "main.js").write_text("export const GAME = 1;\n", encoding="utf-8") + (root / "src" / "sprite.bin").write_bytes(bytes([0, 1, 2, 255])) + (root / "assets" / "hero.webp").write_bytes(b"runtime-asset") + (root / "dist" / "bundle.iife.js").write_bytes(b"runtime-bundle") + (root / "evidence" / "qa.md").write_text("evidence", encoding="utf-8") + (root / "src" / "__pycache__").mkdir() + (root / "src" / "__pycache__" / "ignored.pyc").write_bytes(b"cache") + + +def _build(root: Path) -> dict: + return store.build_source_archive( + root, TENANT_ID, GAME_ID, REVISION_ID, engine="custom-canvas-littlejs", + ) + + +def test_build_and_verify_engine_agnostic_source_archive(tmp_path: Path): + _make_source(tmp_path) + manifest = _build(tmp_path) + store.verify_source_archive(manifest) + + assert manifest["schemaVersion"] == "GameSourceArchive/1" + assert manifest["keyPrefix"] == f"tenants/{TENANT_ID}/games/{GAME_ID}/source-revisions/{REVISION_ID}" + assert {item["sourcePath"] for item in manifest["objects"]} == { + "src/main.js", "src/sprite.bin", + } + assert all(item["path"].startswith("source/") for item in manifest["objects"]) + assert manifest["totals"]["objects"] == 2 + assert manifest["totals"]["bytes"] == len("export const GAME = 1;\n".encode()) + 4 + assert manifest["sourceHash"] == store.source_hash_from_files( + [(item["sourcePath"], tmp_path / item["sourcePath"]) for item in manifest["objects"]] + ) + + schema_path = Path(__file__).resolve().parents[3] / "contracts/game-source-archive.schema.json" + schema = json.loads(schema_path.read_text(encoding="utf-8")) + errors = sorted(Draft202012Validator(schema).iter_errors(manifest), key=lambda item: list(item.path)) + assert not errors, [error.message for error in errors] + assert _build(tmp_path)["manifestHash"] == manifest["manifestHash"] + + +def test_source_hash_is_binary_safe_and_mutation_is_detected(tmp_path: Path): + _make_source(tmp_path) + manifest = _build(tmp_path) + (tmp_path / "src" / "sprite.bin").write_bytes(bytes([0, 1, 3, 255])) + with pytest.raises(store.ArtifactError, match="源归档 hash"): + store.build_source_archive( + tmp_path, TENANT_ID, GAME_ID, REVISION_ID, + engine="custom-canvas-littlejs", expected_source_hash=manifest["sourceHash"], + ) + + manifest = _build(tmp_path) + manifest["objects"][0]["bytes"] += 1 + with pytest.raises(store.ArtifactError, match="manifestHash"): + store.verify_source_archive(manifest) + + +def test_source_hash_sorts_after_nfc_normalization(tmp_path: Path): + nfd_name = unicodedata.normalize("NFD", "á.js") + first = tmp_path / nfd_name + second = tmp_path / "z.js" + first.write_bytes(b"a") + second.write_bytes(b"z") + files = [(nfd_name, first), ("z.js", second)] + assert store.source_hash_from_files(files) == artifact_store._source_archive_hash(files)[2] + + +def test_default_runtime_manifest_is_not_source(tmp_path: Path): + _make_source(tmp_path) + (tmp_path / "game-package.json").write_text("runtime manifest", encoding="utf-8") + manifest = _build(tmp_path) + assert "game-package.json" not in {item["sourcePath"] for item in manifest["objects"]} + + +def test_empty_source_tree_is_rejected_consistently(tmp_path: Path): + (tmp_path / "src").mkdir() + (tmp_path / "src" / "empty.js").write_bytes(b"") + with pytest.raises(store.ArtifactError, match="bytes|全空"): + _build(tmp_path) + + +def test_cli_build_reports_source_archive_identity(tmp_path: Path): + _make_source(tmp_path) + script = Path(__file__).resolve().parents[1] / "scripts/game_source_archive_sync.py" + result = subprocess.run( + [ + sys.executable, str(script), + "--root", str(tmp_path), + "--tenant-id", TENANT_ID, + "--game-id", GAME_ID, + "--revision-id", REVISION_ID, + "--engine", "custom-canvas-littlejs", + ], + check=False, + capture_output=True, + text=True, + env={**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1])}, + ) + assert result.returncode == 0, result.stderr + output = json.loads(result.stdout) + assert output["gameId"] == GAME_ID + assert output["objects"] == 2 + assert output["sourceHash"] == _build(tmp_path)["sourceHash"] + + +def test_source_archive_rejects_unsafe_paths_and_symlinks(tmp_path: Path): + _make_source(tmp_path) + outside = tmp_path.parent / "outside-source-secret.bin" + outside.write_bytes(b"secret") + (tmp_path / "src" / "linked.bin").symlink_to(outside) + with pytest.raises(store.ArtifactError, match="符号链接"): + _build(tmp_path) + + +class _Response: + """模拟 MinIO 响应，覆盖上传器使用的连接释放协议。""" + + def __init__(self, data: bytes): + self._data = data + self._offset = 0 + + def read(self, size: int = -1) -> bytes: + if size < 0: + size = len(self._data) - self._offset + chunk = self._data[self._offset:self._offset + size] + self._offset += len(chunk) + return chunk + + def close(self) -> None: + return None + + def release_conn(self) -> None: + return None + + +class _NoSuchKey(Exception): + code = "NoSuchKey" + + +class _FakeMinio: + """内存对象存储，验证源桶隔离、幂等和原子下载。""" + + def __init__(self): + self.objects: dict[tuple[str, str], bytes] = {} + self.put_calls: list[tuple[str, str]] = [] + + def put_object(self, bucket: str, key: str, stream: BytesIO, *, length: int, content_type: str): + data = stream.read() + assert len(data) == length + assert content_type + self.put_calls.append((bucket, key)) + self.objects[(bucket, key)] = data + + def get_object(self, bucket: str, key: str) -> _Response: + if (bucket, key) not in self.objects: + raise _NoSuchKey(key) + return _Response(self.objects[(bucket, key)]) + + +def test_upload_is_idempotent_and_download_is_atomic(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + _make_source(tmp_path) + manifest = _build(tmp_path) + client = _FakeMinio() + monkeypatch.setattr(store, "_minio_client", lambda _config: client) + config = {"source_bucket": "game-sources"} + + with pytest.raises(store.ArtifactError, match="外部单写"): + store.upload_source_archive(tmp_path, manifest, config) + first = store.upload_source_archive(tmp_path, manifest, config, external_single_writer=True) + second = store.upload_source_archive(tmp_path, manifest, config, external_single_writer=True) + assert first["idempotent"] is False + assert second["idempotent"] is True + assert first["status"] == second["status"] == "verified_pending" + assert first["committed"] is second["committed"] is False + assert all(bucket == "game-sources" for bucket, _ in client.put_calls) + assert ("game-sources", first["sourceManifestKey"]) in client.objects + + remote = store.fetch_source_archive( + config, key=first["sourceManifestKey"], + expected_manifest_hash=manifest["manifestHash"], + expected_tenant_id=TENANT_ID, expected_game_id=GAME_ID, + expected_revision_id=REVISION_ID, + ) + target = tmp_path / "materialized" + store.download_source_archive(remote, config, target, expected_manifest_hash=manifest["manifestHash"]) + assert (target / "src" / "main.js").read_text(encoding="utf-8") == "export const GAME = 1;\n" + assert (target / "src" / "sprite.bin").read_bytes() == bytes([0, 1, 2, 255]) + assert not (target / "assets" / "hero.webp").exists() + + +def test_source_upload_preflights_all_files_before_put(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + _make_source(tmp_path) + manifest = _build(tmp_path) + (tmp_path / "src" / "sprite.bin").write_bytes(b"drift") + client = _FakeMinio() + monkeypatch.setattr(store, "_minio_client", lambda _config: client) + with pytest.raises(store.ArtifactError, match="hash 漂移"): + store.upload_source_archive( + tmp_path, manifest, {"source_bucket": "game-sources"}, external_single_writer=True, + ) + assert client.put_calls == [] + + +def test_source_upload_requires_explicit_source_bucket(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + _make_source(tmp_path) + manifest = _build(tmp_path) + monkeypatch.setattr(store, "_minio_client", lambda _config: _FakeMinio()) + with pytest.raises(store.ArtifactError, match="source_bucket"): + store.upload_source_archive(tmp_path, manifest, {}, external_single_writer=True) + + +def test_source_bucket_cannot_alias_artifact_bucket(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + _make_source(tmp_path) + manifest = _build(tmp_path) + monkeypatch.setattr(store, "_minio_client", lambda _config: _FakeMinio()) + with pytest.raises(store.ArtifactError, match="不能与 artifact"): + store.upload_source_archive( + tmp_path, manifest, + {"source_bucket": "game-artifacts", "artifact_bucket": "game-artifacts"}, + external_single_writer=True, + ) + + +def test_source_upload_wraps_sdk_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + _make_source(tmp_path) + manifest = _build(tmp_path) + + class BrokenClient: + def get_object(self, _bucket, _key): + raise _NoSuchKey("missing marker") + + def put_object(self, *_args, **_kwargs): + raise RuntimeError("network down") + + monkeypatch.setattr(store, "_minio_client", lambda _config: BrokenClient()) + with pytest.raises(store.ArtifactError, match="对象上传失败"): + store.upload_source_archive( + tmp_path, manifest, {"source_bucket": "game-sources"}, external_single_writer=True, + ) + + +def test_source_archive_does_not_overwrite_different_marker(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + _make_source(tmp_path) + manifest = _build(tmp_path) + client = _FakeMinio() + monkeypatch.setattr(store, "_minio_client", lambda _config: client) + config = {"source_bucket": "game-sources"} + store.upload_source_archive(tmp_path, manifest, config, external_single_writer=True) + changed = dict(manifest) + changed["engine"] = "littlejs" + unsigned = dict(changed) + unsigned.pop("manifestHash") + changed["manifestHash"] = artifact_store._hash_bytes( + store._MANIFEST_HASH_DOMAIN + store._canonical_json(unsigned), + ) + puts_before = list(client.put_calls) + with pytest.raises(store.ArtifactError, match="拒绝覆盖"): + store.upload_source_archive(tmp_path, changed, config, external_single_writer=True) + assert client.put_calls == puts_before + + +def test_download_failure_leaves_no_partial_directory(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + _make_source(tmp_path) + manifest = _build(tmp_path) + client = _FakeMinio() + monkeypatch.setattr(store, "_minio_client", lambda _config: client) + config = {"source_bucket": "game-sources"} + store.upload_source_archive(tmp_path, manifest, config, external_single_writer=True) + first = next(item for item in manifest["objects"] if item["sourcePath"] == "src/main.js") + client.objects[("game-sources", first["key"])] = b"wrong" + target = tmp_path / "materialized" + with pytest.raises(store.ArtifactError, match="下载对象 hash"): + store.download_source_archive(manifest, config, target, expected_manifest_hash=manifest["manifestHash"]) + assert not target.exists() + assert not list(tmp_path.glob(".materialized.partial-*")) diff --git a/tier2/gen-worker/worker/game_artifact_storage_store.py b/tier2/gen-worker/worker/game_artifact_storage_store.py new file mode 100644 index 00000000..0fa36c69 --- /dev/null +++ b/tier2/gen-worker/worker/game_artifact_storage_store.py @@ -0,0 +1,330 @@ +"""game_artifact_storage 的数据库状态写入与条件提交。 + +对象存储上传器只得到 ``verified_pending``；本模块把已验证的 manifest 身份 +写入 game-cloud 业务库，并以 ``tenant/game/version + hash`` 条件把 pending +推进到 committed。它不执行对象上传，也不接受调用方重新拼接 bucket/key。 +""" + +from __future__ import annotations + +import re +from datetime import datetime +from typing import Callable, Mapping + +from .game_artifact_store import ( + ArtifactError, + _ARTIFACT_MANIFEST_KEY_RE, + _HASH_RE, + _canonical_json, + _validate_db_id, + verify_manifest, +) +from .game_source_archive_store import verify_source_archive + +_TABLE = "game_artifact_storage" +_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_SOURCE_MANIFEST_KEY_RE = re.compile( + r"^tenants/[A-Za-z0-9][A-Za-z0-9._-]*/games/[1-9][0-9]*/" + r"source-revisions/[A-Za-z0-9][A-Za-z0-9._-]*/source-manifest\.json$" +) +_IMMUTABLE_FIELDS = ( + "artifact_bucket", "artifact_manifest_key", "artifact_manifest_hash", "artifact_manifest_bytes", + "runtime_manifest_key", "runtime_manifest_hash", + "source_bucket", "source_provider", "source_game_id", "source_revision_id", + "source_manifest_key", "source_manifest_hash", "source_hash", "package_type", + "object_count", "total_bytes", "asset_hash", "bundle_hash", +) +_SELECT_COLUMNS = ", ".join(( + "id", "tenant_id", "game_id", "version_id", "artifact_bucket", "artifact_manifest_key", + "artifact_manifest_hash", "artifact_manifest_bytes", "source_bucket", "source_provider", + "runtime_manifest_key", "runtime_manifest_hash", + "source_game_id", "source_revision_id", "source_manifest_key", "source_manifest_hash", + "source_hash", "package_type", "object_count", "total_bytes", "asset_hash", "bundle_hash", + "status", "failure_reason", "committed_at", "creator", "create_time", "updater", "update_time", +)) + + +class StorageRecordError(ArtifactError): + """对象存储状态记录不满足数据库边界或 CAS 条件。""" + + +def _bucket(value: str, label: str) -> str: + """桶名必须由受信配置提供，禁止通过路径片段注入对象 key。""" + if not isinstance(value, str) or not value or "/" in value or "\\" in value: + raise StorageRecordError(f"{label} 非法或未配置") + return value + + +def _validate_record(record: Mapping) -> None: + """校验 writer 输入的不可变字段，避免绕过 manifest builder 写入伪身份。""" + for field in ("tenant_id", "game_id", "version_id"): + value = record.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise StorageRecordError(f"{field} 必须是正整数") + _bucket(record.get("artifact_bucket", ""), "artifact_bucket") + artifact_key = str(record.get("artifact_manifest_key", "")) + if not _ARTIFACT_MANIFEST_KEY_RE.fullmatch(artifact_key): + raise StorageRecordError("artifact_manifest_key 非法") + expected_artifact_key = ( + f"tenants/{record['tenant_id']}/games/{record['game_id']}" + f"/versions/{record['version_id']}/artifact-manifest.json" + ) + if artifact_key != expected_artifact_key: + raise StorageRecordError("artifact_manifest_key 与 tenant/game/version 不一致") + if not _HASH_RE.fullmatch(str(record.get("artifact_manifest_hash", ""))): + raise StorageRecordError("artifact_manifest_hash 非法") + if not isinstance(record.get("artifact_manifest_bytes"), int) or record["artifact_manifest_bytes"] <= 0: + raise StorageRecordError("artifact_manifest_bytes 非法") + runtime_key = str(record.get("runtime_manifest_key", "")) + if runtime_key != f"tenants/{record['tenant_id']}/games/{record['game_id']}/versions/{record['version_id']}/manifest.json": + raise StorageRecordError("runtime_manifest_key 与 tenant/game/version 不一致") + if not _HASH_RE.fullmatch(str(record.get("runtime_manifest_hash", ""))): + raise StorageRecordError("runtime_manifest_hash 非法") + provider = record.get("source_provider") + if provider not in {"tier2_source_project", "game_source_archive"}: + raise StorageRecordError("source_provider 不受支持") + if provider == "game_source_archive": + _bucket(record.get("source_bucket", ""), "source_bucket") + if record.get("source_bucket") in {record.get("artifact_bucket"), ""}: + raise StorageRecordError("source_bucket 不能与 artifact_bucket 复用") + source_game_id = str(record.get("source_game_id", "")) + if not source_game_id.isdigit() or source_game_id.startswith("0") or int(source_game_id) <= 0: + raise StorageRecordError("game_source_archive 的 source_game_id 必须是正整数") + revision_id = str(record.get("source_revision_id", "")) + if not _ID_RE.fullmatch(revision_id): + raise StorageRecordError("source_revision_id 非法") + source_key = str(record.get("source_manifest_key", "")) + if not _SOURCE_MANIFEST_KEY_RE.fullmatch(source_key): + raise StorageRecordError("source_manifest_key 非法") + expected_source_key = ( + f"tenants/{record['tenant_id']}/games/{source_game_id}" + f"/source-revisions/{revision_id}/source-manifest.json" + ) + if source_key != expected_source_key: + raise StorageRecordError("source_manifest_key 与源修订身份不一致") + if not _HASH_RE.fullmatch(str(record.get("source_manifest_hash", ""))): + raise StorageRecordError("source_manifest_hash 非法") + else: + if not _ID_RE.fullmatch(str(record.get("source_game_id", ""))): + raise StorageRecordError("source_game_id 非法") + if not _ID_RE.fullmatch(str(record.get("source_revision_id", ""))): + raise StorageRecordError("source_revision_id 非法") + if any(record.get(field, "") for field in ("source_bucket", "source_manifest_key", "source_manifest_hash")): + raise StorageRecordError("Tier2 外部源修订不能伪造 game-sources 清单") + if not _HASH_RE.fullmatch(str(record.get("source_hash", ""))): + raise StorageRecordError("source_hash 非法") + if record.get("status") != "pending": + raise StorageRecordError("新对象存储记录必须从 pending 开始") + if record.get("package_type") not in {"phaser", "littlejs", "canvas"}: + raise StorageRecordError("package_type 不受支持") + if provider == "tier2_source_project" and record.get("package_type") != "phaser": + raise StorageRecordError("tier2_source_project 只允许 package_type=phaser") + for field in ("object_count", "total_bytes"): + if not isinstance(record.get(field), int) or record[field] < 0: + raise StorageRecordError(f"{field} 非法") + for field in ("asset_hash", "bundle_hash"): + if record.get(field) and not _HASH_RE.fullmatch(str(record[field])): + raise StorageRecordError(f"{field} 非法") + + +def build_storage_record(artifact_manifest: Mapping, *, artifact_bucket: str, + source_bucket: str = "", source_archive: Mapping | None = None, + creator: str = "", updater: str = "") -> dict: + """从已验证的 artifact/source manifest 派生数据库记录，不接受手填 key/hash。""" + verify_manifest(artifact_manifest) + tenant_id = str(artifact_manifest["tenantId"]) + _validate_db_id(tenant_id, "tenantId") + game_id = str(artifact_manifest["gameId"]) + version_id = str(artifact_manifest["versionId"]) + artifact_bucket = _bucket(artifact_bucket, "artifact_bucket") + revision = artifact_manifest["sourceRevision"] + provider = revision["provider"] + source_record = { + "source_bucket": "", + "source_provider": provider, + "source_game_id": str(revision["gameId"]), + "source_revision_id": str(revision["revisionId"]), + "source_manifest_key": "", + "source_manifest_hash": "", + } + if provider == "game_source_archive": + if source_archive is None: + raise StorageRecordError("game_source_archive 必须同时提供已验证 source archive manifest") + verify_source_archive(source_archive) + if ( + source_archive["tenantId"] != tenant_id + or source_archive["gameId"] != revision["gameId"] + or source_archive["revisionId"] != revision["revisionId"] + or source_archive["sourceHash"] != revision["sourceHash"] + ): + raise StorageRecordError("artifact/source manifest 身份或 sourceHash 不一致") + source_bucket = _bucket(source_bucket, "source_bucket") + if source_bucket == artifact_bucket: + raise StorageRecordError("source_bucket 不能与 artifact_bucket 复用") + source_record.update({ + "source_bucket": source_bucket, + "source_manifest_key": f"{source_archive['keyPrefix']}/source-manifest.json", + "source_manifest_hash": source_archive["manifestHash"], + }) + elif source_archive is not None or source_bucket: + raise StorageRecordError("Tier2 sourceRevision 不应绑定 game-sources 清单") + record = { + "tenant_id": int(tenant_id), + "game_id": int(game_id), + "version_id": int(version_id), + "artifact_bucket": artifact_bucket, + "artifact_manifest_key": f"{artifact_manifest['keyPrefix']}/artifact-manifest.json", + "artifact_manifest_hash": artifact_manifest["manifestHash"], + "artifact_manifest_bytes": len(_canonical_json(artifact_manifest)), + "runtime_manifest_key": artifact_manifest["runtimeManifest"]["key"], + "runtime_manifest_hash": artifact_manifest["runtimeManifest"]["sha256"], + **source_record, + "source_hash": revision["sourceHash"], + "package_type": artifact_manifest["packageType"], + "object_count": artifact_manifest["totals"]["objects"], + "total_bytes": artifact_manifest["totals"]["bytes"], + "asset_hash": artifact_manifest["assetHash"], + "bundle_hash": artifact_manifest["bundleHash"] or "", + "status": "pending", + "failure_reason": "", + "creator": str(creator), + "updater": str(updater), + } + _validate_record(record) + return record + + +def _is_duplicate_error(exc: Exception) -> bool: + """识别 MySQL duplicate-key 错误，不吞其它数据库故障。""" + if getattr(exc, "errno", None) == 1062 or getattr(exc, "args", (None,))[0:1] == (1062,): + return True + return "duplicate" in str(exc).lower() or "1062" in str(exc) + + +class MySqlArtifactStorageStore: + """使用调用方连接工厂写入 game_artifact_storage。""" + + def __init__(self, connection_factory: Callable[[], object]): + self._connection_factory = connection_factory + + def _select_identity(self, connection, record: Mapping) -> dict | None: + sql = ( + f"SELECT {_SELECT_COLUMNS} FROM {_TABLE} " + "WHERE tenant_id=%s AND game_id=%s AND version_id=%s LIMIT 1" + ) + with connection.cursor() as cursor: + cursor.execute(sql, (record["tenant_id"], record["game_id"], record["version_id"])) + row = cursor.fetchone() + return dict(row) if row else None + + @staticmethod + def _same_immutable_fields(existing: Mapping, record: Mapping) -> bool: + return all(existing.get(field) == record.get(field) for field in _IMMUTABLE_FIELDS) + + @staticmethod + def _receipt(row: Mapping, *, idempotent: bool) -> dict: + return { + "id": row.get("id"), + "tenantId": str(row["tenant_id"]), + "gameId": str(row["game_id"]), + "versionId": str(row["version_id"]), + "artifactManifestHash": row["artifact_manifest_hash"], + "sourceManifestHash": row.get("source_manifest_hash", ""), + "status": row["status"], + "idempotent": idempotent, + "committed": row["status"] == "committed", + } + + def create_pending(self, record: Mapping) -> dict: + """条件创建 pending；唯一键冲突时仅允许完全相同记录幂等命中。""" + _validate_record(record) + connection = self._connection_factory() + sql = ( + f"INSERT INTO {_TABLE} (tenant_id, game_id, version_id, artifact_bucket, artifact_manifest_key, " + "artifact_manifest_hash, artifact_manifest_bytes, runtime_manifest_key, runtime_manifest_hash, source_bucket, source_provider, source_game_id, " + "source_revision_id, source_manifest_key, source_manifest_hash, source_hash, package_type, object_count, " + "total_bytes, asset_hash, bundle_hash, status, failure_reason, creator, updater) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + ) + params = tuple(record[field] for field in ( + "tenant_id", "game_id", "version_id", "artifact_bucket", "artifact_manifest_key", + "artifact_manifest_hash", "artifact_manifest_bytes", "runtime_manifest_key", "runtime_manifest_hash", + "source_bucket", "source_provider", + "source_game_id", "source_revision_id", "source_manifest_key", "source_manifest_hash", + "source_hash", "package_type", "object_count", "total_bytes", "asset_hash", "bundle_hash", + "status", "failure_reason", "creator", "updater", + )) + try: + with connection.cursor() as cursor: + cursor.execute(sql, params) + connection.commit() + existing = self._select_identity(connection, record) + if existing is None: + raise StorageRecordError("pending 创建成功但读取不到记录") + return self._receipt(existing, idempotent=False) + except Exception as exc: # noqa: BLE001 - 只把 duplicate 转为幂等路径 + connection.rollback() + if not _is_duplicate_error(exc): + raise StorageRecordError(f"创建 pending 记录失败:{type(exc).__name__}:{exc}") from exc + existing = self._select_identity(connection, record) + if existing is None: + raise StorageRecordError("唯一键冲突后找不到对象存储记录") from exc + if not self._same_immutable_fields(existing, record): + raise StorageRecordError("同一租户游戏版本已有不同对象清单，不可覆盖") from exc + return self._receipt(existing, idempotent=True) + finally: + connection.close() + + def commit_pending(self, record: Mapping, *, committed_at: datetime) -> dict: + """以所有不可变 hash 为条件把 pending CAS 到 committed。""" + _validate_record(record) + if not isinstance(committed_at, datetime): + raise StorageRecordError("committed_at 必须由调用方显式提供") + connection = self._connection_factory() + sql = ( + f"UPDATE {_TABLE} SET status='committed', committed_at=%s, updater=%s " + "WHERE tenant_id=%s AND game_id=%s AND version_id=%s AND status='pending' " + "AND artifact_manifest_hash=%s AND source_hash=%s AND source_manifest_hash=%s" + ) + params = ( + committed_at, record.get("updater", ""), record["tenant_id"], record["game_id"], record["version_id"], + record["artifact_manifest_hash"], record["source_hash"], record.get("source_manifest_hash", ""), + ) + try: + with connection.cursor() as cursor: + cursor.execute(sql, params) + affected = int(getattr(cursor, "rowcount", 0)) + if affected: + connection.commit() + row = self._select_identity(connection, record) + if row is None: + raise StorageRecordError("CAS committed 成功但读取不到记录") + return self._receipt(row, idempotent=False) + existing = self._select_identity(connection, record) + connection.rollback() + if existing and existing.get("status") == "committed" and self._same_immutable_fields(existing, record): + return self._receipt(existing, idempotent=True) + raise StorageRecordError("对象存储 CAS 未命中：记录不存在、hash 不一致或已被提交") + except StorageRecordError: + raise + except Exception as exc: # noqa: BLE001 - DB 错误必须显式失败 + connection.rollback() + raise StorageRecordError(f"提交对象存储记录失败:{type(exc).__name__}:{exc}") from exc + finally: + connection.close() + + def get_committed(self, *, tenant_id: int, game_id: int, version_id: int) -> dict | None: + """只读取 committed 记录，pending/abandoned 对生产消费不可见。""" + for name, value in (("tenant_id", tenant_id), ("game_id", game_id), ("version_id", version_id)): + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise StorageRecordError(f"{name} 必须是正整数") + connection = self._connection_factory() + try: + row = self._select_identity(connection, { + "tenant_id": tenant_id, "game_id": game_id, "version_id": version_id, + }) + if not row or row.get("status") != "committed": + return None + return self._receipt(row, idempotent=True) + finally: + connection.close() diff --git a/tier2/gen-worker/worker/game_artifact_store.py b/tier2/gen-worker/worker/game_artifact_store.py new file mode 100644 index 00000000..2573a02a --- /dev/null +++ b/tier2/gen-worker/worker/game_artifact_store.py @@ -0,0 +1,748 @@ +"""游戏包对象存储工具。 + +本模块只处理平台侧的分类、哈希、manifest 和 MinIO/S3 读写；具体游戏目录只是一次性上传输入， +不是生产消费来源。实现使用标准库完成本地快照，只有真正执行对象存储读写时才惰性导入 minio， +让契约测试和离线 manifest 构建不依赖网络或第三方 SDK。 +""" + +from __future__ import annotations + +import hashlib +import json +import mimetypes +import os +import posixpath +import re +import shutil +import tempfile +import unicodedata +from io import BytesIO +from pathlib import Path +from typing import Iterable, Mapping + +SCHEMA_VERSION = "GameArtifactManifest/1" +_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_DB_ID_RE = re.compile(r"^[1-9][0-9]*$") +_HASH_RE = re.compile(r"^[a-f0-9]{64}$") +_ARTIFACT_MANIFEST_KEY_RE = re.compile( + r"^tenants/[A-Za-z0-9][A-Za-z0-9._-]*/games/[1-9][0-9]*/" + r"versions/[1-9][0-9]*/artifact-manifest\.json$" +) +_CATEGORY_ORDER = ("asset", "runtime", "evidence") +_CATEGORY_DIR = {"asset": "assets", "runtime": "runtime", "evidence": "evidence"} +_CATEGORY_STORE = {"asset": "artifact", "runtime": "artifact", "evidence": "evidence"} +_TRANSIENT_PARTS = frozenset({ + ".git", ".idea", "node_modules", "__pycache__", ".pytest_cache", "target", + "coverage", ".DS_Store", "_store", "_cheap-sessions", "_tier2-gen", +}) +_SECRET_NAMES = frozenset({ + ".env", ".env.local", ".env.production", "credentials.json", "service-account.json", +}) +_MANIFEST_HASH_DOMAIN = b"GameArtifactManifest/1\0" +_TREE_HASH_DOMAIN = b"GameArtifactTreeHash/1\0" +_SOURCE_TREE_HASH_DOMAIN = b"GameSourceArchiveTreeHash/1\0" +_MAX_OBJECTS = 10_000 +_MAX_PATH_BYTES = 1_024 +_MAX_OBJECT_BYTES = 100 * 1024 * 1024 +_MAX_RUNTIME_ASSET_BYTES = 10 * 1024 * 1024 +_MAX_MANIFEST_BYTES = 8 * 1024 * 1024 +_REMOTE_READ_CHUNK_BYTES = 1024 * 1024 + + +class ArtifactError(ValueError): + """对象包不满足安全或一致性约束时抛出的稳定异常。""" + + +def _validate_id(value: str, label: str) -> str: + """校验 gameId/versionId，避免对象 key 被注入路径片段。""" + if not isinstance(value, str) or not _ID_RE.fullmatch(value): + raise ArtifactError(f"{label} 非法:{value!r}") + return value + + +def _validate_db_id(value: str, label: str) -> str: + """生产 game/version 身份必须来自数据库正整数主键，禁止 Tier2 slug 冒充。""" + if not isinstance(value, str) or not _DB_ID_RE.fullmatch(value): + raise ArtifactError(f"{label} 必须是数据库正整数 ID:{value!r}") + return value + + +def _safe_rel(path: str) -> str: + """把本地相对路径规范化为 POSIX，并拒绝绝对路径、父级跳转和空路径。""" + normalized = unicodedata.normalize("NFC", posixpath.normpath(path.replace(os.sep, "/"))) + if not normalized or normalized in (".", "..") or normalized.startswith("../") or normalized.startswith("/"): + raise ArtifactError(f"对象相对路径非法:{path!r}") + if any(part in ("", ".", "..") for part in normalized.split("/")): + raise ArtifactError(f"对象相对路径含危险片段:{path!r}") + if len(normalized.encode("utf-8")) > _MAX_PATH_BYTES: + raise ArtifactError(f"对象相对路径超过 {_MAX_PATH_BYTES} 字节:{path!r}") + return normalized + + +def _canonical_json(value: Mapping) -> bytes: + """使用无空格、排序键的 JSON 字节，保证跨语言 manifest hash 可复算。""" + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def _hash_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _file_digest(path: Path) -> tuple[int, str]: + """流式计算本地文件大小与 SHA-256，避免把大文件整体读入内存。""" + digest = hashlib.sha256() + size = 0 + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + size += len(chunk) + if size > _MAX_OBJECT_BYTES: + raise ArtifactError(f"单文件超过 {_MAX_OBJECT_BYTES} 字节:{path.name}") + digest.update(chunk) + return size, digest.hexdigest() + + +def _source_project_hash(files: Iterable[tuple[str, Path]]) -> tuple[int, int, str]: + """按 SourceProjectStore 的 path\0content\0 口径复算源工程 contentHash。""" + rows = sorted(files, key=lambda item: item[0]) + digest = hashlib.sha256() + total_bytes = 0 + for rel, path in rows: + data = path.read_bytes() + if len(data) > _MAX_OBJECT_BYTES: + raise ArtifactError(f"源文件超过 {_MAX_OBJECT_BYTES} 字节:{rel}") + try: + content = data.decode("utf-8") + except UnicodeDecodeError as exc: + raise ArtifactError(f"源工程文件必须是 UTF-8 文本:{rel}") from exc + digest.update((rel + "\0").encode("utf-8")) + digest.update((content + "\0").encode("utf-8")) + total_bytes += len(data) + if not rows: + raise ArtifactError("游戏包缺少源工程文件") + return len(rows), total_bytes, digest.hexdigest() + + +def _source_archive_rows(files: Iterable[tuple[str, Path]]) -> list[dict]: + """按二进制内容构造引擎无关源归档的稳定文件行。""" + rows: list[dict] = [] + seen: set[str] = set() + normalized_files = [(_safe_rel(rel), path) for rel, path in files] + for normalized, path in sorted(normalized_files, key=lambda item: item[0]): + if normalized in seen: + raise ArtifactError(f"源归档存在重复路径:{normalized}") + seen.add(normalized) + size, sha256 = _file_digest(path) + rows.append({"path": normalized, "bytes": size, "sha256": sha256}) + if not rows: + raise ArtifactError("游戏包缺少源工程文件") + if len(rows) > _MAX_OBJECTS: + raise ArtifactError(f"源归档对象数量超过 {_MAX_OBJECTS}") + return rows + + +def _source_archive_hash(files: Iterable[tuple[str, Path]]) -> tuple[int, int, str]: + """计算 GameSourceArchive/1 的二进制安全源工程 hash。""" + rows = _source_archive_rows(files) + total_bytes = sum(item["bytes"] for item in rows) + digest = _hash_bytes(_SOURCE_TREE_HASH_DOMAIN + _canonical_json(rows)) + return len(rows), total_bytes, digest + + +def _contract_schema(name: str) -> dict: + """从仓内契约 SoT 读取 JSON Schema。""" + path = Path(__file__).resolve().parents[3] / "contracts" / name + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ArtifactError(f"读取契约失败:{name}:{exc}") from exc + + +def _validate_contract(instance: Mapping, schema_name: str) -> None: + """执行 Draft 2020-12 契约校验，包含 additionalProperties 边界。""" + try: + from jsonschema import Draft202012Validator, FormatChecker + except Exception as exc: # pragma: no cover - 部署缺依赖时响亮失败 + raise ArtifactError(f"缺少 jsonschema 依赖:{exc}") from exc + errors = sorted( + Draft202012Validator( + _contract_schema(schema_name), format_checker=FormatChecker(), + ).iter_errors(instance), + key=lambda item: tuple(str(part) for part in item.absolute_path), + ) + if errors: + first = errors[0] + location = "/".join(str(part) for part in first.absolute_path) or "#" + raise ArtifactError(f"{schema_name} 校验失败:{location}:{first.message}") + + +def _validate_game_package(data: bytes, game_id: str, version_id: str) -> dict: + """验证运行时 manifest 是完整 GamePackage/1，且身份与对象版本一致。""" + try: + game_package = json.loads(data.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ArtifactError("GamePackage manifest 不是合法 UTF-8 JSON") from exc + if not isinstance(game_package, Mapping): + raise ArtifactError("GamePackage manifest 顶层必须是对象") + _validate_contract(game_package, "game-package.schema.json") + if str(game_package.get("gameId")) != game_id or str(game_package.get("versionId")) != version_id: + raise ArtifactError("GamePackage gameId/versionId 与对象版本不匹配") + return dict(game_package) + + +def _hash_objects(objects: Iterable[Mapping]) -> str: + """对对象清单的稳定摘要，不读取对象内容之外的机器路径和时间。""" + rows = [ + {"category": item["category"], "path": item["path"], "bytes": item["bytes"], "sha256": item["sha256"]} + for item in objects + ] + rows.sort(key=lambda item: (item["category"], item["path"])) + return _hash_bytes(_TREE_HASH_DOMAIN + _canonical_json(rows)) + + +def _bundle_object(objects: Iterable[Mapping]) -> Mapping | None: + """按固定优先级挑选 runtime bundle，避免排序变化导致身份漂移。""" + runtime_js = [item for item in objects if item.get("category") == "runtime" and str(item.get("path", "")).endswith(".js")] + if not runtime_js: + return None + runtime_js.sort(key=lambda item: (0 if str(item["path"]).endswith("bundle.iife.js") else 1, + 0 if str(item["path"]).endswith("entry-bundle.js") else 1, + str(item["path"]))) + return runtime_js[0] + + +def _classify(rel: str) -> tuple[str, str]: + """按路径把游戏输入映射为 source/asset/runtime/evidence 和包内路径。""" + path = _safe_rel(rel) + parts = path.split("/") + name = parts[-1] + if parts[0] == "evidence": + return "evidence", path + if parts[0] == "assets": + return "asset", path + if parts[0] == "dist": + # dist 中只有可执行 bundle/meta 进入 runtime，其余报告留作 evidence。 + if name.endswith(".js") or name.endswith(".js.meta.json") or name.endswith(".html"): + return "runtime", f"runtime/{path[5:]}" + return "evidence", f"evidence/{path}" + if name in {"bundle.iife.js", "entry-bundle.js"} or name.endswith(".bundle.js"): + return "runtime", f"runtime/{path}" + if parts[0] in {"screenshots", "reports"} or name.endswith((".png", ".jpg", ".jpeg", ".webp", ".mp3", ".wav")): + # 游戏根下孤立的二进制不是源码；按证据保存，资产应放在 assets/ 下。 + return "evidence", f"evidence/{path}" + return "source", f"source/{path}" + + +def _is_transient(path: Path, rel: str) -> bool: + """过滤缓存、凭据和临时批跑；过滤规则宁可少收，不把敏感文件上传。""" + if any(part in _TRANSIENT_PARTS for part in path.parts): + return True + if path.name in _SECRET_NAMES or path.name.startswith(".env.") or path.name.endswith((".pem", ".key", ".p12")): + return True + if "/raw/" in f"/{rel}/" or rel.startswith("runs/raw/"): + return True + return False + + +def _iter_files(root: Path) -> Iterable[tuple[str, Path]]: + """以排序顺序枚举真实文件，拒绝符号链接和越界路径。""" + root = root.resolve() + if not root.is_dir(): + raise ArtifactError(f"游戏根目录不存在或不是目录:{root}") + for item in sorted(root.rglob("*"), key=lambda p: p.as_posix()): + rel = item.relative_to(root).as_posix() + if _is_transient(item, rel): + continue + if item.is_symlink(): + raise ArtifactError(f"游戏包禁止符号链接:{rel}") + if item.is_file(): + yield _safe_rel(rel), item + + +def build_manifest(root: Path, tenant_id: str, game_id: str, version_id: str, *, package_type: str, + runtime_manifest_path: str, source_revision: Mapping) -> dict: + """扫描一款游戏目录，生成可上传的对象清单和分类 hash。""" + _validate_id(tenant_id, "tenantId") + _validate_db_id(game_id, "gameId") + _validate_db_id(version_id, "versionId") + if package_type not in {"phaser", "littlejs", "canvas"}: + raise ArtifactError(f"packageType 非法:{package_type}") + key_prefix = f"tenants/{tenant_id}/games/{game_id}/versions/{version_id}" + objects: list[dict] = [] + root = Path(root).resolve() + runtime_manifest_rel = _safe_rel(runtime_manifest_path) + runtime_manifest = None + source_files: list[tuple[str, Path]] = [] + for rel, file_path in _iter_files(root): + if rel == runtime_manifest_rel: + if file_path.stat().st_size > _MAX_OBJECT_BYTES: + raise ArtifactError(f"GamePackage manifest 超过 {_MAX_OBJECT_BYTES} 字节") + data = file_path.read_bytes() + _validate_game_package(data, game_id, version_id) + runtime_manifest = { + "store": "artifact", + "key": f"{key_prefix}/manifest.json", + "sourcePath": rel, + "bytes": len(data), + "sha256": _hash_bytes(data), + "mime": "application/json", + } + continue + category, package_path = _classify(rel) + if category == "source": + source_files.append((rel, file_path)) + continue + size, sha256 = _file_digest(file_path) + mime = mimetypes.guess_type(file_path.name)[0] or "application/octet-stream" + key = f"{key_prefix}/{package_path}" + objects.append({ + "category": category, + "store": _CATEGORY_STORE[category], + "path": package_path, + "sourcePath": rel, + "key": key, + "bytes": size, + "sha256": sha256, + "mime": mime, + }) + if runtime_manifest is None: + raise ArtifactError(f"指定的 GamePackage manifest 不存在:{runtime_manifest_rel}") + provider = str(source_revision.get("provider", "")) + source_game_id = _validate_id(str(source_revision.get("gameId", "")), "sourceRevision.gameId") + revision_id = _validate_id(str(source_revision.get("revisionId", "")), "sourceRevision.revisionId") + manifest_ref = str(source_revision.get("manifestRef", "")) + if provider not in {"tier2_source_project", "game_source_archive"}: + raise ArtifactError(f"sourceRevision.provider 非法:{provider}") + if provider == "tier2_source_project" and package_type != "phaser": + raise ArtifactError("tier2_source_project 只允许 packageType=phaser") + if provider == "game_source_archive": + _validate_db_id(source_game_id, "sourceRevision.gameId") + source_objects, source_bytes, source_hash = _source_archive_hash(source_files) + expected_manifest_ref = f"game-source-archive:{source_game_id}:{revision_id}" + else: + source_objects, source_bytes, source_hash = _source_project_hash(source_files) + expected_manifest_ref = f"tier2-source-project:{source_game_id}:{revision_id}" + if manifest_ref != expected_manifest_ref: + raise ArtifactError("sourceRevision.manifestRef 与 provider/gameId/revisionId 不匹配") + if str(source_revision.get("sourceHash", "")) != source_hash: + raise ArtifactError("本地源工程 hash 与 sourceRevision.sourceHash 不匹配") + source_revision_record = { + "provider": provider, + "gameId": source_game_id, + "revisionId": revision_id, + "manifestRef": manifest_ref, + "sourceHash": source_hash, + "objects": source_objects, + "bytes": source_bytes, + } + objects.sort(key=lambda item: (item["category"], item["path"])) + if len(objects) > _MAX_OBJECTS: + raise ArtifactError(f"对象数量超过 {_MAX_OBJECTS}") + by_category = { + category: [item for item in objects if item["category"] == category] + for category in _CATEGORY_ORDER + } + bundle = _bundle_object(by_category["runtime"]) + runtime_asset_bytes = sum(item["bytes"] for item in by_category["runtime"] + by_category["asset"]) + if runtime_asset_bytes > _MAX_RUNTIME_ASSET_BYTES: + raise ArtifactError(f"运行包与素材合计超过 {_MAX_RUNTIME_ASSET_BYTES} 字节") + manifest = { + "schemaVersion": SCHEMA_VERSION, + "tenantId": tenant_id, + "gameId": game_id, + "versionId": version_id, + "packageType": package_type, + "keyPrefix": key_prefix, + "sourceRevision": source_revision_record, + "runtimeManifest": runtime_manifest, + "objects": objects, + "totals": { + "objects": len(objects), + "bytes": sum(item["bytes"] for item in objects), + "assetBytes": sum(item["bytes"] for item in by_category["asset"]), + "runtimeBytes": sum(item["bytes"] for item in by_category["runtime"]), + "evidenceBytes": sum(item["bytes"] for item in by_category["evidence"]), + }, + "assetHash": _hash_objects(by_category["asset"]), + "bundleHash": bundle["sha256"] if bundle else None, + } + manifest["manifestHash"] = _hash_bytes(_MANIFEST_HASH_DOMAIN + _canonical_json(manifest)) + if len(_canonical_json(manifest)) > _MAX_MANIFEST_BYTES: + raise ArtifactError(f"artifact manifest 超过 {_MAX_MANIFEST_BYTES} 字节") + return manifest + + +def verify_manifest(manifest: Mapping) -> None: + """验证 manifest 自身 hash、对象路径和 totals，失败即拒绝消费。""" + _validate_contract(manifest, "game-artifact-manifest.schema.json") + if len(_canonical_json(manifest)) > _MAX_MANIFEST_BYTES: + raise ArtifactError(f"artifact manifest 超过 {_MAX_MANIFEST_BYTES} 字节") + if manifest.get("schemaVersion") != SCHEMA_VERSION: + raise ArtifactError("manifest schemaVersion 不受支持") + _validate_id(str(manifest.get("tenantId", "")), "tenantId") + _validate_db_id(str(manifest.get("gameId", "")), "gameId") + _validate_db_id(str(manifest.get("versionId", "")), "versionId") + declared = manifest.get("manifestHash") + if not isinstance(declared, str) or not _HASH_RE.fullmatch(declared): + raise ArtifactError("manifestHash 非法") + unsigned = dict(manifest) + unsigned.pop("manifestHash", None) + if _hash_bytes(_MANIFEST_HASH_DOMAIN + _canonical_json(unsigned)) != declared: + raise ArtifactError("manifestHash 不匹配") + if manifest.get("packageType") not in {"phaser", "littlejs", "canvas"}: + raise ArtifactError("manifest packageType 不受支持") + key_prefix = str(manifest.get("keyPrefix", "")) + if key_prefix != f"tenants/{manifest['tenantId']}/games/{manifest['gameId']}/versions/{manifest['versionId']}": + raise ArtifactError("manifest keyPrefix 与 tenant/game/version 不匹配") + source_revision = manifest["sourceRevision"] + provider = source_revision["provider"] + if provider not in {"tier2_source_project", "game_source_archive"}: + raise ArtifactError("sourceRevision.provider 不受支持") + if provider == "tier2_source_project" and manifest.get("packageType") != "phaser": + raise ArtifactError("tier2_source_project 只允许 packageType=phaser") + if provider == "game_source_archive": + _validate_db_id(str(source_revision["gameId"]), "sourceRevision.gameId") + expected_source_ref = f"game-source-archive:{source_revision['gameId']}:{source_revision['revisionId']}" + else: + _validate_id(str(source_revision["gameId"]), "sourceRevision.gameId") + expected_source_ref = f"tier2-source-project:{source_revision['gameId']}:{source_revision['revisionId']}" + if source_revision["manifestRef"] != expected_source_ref: + raise ArtifactError("sourceRevision.manifestRef 与身份字段不匹配") + objects = manifest.get("objects") or [] + totals = manifest.get("totals") or {} + if len(objects) != totals.get("objects"): + raise ArtifactError("manifest totals.objects 不匹配") + if len(objects) > _MAX_OBJECTS: + raise ArtifactError(f"对象数量超过 {_MAX_OBJECTS}") + seen: set[str] = set() + seen_sources: set[str] = set() + sums = {category: 0 for category in _CATEGORY_ORDER} + grouped: dict[str, list[Mapping]] = {category: [] for category in _CATEGORY_ORDER} + for item in objects: + raw_path = str(item.get("path", "")) + raw_source_path = str(item.get("sourcePath", "")) + path = _safe_rel(raw_path) + source_path = _safe_rel(raw_source_path) + if raw_path != path or raw_source_path != source_path: + raise ArtifactError(f"manifest 路径必须使用 NFC 规范形式:{raw_source_path}") + key = str(item.get("key", "")) + if path in seen: + raise ArtifactError(f"manifest 存在重复路径:{path}") + if source_path in seen_sources: + raise ArtifactError(f"manifest 存在重复源路径:{source_path}") + seen.add(path) + seen_sources.add(source_path) + category = str(item.get("category", "")) + if category not in _CATEGORY_DIR: + raise ArtifactError(f"对象 category 非法:{path}") + expected_path_prefix = f"{_CATEGORY_DIR[category]}/" + if not path.startswith(expected_path_prefix): + raise ArtifactError(f"对象 path 与 category 不匹配:{path}") + if item.get("store") != _CATEGORY_STORE[category]: + raise ArtifactError(f"对象 store 与 category 不匹配:{path}") + expected_key = f"{key_prefix}/{path}" + if key != expected_key: + raise ArtifactError(f"对象 key 与 path 不匹配:{key}") + if not isinstance(item.get("bytes"), int) or item["bytes"] < 0: + raise ArtifactError(f"对象 bytes 非法:{path}") + if item["bytes"] > _MAX_OBJECT_BYTES: + raise ArtifactError(f"对象 bytes 超限:{path}") + if not _HASH_RE.fullmatch(str(item.get("sha256", ""))): + raise ArtifactError(f"对象 hash 非法:{path}") + if not isinstance(item.get("mime"), str) or not item["mime"]: + raise ArtifactError(f"对象 mime 非法:{path}") + sums[category] += item["bytes"] + grouped[category].append(item) + runtime_manifest = manifest.get("runtimeManifest") + if not isinstance(runtime_manifest, Mapping): + raise ArtifactError("runtimeManifest 必须是对象") + raw_runtime_source_path = str(runtime_manifest.get("sourcePath", "")) + source_path = _safe_rel(raw_runtime_source_path) + if raw_runtime_source_path != source_path: + raise ArtifactError("runtimeManifest sourcePath 必须使用 NFC 规范形式") + if str(runtime_manifest.get("key")) != f"{key_prefix}/manifest.json": + raise ArtifactError("runtimeManifest key 不匹配") + if runtime_manifest.get("store") != "artifact": + raise ArtifactError("runtimeManifest store 必须为 artifact") + if source_path in seen_sources: + raise ArtifactError("runtimeManifest 与对象 sourcePath 重复") + if not isinstance(runtime_manifest.get("bytes"), int) or runtime_manifest["bytes"] < 0: + raise ArtifactError("runtimeManifest bytes 非法") + if runtime_manifest["bytes"] > _MAX_OBJECT_BYTES: + raise ArtifactError("runtimeManifest bytes 超限") + if not _HASH_RE.fullmatch(str(runtime_manifest.get("sha256", ""))): + raise ArtifactError("runtimeManifest hash 非法") + if runtime_manifest.get("mime") != "application/json": + raise ArtifactError("runtimeManifest mime 必须为 application/json") + expected_totals = { + "bytes": sum(sums.values()), + "assetBytes": sums["asset"], + "runtimeBytes": sums["runtime"], + "evidenceBytes": sums["evidence"], + } + for name, expected in expected_totals.items(): + if totals.get(name) != expected: + raise ArtifactError(f"manifest totals.{name} 不匹配") + if sums["runtime"] + sums["asset"] > _MAX_RUNTIME_ASSET_BYTES: + raise ArtifactError("运行包与素材总字节超限") + if manifest.get("assetHash") != _hash_objects(grouped["asset"]): + raise ArtifactError("manifest assetHash 不匹配") + bundle = _bundle_object(grouped["runtime"]) + if manifest.get("bundleHash") != (bundle["sha256"] if bundle else None): + raise ArtifactError("manifest bundleHash 不匹配") + + +def _minio_client(config: Mapping): + """惰性创建 MinIO 客户端；不在 import 期连接网络。""" + try: + from minio import Minio + except Exception as exc: # pragma: no cover - 环境缺 SDK 时由调用方看到稳定错误 + raise ArtifactError(f"缺少 minio Python SDK:{exc}") from exc + endpoint = str(config.get("endpoint", "")).replace("http://", "").replace("https://", "") + if not endpoint or not config.get("access_key") or not config.get("secret_key"): + raise ArtifactError("缺少专用 game artifact 对象存储端点或凭据") + return Minio(endpoint, access_key=config.get("access_key"), secret_key=config.get("secret_key"), + secure=bool(config.get("secure", False))) + + +def _read_remote(client, bucket: str, key: str, *, expected_bytes: int) -> bytes: + """按调用方字节预算流式读取；发现第 expected+1 字节就立即拒绝。""" + if not isinstance(expected_bytes, int) or expected_bytes < 0: + raise ArtifactError(f"远端对象期望字节数非法:{key}") + try: + response = client.get_object(bucket, key) + except Exception as exc: # noqa: BLE001 - 兼容 MinIO S3Error 和测试 fake + if getattr(exc, "code", None) == "NoSuchBucket": + raise ArtifactError(f"对象存储桶不存在:{bucket}") from exc + raise + try: + chunks: list[bytes] = [] + total = 0 + try: + while True: + remaining = expected_bytes - total + chunk = response.read(min(_REMOTE_READ_CHUNK_BYTES, remaining + 1)) + if not chunk: + break + total += len(chunk) + if total > expected_bytes: + raise ArtifactError(f"远端对象超过期望字节数:{key}:{expected_bytes}") + chunks.append(chunk) + return b"".join(chunks) + except ArtifactError: + raise + except Exception as exc: # noqa: BLE001 - SDK/网络读取统一失败语义 + raise ArtifactError(f"读取远端对象失败:{bucket}/{key}:{type(exc).__name__}:{exc}") from exc + finally: + response.close() + response.release_conn() + + +def _bucket(config: Mapping, store: str) -> str: + """把稳定 store 身份映射到环境桶名；manifest 不保存凭据或临时 URL。""" + if store == "artifact": + return str(config.get("artifact_bucket", config.get("bucket", "game-artifacts"))) + if store == "evidence": + return str(config.get("evidence_bucket", "game-evidence")) + if store == "source": + source_bucket = str(config.get("source_bucket", "")) + if not source_bucket: + raise ArtifactError("缺少显式 source_bucket；game-sources 尚未完成权限配置") + if source_bucket in { + str(config.get("artifact_bucket", "")), + str(config.get("evidence_bucket", "")), + }: + raise ArtifactError("source_bucket 不能与 artifact/evidence 桶复用") + return source_bucket + raise ArtifactError(f"未知对象 store:{store}") + + +def _existing_manifest(client, bucket: str, key: str) -> dict | None: + """读取已存在的版本索引；仅把 S3 的对象不存在视为未上传。""" + try: + payload = _read_remote(client, bucket, key, expected_bytes=_MAX_MANIFEST_BYTES) + except ArtifactError: + raise + except Exception as exc: # noqa: BLE001 - 兼容 MinIO S3Error 和测试 fake + if getattr(exc, "code", None) in {"NoSuchKey", "NoSuchObject"}: + return None + raise ArtifactError(f"探测远端 artifact-manifest 失败:{type(exc).__name__}:{exc}") from exc + try: + existing = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ArtifactError("远端 artifact-manifest 已存在但不是合法 JSON") from exc + verify_manifest(existing) + return existing + + +def _verify_remote_object(client, bucket: str, key: str, expected_hash: str, expected_bytes: int) -> None: + """用 GET 字节校验远端对象；不把 HEAD/ETag 当作内容完整性证据。""" + try: + data = _read_remote(client, bucket, key, expected_bytes=expected_bytes) + except ArtifactError: + raise + except Exception as exc: # noqa: BLE001 - 缺对象/SDK 错误统一成稳定失败 + raise ArtifactError(f"读取远端对象失败:{bucket}/{key}:{type(exc).__name__}:{exc}") from exc + if len(data) != expected_bytes or _hash_bytes(data) != expected_hash: + raise ArtifactError(f"上传后对象校验失败:{key}") + + +def _put_remote(client, bucket: str, key: str, stream, *, length: int, content_type: str) -> None: + """统一包装 S3 Put 异常，CLI 只暴露稳定 ArtifactError。""" + try: + client.put_object(bucket, key, stream, length=length, content_type=content_type) + except ArtifactError: + raise + except Exception as exc: # noqa: BLE001 - 兼容 MinIO S3Error/网络异常 + raise ArtifactError(f"对象上传失败:{bucket}/{key}:{type(exc).__name__}:{exc}") from exc + + +def upload_manifest(root: Path, manifest: Mapping, config: Mapping, *, + external_single_writer: bool = False) -> dict: + """在外部单写锁内上传并校验对象；波次 1 只返回 verified_pending。""" + verify_manifest(manifest) + if external_single_writer is not True: + raise ArtifactError("波次 1 上传必须由调用方持有外部单写锁") + client = _minio_client(config) + buckets = {store: _bucket(config, store) for store in ("artifact", "evidence")} + root = Path(root).resolve() + manifest_key = f"{manifest['keyPrefix']}/artifact-manifest.json" + existing = _existing_manifest(client, buckets["artifact"], manifest_key) + if existing is not None: + if existing["manifestHash"] == manifest["manifestHash"]: + for item in manifest["objects"]: + _verify_remote_object(client, buckets[item["store"]], item["key"], + item["sha256"], item["bytes"]) + runtime_manifest = manifest["runtimeManifest"] + _verify_remote_object(client, buckets["artifact"], runtime_manifest["key"], + runtime_manifest["sha256"], runtime_manifest["bytes"]) + return { + "artifactBucket": buckets["artifact"], + "artifactManifestKey": manifest_key, + "manifestHash": manifest["manifestHash"], + "idempotent": True, + "status": "verified_pending", + "committed": False, + } + raise ArtifactError("同一 gameId/versionId 已存在不同 artifact-manifest，拒绝覆盖") + # 先把所有正文和 GamePackage 做完本地预检，再开始任何远端 Put，避免后一个文件漂移 + # 时留下没有数据库 pending 记录的半上传对象。 + local_objects: list[tuple[Mapping, Path, int]] = [] + for item in manifest["objects"]: + package_path = str(item["path"]) + source = root / _safe_rel(str(item["sourcePath"])) + if source.is_symlink() or not source.is_file(): + raise ArtifactError(f"manifest 对应本地文件不存在:{package_path}") + size, sha256 = _file_digest(source) + if size != item["bytes"]: + raise ArtifactError(f"本地对象字节数漂移:{package_path}") + if sha256 != item["sha256"]: + raise ArtifactError(f"本地对象 hash 漂移:{package_path}") + local_objects.append((item, source, size)) + runtime_manifest = manifest["runtimeManifest"] + source = root / _safe_rel(str(runtime_manifest["sourcePath"])) + if source.is_symlink() or not source.is_file(): + raise ArtifactError("GamePackage manifest 本地文件不存在") + payload = source.read_bytes() + if len(payload) != runtime_manifest["bytes"] or _hash_bytes(payload) != runtime_manifest["sha256"]: + raise ArtifactError("GamePackage manifest 本地 hash 漂移") + _validate_game_package(payload, str(manifest["gameId"]), str(manifest["versionId"])) + # 所有本地输入都已验证，以下才产生远端副作用。 + for item, source, size in local_objects: + with source.open("rb") as stream: + _put_remote(client, buckets[item["store"]], item["key"], stream, length=size, + content_type=item["mime"]) + _put_remote(client, buckets["artifact"], runtime_manifest["key"], BytesIO(payload), length=len(payload), + content_type="application/json") + # 先对正文对象做 GET 全量复算，再写 marker；marker 仍不是数据库 CAS 提交记录。 + for item in manifest["objects"]: + _verify_remote_object(client, buckets[item["store"]], item["key"], item["sha256"], item["bytes"]) + _verify_remote_object(client, buckets["artifact"], runtime_manifest["key"], + runtime_manifest["sha256"], runtime_manifest["bytes"]) + payload = _canonical_json(manifest) + _put_remote(client, buckets["artifact"], manifest_key, BytesIO(payload), length=len(payload), + content_type="application/json") + remote_bytes = _read_remote( + client, buckets["artifact"], manifest_key, expected_bytes=len(payload), + ) + if remote_bytes != payload: + raise ArtifactError("上传后 manifest 下载 hash 不匹配") + return { + "artifactBucket": buckets["artifact"], + "artifactManifestKey": manifest_key, + "manifestHash": manifest["manifestHash"], + "idempotent": False, + "status": "verified_pending", + "committed": False, + } + + +def fetch_manifest(config: Mapping, *, key: str, expected_manifest_hash: str, + expected_tenant_id: str, expected_game_id: str, expected_version_id: str) -> dict: + """从对象存储读取 artifact-manifest，不接受任意 URL，调用方需传入受信 key。""" + client = _minio_client(config) + bucket = _bucket(config, "artifact") + if not _ARTIFACT_MANIFEST_KEY_RE.fullmatch(key): + raise ArtifactError("artifact-manifest key 非法") + if not _HASH_RE.fullmatch(expected_manifest_hash): + raise ArtifactError("expectedManifestHash 非法") + _validate_id(expected_tenant_id, "expectedTenantId") + _validate_db_id(expected_game_id, "expectedGameId") + _validate_db_id(expected_version_id, "expectedVersionId") + payload = _read_remote(client, bucket, key, expected_bytes=_MAX_MANIFEST_BYTES) + try: + manifest = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ArtifactError("远端 artifact-manifest 不是合法 JSON") from exc + verify_manifest(manifest) + if manifest["manifestHash"] != expected_manifest_hash: + raise ArtifactError("artifact-manifest 与数据库期望 hash 不匹配") + if (manifest["tenantId"], manifest["gameId"], manifest["versionId"]) != ( + expected_tenant_id, expected_game_id, expected_version_id, + ): + raise ArtifactError("artifact-manifest 与数据库期望身份不匹配") + if key != f"{manifest['keyPrefix']}/artifact-manifest.json": + raise ArtifactError("artifact-manifest key 与内容身份不匹配") + return manifest + + +def download_manifest(manifest: Mapping, config: Mapping, target: Path, *, expected_manifest_hash: str) -> Path: + """按可信 hash 物化到临时目录，全量校验成功后原子交付。""" + verify_manifest(manifest) + if manifest["manifestHash"] != expected_manifest_hash: + raise ArtifactError("物化 manifest 与数据库期望 hash 不匹配") + client = _minio_client(config) + target = Path(target).resolve() + if target.exists(): + raise ArtifactError(f"物化目标已存在:{target}") + target.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=f".{target.name}.partial-", dir=target.parent)).resolve() + try: + for item in manifest["objects"]: + rel = _safe_rel(str(item["path"])) + destination = (staging / rel).resolve() + if staging not in destination.parents: + raise ArtifactError(f"下载路径越界:{rel}") + destination.parent.mkdir(parents=True, exist_ok=True) + data = _read_remote( + client, _bucket(config, str(item["store"])), item["key"], + expected_bytes=item["bytes"], + ) + if len(data) != item["bytes"] or _hash_bytes(data) != item["sha256"]: + raise ArtifactError(f"下载对象 hash 不匹配:{rel}") + destination.write_bytes(data) + runtime_manifest = manifest["runtimeManifest"] + destination = staging / "manifest.json" + data = _read_remote( + client, _bucket(config, "artifact"), runtime_manifest["key"], + expected_bytes=runtime_manifest["bytes"], + ) + if len(data) != runtime_manifest["bytes"] or _hash_bytes(data) != runtime_manifest["sha256"]: + raise ArtifactError("下载 GamePackage manifest hash 不匹配") + _validate_game_package(data, str(manifest["gameId"]), str(manifest["versionId"])) + destination.write_bytes(data) + (staging / "artifact-manifest.json").write_bytes(_canonical_json(manifest)) + staging.replace(target) + except Exception: + shutil.rmtree(staging, ignore_errors=True) + raise + return target diff --git a/tier2/gen-worker/worker/game_source_archive_store.py b/tier2/gen-worker/worker/game_source_archive_store.py new file mode 100644 index 00000000..034648c6 --- /dev/null +++ b/tier2/gen-worker/worker/game_source_archive_store.py @@ -0,0 +1,353 @@ +"""引擎无关游戏源归档工具。 + +源归档只保存源码和源配置；运行包、发布素材、证据仍由 +``game_artifact_store`` 管理。两类清单使用不同 schema、桶和 hash 域，避免 +LittleJS/Canvas 游戏被误标成 Tier2 Phaser 源工程。 +""" + +from __future__ import annotations + +import json +import mimetypes +import re +import shutil +import tempfile +from io import BytesIO +from pathlib import Path +from typing import Iterable, Mapping + +from .game_artifact_store import ( + ArtifactError, + _HASH_RE, + _MAX_MANIFEST_BYTES, + _MAX_OBJECTS, + _MAX_OBJECT_BYTES, + _SOURCE_TREE_HASH_DOMAIN, + _bucket, + _canonical_json, + _classify, + _file_digest, + _hash_bytes, + _iter_files, + _minio_client, + _put_remote, + _read_remote, + _safe_rel, + _validate_contract, + _validate_db_id, + _validate_id, +) + +SCHEMA_VERSION = "GameSourceArchive/1" +_MANIFEST_HASH_DOMAIN = b"GameSourceArchive/1\0" +_SOURCE_MANIFEST_KEY_RE = re.compile( + r"^tenants/[A-Za-z0-9][A-Za-z0-9._-]*/games/[1-9][0-9]*/" + r"source-revisions/[A-Za-z0-9][A-Za-z0-9._-]*/source-manifest\.json$" +) +_ENGINE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]*$") + + +def _source_hash_from_rows(rows: Iterable[Mapping]) -> str: + """按 manifest 中的路径、字节数和内容 hash 复算源树摘要。""" + canonical_rows = sorted( + [ + {"path": _safe_rel(str(row["path"])), "bytes": row["bytes"], "sha256": row["sha256"]} + for row in rows + ], + key=lambda row: row["path"], + ) + return _hash_bytes(_SOURCE_TREE_HASH_DOMAIN + _canonical_json(canonical_rows)) + + +def source_hash_from_files(files: Iterable[tuple[str, Path]]) -> str: + """对一组源码文件按字节计算可跨引擎复算的源 hash。""" + rows: list[dict] = [] + seen: set[str] = set() + for rel, path in sorted(files, key=lambda item: item[0]): + normalized = _safe_rel(rel) + if normalized in seen: + raise ArtifactError(f"源归档存在重复路径:{normalized}") + seen.add(normalized) + size, sha256 = _file_digest(path) + rows.append({"path": normalized, "bytes": size, "sha256": sha256}) + if not rows: + raise ArtifactError("游戏包缺少源工程文件") + if len(rows) > _MAX_OBJECTS: + raise ArtifactError(f"源归档对象数量超过 {_MAX_OBJECTS}") + return _source_hash_from_rows(rows) + + +def _source_files(root: Path, exclude_paths: Iterable[str], runtime_manifest_path: str | None) -> list[tuple[str, Path]]: + """从游戏目录挑出与运行制品清单相同口径的 source 类文件。""" + excluded = {_safe_rel(path) for path in exclude_paths} + if runtime_manifest_path: + excluded.add(_safe_rel(runtime_manifest_path)) + # 两份对象清单是上传器生成的 marker，不属于源工程正文。 + excluded.update({"artifact-manifest.json", "source-manifest.json"}) + files: list[tuple[str, Path]] = [] + for rel, path in _iter_files(root): + category, _ = _classify(rel) + if category == "source" and rel not in excluded: + files.append((rel, path)) + return files + + +def build_source_archive(root: Path, tenant_id: str, game_id: str, revision_id: str, *, engine: str, + exclude_paths: Iterable[str] = (), runtime_manifest_path: str | None = "game-package.json", + expected_source_hash: str | None = None) -> dict: + """扫描源码并生成可上传的 GameSourceArchive/1 清单。""" + _validate_id(tenant_id, "tenantId") + _validate_db_id(game_id, "gameId") + _validate_id(revision_id, "revisionId") + if not isinstance(engine, str) or not _ENGINE_RE.fullmatch(engine) or len(engine) > 64: + raise ArtifactError(f"engine 非法:{engine!r}") + root = Path(root).resolve() + files = _source_files(root, exclude_paths, runtime_manifest_path) + if not files: + raise ArtifactError("游戏包缺少源工程文件") + key_prefix = f"tenants/{tenant_id}/games/{game_id}/source-revisions/{revision_id}" + objects: list[dict] = [] + for rel, path in files: + size, sha256 = _file_digest(path) + package_path = f"source/{_safe_rel(rel)}" + objects.append({ + "path": package_path, + "sourcePath": _safe_rel(rel), + "key": f"{key_prefix}/{package_path}", + "bytes": size, + "sha256": sha256, + "mime": mimetypes.guess_type(path.name)[0] or "application/octet-stream", + }) + objects.sort(key=lambda item: item["path"]) + source_hash = _source_hash_from_rows( + {"path": item["sourcePath"], "bytes": item["bytes"], "sha256": item["sha256"]} + for item in objects + ) + if expected_source_hash is not None and source_hash != expected_source_hash: + raise ArtifactError("源归档 hash 与期望 sourceHash 不匹配") + manifest = { + "schemaVersion": SCHEMA_VERSION, + "tenantId": tenant_id, + "gameId": game_id, + "revisionId": revision_id, + "engine": engine, + "keyPrefix": key_prefix, + "objects": objects, + "totals": {"objects": len(objects), "bytes": sum(item["bytes"] for item in objects)}, + "sourceHash": source_hash, + } + manifest["manifestHash"] = _hash_bytes(_MANIFEST_HASH_DOMAIN + _canonical_json(manifest)) + if len(_canonical_json(manifest)) > _MAX_MANIFEST_BYTES: + raise ArtifactError(f"source manifest 超过 {_MAX_MANIFEST_BYTES} 字节") + verify_source_archive(manifest) + return manifest + + +def verify_source_archive(manifest: Mapping) -> None: + """校验源归档清单的身份、路径、汇总和 hash 链。""" + _validate_contract(manifest, "game-source-archive.schema.json") + if len(_canonical_json(manifest)) > _MAX_MANIFEST_BYTES: + raise ArtifactError(f"source manifest 超过 {_MAX_MANIFEST_BYTES} 字节") + if manifest.get("schemaVersion") != SCHEMA_VERSION: + raise ArtifactError("source manifest schemaVersion 不受支持") + _validate_id(str(manifest.get("tenantId", "")), "tenantId") + _validate_db_id(str(manifest.get("gameId", "")), "gameId") + _validate_id(str(manifest.get("revisionId", "")), "revisionId") + if not isinstance(manifest.get("engine"), str) or not _ENGINE_RE.fullmatch(manifest["engine"]): + raise ArtifactError("source manifest engine 非法") + key_prefix = str(manifest.get("keyPrefix", "")) + expected_prefix = ( + f"tenants/{manifest['tenantId']}/games/{manifest['gameId']}" + f"/source-revisions/{manifest['revisionId']}" + ) + if key_prefix != expected_prefix: + raise ArtifactError("source manifest keyPrefix 与身份不匹配") + declared = manifest.get("manifestHash") + if not isinstance(declared, str) or not _HASH_RE.fullmatch(declared): + raise ArtifactError("source manifestHash 非法") + unsigned = dict(manifest) + unsigned.pop("manifestHash", None) + if _hash_bytes(_MANIFEST_HASH_DOMAIN + _canonical_json(unsigned)) != declared: + raise ArtifactError("source manifestHash 不匹配") + objects = manifest.get("objects") or [] + if not objects or len(objects) > _MAX_OBJECTS: + raise ArtifactError("source manifest objects 数量非法") + seen_paths: set[str] = set() + seen_sources: set[str] = set() + rows: list[dict] = [] + total_bytes = 0 + for item in objects: + raw_path = str(item.get("path", "")) + raw_source_path = str(item.get("sourcePath", "")) + path = _safe_rel(raw_path) + source_path = _safe_rel(raw_source_path) + if raw_path != path or raw_source_path != source_path: + raise ArtifactError(f"source manifest 路径必须使用 NFC 规范形式:{raw_source_path}") + if path != f"source/{source_path}": + raise ArtifactError(f"source path 与 sourcePath 不匹配:{path}") + if path in seen_paths or source_path in seen_sources: + raise ArtifactError(f"source manifest 存在重复路径:{path}") + seen_paths.add(path) + seen_sources.add(source_path) + expected_key = f"{key_prefix}/{path}" + if item.get("key") != expected_key: + raise ArtifactError(f"source object key 与 path 不匹配:{path}") + if not isinstance(item.get("bytes"), int) or item["bytes"] < 0 or item["bytes"] > _MAX_OBJECT_BYTES: + raise ArtifactError(f"source object bytes 非法:{path}") + if not _HASH_RE.fullmatch(str(item.get("sha256", ""))): + raise ArtifactError(f"source object hash 非法:{path}") + if not isinstance(item.get("mime"), str) or not item["mime"]: + raise ArtifactError(f"source object mime 非法:{path}") + total_bytes += item["bytes"] + rows.append({"path": source_path, "bytes": item["bytes"], "sha256": item["sha256"]}) + totals = manifest.get("totals") or {} + if totals.get("objects") != len(objects) or totals.get("bytes") != total_bytes: + raise ArtifactError("source manifest totals 不匹配") + if total_bytes < 1: + raise ArtifactError("source manifest 不能是全空源工程") + if manifest.get("sourceHash") != _source_hash_from_rows(rows): + raise ArtifactError("source manifest sourceHash 不匹配") + + +def _existing_source_manifest(client, bucket: str, key: str) -> dict | None: + """读取已存在的源 marker；只有对象不存在才视为未上传。""" + try: + payload = _read_remote(client, bucket, key, expected_bytes=_MAX_MANIFEST_BYTES) + except Exception as exc: # noqa: BLE001 - 兼容 MinIO S3Error 和测试 fake + if getattr(exc, "code", None) in {"NoSuchKey", "NoSuchObject"}: + return None + raise ArtifactError(f"探测远端 source-manifest 失败:{type(exc).__name__}:{exc}") from exc + try: + existing = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ArtifactError("远端 source-manifest 已存在但不是合法 JSON") from exc + verify_source_archive(existing) + return existing + + +def _verify_remote_source_object(client, bucket: str, item: Mapping) -> None: + """用 GET 全量复算源对象，不能用 ETag 替代内容校验。""" + try: + data = _read_remote(client, bucket, str(item["key"]), expected_bytes=item["bytes"]) + except ArtifactError: + raise + except Exception as exc: # noqa: BLE001 - 缺对象/SDK 错误统一成稳定失败 + raise ArtifactError(f"读取远端源对象失败:{bucket}/{item['key']}:{type(exc).__name__}:{exc}") from exc + if len(data) != item["bytes"] or _hash_bytes(data) != item["sha256"]: + raise ArtifactError(f"上传后源对象校验失败:{item['path']}") + + +def upload_source_archive(root: Path, manifest: Mapping, config: Mapping, *, + external_single_writer: bool = False) -> dict: + """在外部单写锁内上传源对象并返回 verified_pending。""" + verify_source_archive(manifest) + if external_single_writer is not True: + raise ArtifactError("波次 1 源归档上传必须由调用方持有外部单写锁") + client = _minio_client(config) + bucket = _bucket(config, "source") + root = Path(root).resolve() + manifest_key = f"{manifest['keyPrefix']}/source-manifest.json" + existing = _existing_source_manifest(client, bucket, manifest_key) + if existing is not None: + if existing["manifestHash"] != manifest["manifestHash"]: + raise ArtifactError("同一源修订已存在不同 source-manifest，拒绝覆盖") + for item in manifest["objects"]: + _verify_remote_source_object(client, bucket, item) + return { + "sourceBucket": bucket, + "sourceManifestKey": manifest_key, + "sourceHash": manifest["sourceHash"], + "manifestHash": manifest["manifestHash"], + "idempotent": True, + "status": "verified_pending", + "committed": False, + } + # 先验证整个本地源目录，再开始远端写入，失败不会留下未登记的 partial。 + local_objects: list[tuple[Mapping, Path, int]] = [] + for item in manifest["objects"]: + source = root / _safe_rel(str(item["sourcePath"])) + if source.is_symlink() or not source.is_file(): + raise ArtifactError(f"source manifest 对应本地文件不存在:{item['sourcePath']}") + size, sha256 = _file_digest(source) + if size != item["bytes"] or sha256 != item["sha256"]: + raise ArtifactError(f"本地源对象 hash 漂移:{item['sourcePath']}") + local_objects.append((item, source, size)) + for item, source, size in local_objects: + with source.open("rb") as stream: + _put_remote(client, bucket, item["key"], stream, length=size, content_type=item["mime"]) + for item in manifest["objects"]: + _verify_remote_source_object(client, bucket, item) + payload = _canonical_json(manifest) + _put_remote(client, bucket, manifest_key, BytesIO(payload), length=len(payload), content_type="application/json") + if _read_remote(client, bucket, manifest_key, expected_bytes=len(payload)) != payload: + raise ArtifactError("上传后 source-manifest 下载 hash 不匹配") + return { + "sourceBucket": bucket, + "sourceManifestKey": manifest_key, + "sourceHash": manifest["sourceHash"], + "manifestHash": manifest["manifestHash"], + "idempotent": False, + "status": "verified_pending", + "committed": False, + } + + +def fetch_source_archive(config: Mapping, *, key: str, expected_manifest_hash: str, + expected_tenant_id: str, expected_game_id: str, + expected_revision_id: str) -> dict: + """按受信 key/hash/身份读取源清单，拒绝任意 URL 或跨版本清单。""" + if not _SOURCE_MANIFEST_KEY_RE.fullmatch(key): + raise ArtifactError("source-manifest key 非法") + if not _HASH_RE.fullmatch(expected_manifest_hash): + raise ArtifactError("expected source manifest hash 非法") + _validate_id(expected_tenant_id, "expectedTenantId") + _validate_db_id(expected_game_id, "expectedGameId") + _validate_id(expected_revision_id, "expectedRevisionId") + client = _minio_client(config) + payload = _read_remote(client, _bucket(config, "source"), key, expected_bytes=_MAX_MANIFEST_BYTES) + try: + manifest = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ArtifactError("远端 source-manifest 不是合法 JSON") from exc + verify_source_archive(manifest) + if manifest["manifestHash"] != expected_manifest_hash: + raise ArtifactError("source-manifest 与数据库期望 hash 不匹配") + if (manifest["tenantId"], manifest["gameId"], manifest["revisionId"]) != ( + expected_tenant_id, expected_game_id, expected_revision_id, + ): + raise ArtifactError("source-manifest 与数据库期望身份不匹配") + if key != f"{manifest['keyPrefix']}/source-manifest.json": + raise ArtifactError("source-manifest key 与内容身份不匹配") + return manifest + + +def download_source_archive(manifest: Mapping, config: Mapping, target: Path, *, + expected_manifest_hash: str) -> Path: + """逐文件校验并原子物化源码；失败时清理 staging 目录。""" + verify_source_archive(manifest) + if manifest["manifestHash"] != expected_manifest_hash: + raise ArtifactError("物化 source manifest 与数据库期望 hash 不匹配") + client = _minio_client(config) + target = Path(target).resolve() + if target.exists(): + raise ArtifactError(f"物化目标已存在:{target}") + target.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=f".{target.name}.partial-", dir=target.parent)).resolve() + try: + for item in manifest["objects"]: + rel = _safe_rel(str(item["sourcePath"])) + destination = (staging / rel).resolve() + if staging not in destination.parents: + raise ArtifactError(f"下载源路径越界:{rel}") + destination.parent.mkdir(parents=True, exist_ok=True) + data = _read_remote( + client, _bucket(config, "source"), item["key"], expected_bytes=item["bytes"], + ) + if len(data) != item["bytes"] or _hash_bytes(data) != item["sha256"]: + raise ArtifactError(f"下载对象 hash 不匹配:{rel}") + destination.write_bytes(data) + staging.replace(target) + except Exception: + shutil.rmtree(staging, ignore_errors=True) + raise + return target