b0bc7a8745
一、技能重组（动作-对象命名） - 旧目录 clean/confirm/continuation/db/detect/embed/… 重组为 clean-book-text/decide-candidate/write-next-chapter/access-database/ check-content-consistency/embed-knowledge/…（git 识别为 rename，内容保持） - agents/*.md、AGENTS.md/CLAUDE.md 收编、example_skill 登记表同步新名 二、先审后入创作闭环（本次核心） 正文接受从"机械门一过就写正典"改为"机械门+语义审查双通过+用户批准+单事务原子提交"， DB 级兜底，编排层跳步即被硬拒。 - candidate_cas.py + example_candidate_cas(109)：持久化 CAS 状态链 - fact_delta.py + example_fact_delta/example_fact_ledger(106)：结构化事实增量， 模型只提六型闭集增量+正文证据引文，仅用户批准的增量随正文同事务入账本 - projection_registry.py + example_projection_run(107)：投影登记与恢复 - acceptance_state.py：接受前置实时状态重读 - lesson_registry.py + example_lesson(108)：经验升格链，禁止自动升格 - DDL 105：example_candidate 增 semantic_status/semantic_report_sha256 - write_canonical.accept：语义兜底+同事务合并增量+登记投影； run_writer_pipeline/persist_writer_run/run_writer_semantic_detector/step2 接入全链 - claude_runtime：兼容新 CLI modelUsage 信息字段 三、审查修复（独立子代理四维审查后） - 事实增量 propose→approve 翻态正道，不撞唯一键 - 冻结配置探针重刷（CLI 2.1.211→2.1.231 漂移），profileSha256/adapterVersion 再登记 - 可视化合同悬空路径/五六空间矛盾、 SoT 旧技能名漂移、行尾空白清理 测试：离线 65 套 + 真实库集成 5 套（CAS/接受故障注入/事实增量/投影/经验升格）+ 回放 79 项全绿。 创作内容（docs/design、生成正文 artifacts）按"框架与创作分开"未入本提交。
648 lines
27 KiB
Python
648 lines
27 KiB
Python
#!/usr/bin/env python3
|
||
"""离线评测敏感原文的短生命周期 raw vault。
|
||
|
||
raw 字节只能在非敏感 lease 已原子落盘后写入。所有路径都由管理器生成并通过
|
||
dir-fd、O_NOFOLLOW 和相对分量访问；日志与回执不包含 vault 绝对路径。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import pathlib
|
||
import secrets
|
||
import stat
|
||
from dataclasses import dataclass
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Any, Mapping, Sequence
|
||
|
||
|
||
HASH_PATTERN = __import__("re").compile(r"^sha256:[0-9a-f]{64}$")
|
||
APPROVED_ROLES = frozenset({"用户", "创始人", "user", "founder"})
|
||
MAX_RETENTION = timedelta(hours=24)
|
||
ARCHIVE_PREFIX = "muse-raw-archive-"
|
||
|
||
|
||
class RawVaultError(RuntimeError):
|
||
"""携带稳定码且不暴露 raw 路径或原文的 vault 错误。"""
|
||
|
||
def __init__(self, code: str, message: str) -> None:
|
||
super().__init__(message)
|
||
self.code = code
|
||
self.acceptance_eligible = False
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class VaultLease:
|
||
"""调用方可持有的非敏感 vault lease 身份。"""
|
||
|
||
authorization_id: str
|
||
approved_by: str
|
||
run_id: str
|
||
vault_id: str
|
||
source_version: str
|
||
content_hashes: tuple[str, ...]
|
||
created_at: str
|
||
retain_until: str
|
||
purpose: str
|
||
|
||
|
||
def _utc_now() -> datetime:
|
||
"""返回带时区的 UTC 当前时间，便于测试和审计。"""
|
||
|
||
return datetime.now(timezone.utc)
|
||
|
||
|
||
def _canonical_json(value: Any) -> str:
|
||
"""生成非敏感 journal 的规范 JSON。"""
|
||
|
||
return json.dumps(
|
||
value,
|
||
ensure_ascii=False,
|
||
sort_keys=True,
|
||
separators=(",", ":"),
|
||
allow_nan=False,
|
||
)
|
||
|
||
|
||
def _fsync_directory(path: pathlib.Path) -> None:
|
||
"""持久化目录项变更，避免 rename 仅停留在页缓存。"""
|
||
|
||
descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||
try:
|
||
os.fsync(descriptor)
|
||
finally:
|
||
os.close(descriptor)
|
||
|
||
|
||
def _atomic_write_json(path: pathlib.Path, value: Mapping[str, Any]) -> None:
|
||
"""以 0600 临时文件、fsync、rename 和目录 fsync 原子发布 JSON。"""
|
||
|
||
temporary = path.parent / f".{path.name}.{secrets.token_hex(8)}.tmp"
|
||
descriptor = os.open(
|
||
temporary,
|
||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
|
||
0o600,
|
||
)
|
||
try:
|
||
encoded = (_canonical_json(value) + "\n").encode("utf-8")
|
||
with os.fdopen(descriptor, "wb", closefd=True) as handle:
|
||
handle.write(encoded)
|
||
handle.flush()
|
||
os.fsync(handle.fileno())
|
||
os.replace(temporary, path)
|
||
os.chmod(path, 0o600, follow_symlinks=False)
|
||
_fsync_directory(path.parent)
|
||
except BaseException:
|
||
try:
|
||
temporary.unlink(missing_ok=True)
|
||
finally:
|
||
raise
|
||
|
||
|
||
def _parse_time(value: str, field: str) -> datetime:
|
||
"""解析必须带时区的 ISO-8601 时间。"""
|
||
|
||
if not isinstance(value, str) or not value:
|
||
raise RawVaultError("RAW_LEASE_INVALID", f"{field} 不能为空")
|
||
try:
|
||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||
except ValueError as exc:
|
||
raise RawVaultError("RAW_LEASE_INVALID", f"{field} 格式非法") from exc
|
||
if parsed.tzinfo is None:
|
||
raise RawVaultError("RAW_LEASE_INVALID", f"{field} 必须带时区")
|
||
return parsed.astimezone(timezone.utc)
|
||
|
||
|
||
class RawVaultManager:
|
||
"""管理一个 journal 命名空间内的不可猜测 raw vault。"""
|
||
|
||
def __init__(
|
||
self,
|
||
journal_root: str | pathlib.Path,
|
||
*,
|
||
vault_root: str | pathlib.Path = "/private/tmp",
|
||
) -> None:
|
||
"""创建受限 journal，并拒绝 journal/vault 根软链接。"""
|
||
|
||
self.journal_root = pathlib.Path(journal_root).absolute()
|
||
self.vault_root = pathlib.Path(vault_root).absolute()
|
||
for path, create in ((self.vault_root, False), (self.journal_root, True)):
|
||
if path.is_symlink():
|
||
raise RawVaultError("RAW_PATH_INVALID", "vault 管理根不能是软链接")
|
||
if create:
|
||
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||
if not path.is_dir():
|
||
raise RawVaultError("RAW_PATH_INVALID", "vault 管理根必须是目录")
|
||
# `/private/tmp` 是系统共享根，绝不能改权限；只有本管理器创建的 journal 固定 0700。
|
||
if create:
|
||
os.chmod(path, 0o700)
|
||
self.leases_root = self.journal_root / "leases"
|
||
if self.leases_root.is_symlink():
|
||
raise RawVaultError("RAW_PATH_INVALID", "lease 目录不能是软链接")
|
||
self.leases_root.mkdir(exist_ok=True, mode=0o700)
|
||
os.chmod(self.leases_root, 0o700)
|
||
namespace = hashlib.sha256(str(self.journal_root).encode("utf-8")).hexdigest()[:16]
|
||
self.vault_prefix = f"muse-raw-vault-{namespace}-"
|
||
|
||
def _lease_path(self, vault_id: str) -> pathlib.Path:
|
||
"""把受控 vault ID 映射到非敏感 lease 文件。"""
|
||
|
||
if not isinstance(vault_id, str) or not __import__("re").fullmatch(r"[0-9a-f]{32}", vault_id):
|
||
raise RawVaultError("RAW_LEASE_INVALID", "vault ID 非法")
|
||
return self.leases_root / f"{vault_id}.json"
|
||
|
||
def _vault_name(self, vault_id: str) -> str:
|
||
"""生成同一管理器命名空间内的 vault 目录名。"""
|
||
|
||
self._lease_path(vault_id)
|
||
return f"{self.vault_prefix}{vault_id}"
|
||
|
||
def _vault_path(self, vault_id: str) -> pathlib.Path:
|
||
"""仅供内部文件操作生成 vault 路径，调用结果绝不写入回执。"""
|
||
|
||
return self.vault_root / self._vault_name(vault_id)
|
||
|
||
def create_vault(
|
||
self,
|
||
*,
|
||
authorization_id: str,
|
||
approved_by: str,
|
||
run_id: str,
|
||
source_version: str,
|
||
content_hashes: Sequence[str],
|
||
purpose: str,
|
||
retain_until: str,
|
||
min_remaining: timedelta | None = None,
|
||
) -> VaultLease:
|
||
"""先持久化 lease，再创建 0700 不可猜测 vault。"""
|
||
|
||
text_fields = {
|
||
"authorizationId": authorization_id,
|
||
"runId": run_id,
|
||
"sourceVersion": source_version,
|
||
"purpose": purpose,
|
||
}
|
||
if any(not isinstance(value, str) or not value.strip() for value in text_fields.values()):
|
||
raise RawVaultError("RAW_LEASE_INVALID", "lease 身份字段不能为空")
|
||
if approved_by not in APPROVED_ROLES:
|
||
raise RawVaultError("RAW_LEASE_INVALID", "lease 审批角色不受信任")
|
||
hashes = tuple(content_hashes)
|
||
if not hashes or any(not isinstance(value, str) or not HASH_PATTERN.fullmatch(value) for value in hashes):
|
||
raise RawVaultError("RAW_LEASE_INVALID", "contentHashes 必须是非空 SHA-256 集合")
|
||
now = _utc_now()
|
||
retention = _parse_time(retain_until, "retainUntil")
|
||
if retention <= now or retention - now > MAX_RETENTION:
|
||
raise RawVaultError("RAW_LEASE_INVALID", "raw 保留期限必须在未来 24 小时内")
|
||
# WHY: 调用方可声明「最低剩余租期」。若剩余窗口连一次最长调用加清理余量都覆盖不了
|
||
# （例如只有 27 秒的租约），就必须在落 lease / 建 vault / 调模型之前稳定失败关闭，
|
||
# 而不是先创建再靠到期清理——那样会把敏感 raw 短暂暴露在授权窗口边缘。
|
||
if min_remaining is not None:
|
||
if not isinstance(min_remaining, timedelta) or min_remaining < timedelta(0):
|
||
raise RawVaultError("RAW_LEASE_INVALID", "minRemaining 必须是非负时间间隔")
|
||
if retention - now < min_remaining:
|
||
raise RawVaultError(
|
||
"RAW_LEASE_INSUFFICIENT_RETENTION",
|
||
"raw 剩余租期不足以覆盖执行与清理",
|
||
)
|
||
|
||
vault_id = secrets.token_hex(16)
|
||
lease = VaultLease(
|
||
authorization_id=authorization_id,
|
||
approved_by=approved_by,
|
||
run_id=run_id,
|
||
vault_id=vault_id,
|
||
source_version=source_version,
|
||
content_hashes=hashes,
|
||
created_at=now.isoformat(),
|
||
retain_until=retention.isoformat(),
|
||
purpose=purpose,
|
||
)
|
||
record = self._lease_record(lease, status="open")
|
||
lease_path = self._lease_path(vault_id)
|
||
if lease_path.exists():
|
||
raise RawVaultError("RAW_LEASE_INVALID", "vault ID 冲突")
|
||
# 顺序是安全合同：lease fsync 完成前，磁盘上不能出现 raw vault。
|
||
_atomic_write_json(lease_path, record)
|
||
try:
|
||
os.mkdir(self._vault_path(vault_id), 0o700)
|
||
os.chmod(self._vault_path(vault_id), 0o700, follow_symlinks=False)
|
||
_fsync_directory(self.vault_root)
|
||
except OSError as exc:
|
||
# lease 保持 open，恢复扫描会补记缺失或清理半成品，不能静默撤销审计记录。
|
||
raise RawVaultError("RAW_CREATE_FAILED", "raw vault 创建失败") from exc
|
||
return lease
|
||
|
||
def _lease_record(
|
||
self,
|
||
lease: VaultLease,
|
||
*,
|
||
status: str,
|
||
closed_at: str | None = None,
|
||
close_reason: str | None = None,
|
||
) -> dict[str, Any]:
|
||
"""生成不含路径和正文的 lease journal 内容。"""
|
||
|
||
record: dict[str, Any] = {
|
||
"schemaVersion": "raw-vault-lease-v1",
|
||
"authorizationId": lease.authorization_id,
|
||
"approvedBy": lease.approved_by,
|
||
"runId": lease.run_id,
|
||
"vaultId": lease.vault_id,
|
||
"sourceVersion": lease.source_version,
|
||
"contentHashes": list(lease.content_hashes),
|
||
"createdAt": lease.created_at,
|
||
"retainUntil": lease.retain_until,
|
||
"purpose": lease.purpose,
|
||
"status": status,
|
||
}
|
||
if closed_at is not None:
|
||
record["closedAt"] = closed_at
|
||
if close_reason is not None:
|
||
record["closeReason"] = close_reason
|
||
return record
|
||
|
||
def _load_record(self, vault_id: str) -> dict[str, Any]:
|
||
"""以 O_NOFOLLOW 读取并校验 lease journal。"""
|
||
|
||
path = self._lease_path(vault_id)
|
||
try:
|
||
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
||
with os.fdopen(descriptor, "r", encoding="utf-8") as handle:
|
||
record = json.load(handle)
|
||
except (OSError, json.JSONDecodeError) as exc:
|
||
raise RawVaultError("RAW_LEASE_INVALID", "lease journal 不可读取") from exc
|
||
required = {
|
||
"schemaVersion",
|
||
"authorizationId",
|
||
"approvedBy",
|
||
"runId",
|
||
"vaultId",
|
||
"sourceVersion",
|
||
"contentHashes",
|
||
"createdAt",
|
||
"retainUntil",
|
||
"purpose",
|
||
"status",
|
||
}
|
||
if not isinstance(record, dict) or required - set(record) or record.get("vaultId") != vault_id:
|
||
raise RawVaultError("RAW_LEASE_INVALID", "lease journal 结构非法")
|
||
return record
|
||
|
||
def _lease_from_record(self, record: Mapping[str, Any]) -> VaultLease:
|
||
"""把已校验 journal 转回只含非敏感身份的 lease。"""
|
||
|
||
try:
|
||
return VaultLease(
|
||
authorization_id=str(record["authorizationId"]),
|
||
approved_by=str(record["approvedBy"]),
|
||
run_id=str(record["runId"]),
|
||
vault_id=str(record["vaultId"]),
|
||
source_version=str(record["sourceVersion"]),
|
||
content_hashes=tuple(str(item) for item in record["contentHashes"]),
|
||
created_at=str(record["createdAt"]),
|
||
retain_until=str(record["retainUntil"]),
|
||
purpose=str(record["purpose"]),
|
||
)
|
||
except (KeyError, TypeError) as exc:
|
||
raise RawVaultError("RAW_LEASE_INVALID", "lease journal 结构非法") from exc
|
||
|
||
def _assert_open_lease(self, lease: VaultLease) -> None:
|
||
"""确认调用方 lease 与 journal 完全一致且仍为 open。"""
|
||
|
||
record = self._load_record(lease.vault_id)
|
||
if record.get("status") != "open" or self._lease_from_record(record) != lease:
|
||
raise RawVaultError("RAW_LEASE_INVALID", "lease 已关闭或身份不匹配")
|
||
|
||
def write_bytes(self, lease: VaultLease, relative_path: str, content: bytes) -> None:
|
||
"""通过 openat/no-follow 在 vault 内排他创建 0600 raw 文件。"""
|
||
|
||
self._assert_open_lease(lease)
|
||
# WHY: 租约到期后必须立即拒写，避免敏感字节在授权窗口外继续累积。清理/恢复路径
|
||
# 只走 _assert_open_lease/_load_record，不经过这里，因此过期 lease 仍可被 cleanup/recover 收敛。
|
||
if _parse_time(lease.retain_until, "retainUntil") <= _utc_now():
|
||
raise RawVaultError("RAW_LEASE_EXPIRED", "raw 租约已过期，禁止写入")
|
||
if not isinstance(content, bytes):
|
||
raise RawVaultError("RAW_WRITE_FAILED", "raw 内容必须是 bytes")
|
||
if not isinstance(relative_path, str):
|
||
raise RawVaultError("RAW_PATH_INVALID", "raw 相对路径非法")
|
||
candidate = pathlib.PurePosixPath(relative_path)
|
||
parts = candidate.parts
|
||
if candidate.is_absolute() or not parts or any(part in {"", ".", ".."} for part in parts):
|
||
raise RawVaultError("RAW_PATH_INVALID", "raw 相对路径越界")
|
||
|
||
root_fd = os.open(
|
||
self._vault_path(lease.vault_id),
|
||
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
|
||
)
|
||
current_fd = root_fd
|
||
try:
|
||
for part in parts[:-1]:
|
||
try:
|
||
os.mkdir(part, 0o700, dir_fd=current_fd)
|
||
except FileExistsError:
|
||
pass
|
||
next_fd = os.open(
|
||
part,
|
||
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
|
||
dir_fd=current_fd,
|
||
)
|
||
if current_fd != root_fd:
|
||
os.close(current_fd)
|
||
current_fd = next_fd
|
||
descriptor = os.open(
|
||
parts[-1],
|
||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
|
||
0o600,
|
||
dir_fd=current_fd,
|
||
)
|
||
with os.fdopen(descriptor, "wb") as handle:
|
||
handle.write(content)
|
||
handle.flush()
|
||
os.fsync(handle.fileno())
|
||
os.fsync(current_fd)
|
||
except (OSError, ValueError) as exc:
|
||
if isinstance(exc, (NotADirectoryError, FileExistsError, OSError)):
|
||
code = "RAW_PATH_INVALID"
|
||
else:
|
||
code = "RAW_WRITE_FAILED"
|
||
raise RawVaultError(code, "raw 文件创建失败") from exc
|
||
finally:
|
||
if current_fd != root_fd:
|
||
os.close(current_fd)
|
||
os.close(root_fd)
|
||
|
||
def _remove_tree_at(self, parent_fd: int, name: str) -> None:
|
||
"""递归删除目录项，软链接只 unlink，绝不跟随到 vault 外。"""
|
||
|
||
metadata = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
||
if stat.S_ISDIR(metadata.st_mode):
|
||
directory_fd = os.open(
|
||
name,
|
||
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
|
||
dir_fd=parent_fd,
|
||
)
|
||
try:
|
||
for child in os.listdir(directory_fd):
|
||
self._remove_tree_at(directory_fd, child)
|
||
os.fsync(directory_fd)
|
||
finally:
|
||
os.close(directory_fd)
|
||
os.rmdir(name, dir_fd=parent_fd)
|
||
else:
|
||
os.unlink(name, dir_fd=parent_fd)
|
||
|
||
def _delete_vault_name(self, vault_name: str) -> bool:
|
||
"""按受控目录名清理 vault，返回目录是否原本存在。"""
|
||
|
||
if not vault_name.startswith(self.vault_prefix) or "/" in vault_name:
|
||
raise RawVaultError("RAW_PATH_INVALID", "vault 名称非法")
|
||
root_fd = os.open(self.vault_root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||
try:
|
||
try:
|
||
self._remove_tree_at(root_fd, vault_name)
|
||
except FileNotFoundError:
|
||
return False
|
||
os.fsync(root_fd)
|
||
return True
|
||
finally:
|
||
os.close(root_fd)
|
||
|
||
def _close_lease(self, lease: VaultLease, reason: str) -> dict[str, Any]:
|
||
"""原子补记 lease 关闭状态并返回不含路径的清理回执。"""
|
||
|
||
closed_at = _utc_now().isoformat()
|
||
record = self._lease_record(
|
||
lease,
|
||
status="closed",
|
||
closed_at=closed_at,
|
||
close_reason=reason,
|
||
)
|
||
receipt = {
|
||
"schemaVersion": "raw-vault-cleanup-receipt-v1",
|
||
"vaultId": lease.vault_id,
|
||
"runId": lease.run_id,
|
||
"status": "closed",
|
||
"closedAt": closed_at,
|
||
"reason": reason,
|
||
}
|
||
record["cleanupReceiptSha256"] = "sha256:" + hashlib.sha256(
|
||
_canonical_json(receipt).encode("utf-8")
|
||
).hexdigest()
|
||
_atomic_write_json(self._lease_path(lease.vault_id), record)
|
||
return receipt
|
||
|
||
def cleanup(self, lease: VaultLease) -> dict[str, Any]:
|
||
"""清空 vault 后关闭 lease；清理未成功前不写 closed。"""
|
||
|
||
self._assert_open_lease(lease)
|
||
try:
|
||
existed = self._delete_vault_name(self._vault_name(lease.vault_id))
|
||
except OSError as exc:
|
||
raise RawVaultError("RAW_CLEANUP_FAILED", "raw vault 清理失败") from exc
|
||
return self._close_lease(lease, "cleaned" if existed else "vault_missing")
|
||
|
||
def _vault_inventory(self, lease: VaultLease) -> tuple[str, int, int]:
|
||
"""计算不公开文件名的 raw 内容树摘要，并拒绝软链接和非常规文件。"""
|
||
|
||
self._assert_open_lease(lease)
|
||
root = self._vault_path(lease.vault_id)
|
||
entries: list[dict[str, Any]] = []
|
||
|
||
def visit(directory: pathlib.Path, relative: pathlib.PurePosixPath) -> None:
|
||
try:
|
||
children = sorted(os.scandir(directory), key=lambda item: item.name)
|
||
except OSError as exc:
|
||
raise RawVaultError("RAW_MIGRATION_FAILED", "raw vault 无法读取") from exc
|
||
for child in children:
|
||
child_relative = relative / child.name
|
||
try:
|
||
metadata = child.stat(follow_symlinks=False)
|
||
except OSError as exc:
|
||
raise RawVaultError("RAW_MIGRATION_FAILED", "raw 条目无法读取") from exc
|
||
if stat.S_ISLNK(metadata.st_mode):
|
||
raise RawVaultError("RAW_PATH_INVALID", "raw vault 含软链接")
|
||
if stat.S_ISDIR(metadata.st_mode):
|
||
visit(pathlib.Path(child.path), child_relative)
|
||
continue
|
||
if not stat.S_ISREG(metadata.st_mode):
|
||
raise RawVaultError("RAW_PATH_INVALID", "raw vault 含非常规文件")
|
||
digest = hashlib.sha256()
|
||
try:
|
||
descriptor = os.open(child.path, os.O_RDONLY | os.O_NOFOLLOW)
|
||
with os.fdopen(descriptor, "rb", closefd=True) as handle:
|
||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||
digest.update(chunk)
|
||
except OSError as exc:
|
||
raise RawVaultError("RAW_MIGRATION_FAILED", "raw 文件无法校验") from exc
|
||
entries.append(
|
||
{
|
||
"relativePath": child_relative.as_posix(),
|
||
"contentSha256": "sha256:" + digest.hexdigest(),
|
||
"sizeBytes": metadata.st_size,
|
||
}
|
||
)
|
||
|
||
visit(root, pathlib.PurePosixPath())
|
||
archive_hash = "sha256:" + hashlib.sha256(
|
||
_canonical_json(entries).encode("utf-8")
|
||
).hexdigest()
|
||
return archive_hash, len(entries), sum(item["sizeBytes"] for item in entries)
|
||
|
||
def _archive_root(self, archive_root: str | pathlib.Path) -> pathlib.Path:
|
||
"""准备 0700 的受控归档根，并要求与临时 vault 位于同一文件系统。"""
|
||
|
||
raw_path = pathlib.Path(archive_root)
|
||
if not raw_path.is_absolute():
|
||
raise RawVaultError("RAW_ARCHIVE_INVALID", "raw 归档根必须是绝对路径")
|
||
path = raw_path.absolute()
|
||
if path.is_symlink():
|
||
raise RawVaultError("RAW_ARCHIVE_INVALID", "raw 归档根不能是软链接")
|
||
try:
|
||
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||
if path.is_symlink() or not path.is_dir():
|
||
raise OSError("archive root invalid")
|
||
os.chmod(path, 0o700, follow_symlinks=False)
|
||
if os.stat(path, follow_symlinks=False).st_dev != os.stat(
|
||
self.vault_root, follow_symlinks=False
|
||
).st_dev:
|
||
raise RawVaultError(
|
||
"RAW_ARCHIVE_CROSS_DEVICE",
|
||
"raw 归档根必须与临时 vault 位于同一文件系统",
|
||
)
|
||
except RawVaultError:
|
||
raise
|
||
except OSError as exc:
|
||
raise RawVaultError("RAW_ARCHIVE_INVALID", "raw 归档根不可用") from exc
|
||
return path
|
||
|
||
def migrate(
|
||
self, lease: VaultLease, *, archive_root: str | pathlib.Path
|
||
) -> dict[str, Any]:
|
||
"""把完整 raw vault 原子迁移到受控归档；不删除正文，也不公开归档路径。"""
|
||
|
||
self._assert_open_lease(lease)
|
||
archive = self._archive_root(archive_root)
|
||
archive_hash, file_count, total_bytes = self._vault_inventory(lease)
|
||
archive_id = lease.vault_id
|
||
destination_name = f"{ARCHIVE_PREFIX}{archive_id}"
|
||
destination = archive / destination_name
|
||
if destination.exists() or destination.is_symlink():
|
||
raise RawVaultError("RAW_ARCHIVE_CONFLICT", "raw 归档 ID 冲突")
|
||
|
||
migrating_record = self._lease_record(lease, status="migrating")
|
||
migrating_record.update(
|
||
{
|
||
"archiveId": archive_id,
|
||
"archiveSha256": archive_hash,
|
||
"fileCount": file_count,
|
||
"totalBytes": total_bytes,
|
||
}
|
||
)
|
||
_atomic_write_json(self._lease_path(lease.vault_id), migrating_record)
|
||
|
||
source_fd = os.open(
|
||
self.vault_root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
|
||
)
|
||
archive_fd = os.open(archive, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||
try:
|
||
os.rename(
|
||
self._vault_name(lease.vault_id),
|
||
destination_name,
|
||
src_dir_fd=source_fd,
|
||
dst_dir_fd=archive_fd,
|
||
)
|
||
os.fsync(source_fd)
|
||
os.fsync(archive_fd)
|
||
except OSError as exc:
|
||
# WHY: rename 失败且源目录仍在时恢复 open journal，后续仍可重试迁移；若 rename
|
||
# 已发生但收口中断，则保留 migrating 记录和归档内容，绝不退回删除路径。
|
||
if self._vault_path(lease.vault_id).exists():
|
||
_atomic_write_json(
|
||
self._lease_path(lease.vault_id),
|
||
self._lease_record(lease, status="open"),
|
||
)
|
||
raise RawVaultError("RAW_MIGRATION_FAILED", "raw vault 迁移失败") from exc
|
||
finally:
|
||
os.close(archive_fd)
|
||
os.close(source_fd)
|
||
|
||
migrated_at = _utc_now().isoformat()
|
||
receipt = {
|
||
"schemaVersion": "raw-vault-migration-receipt-v1",
|
||
"vaultId": lease.vault_id,
|
||
"runId": lease.run_id,
|
||
"status": "migrated",
|
||
"archiveId": archive_id,
|
||
"archiveSha256": archive_hash,
|
||
"fileCount": file_count,
|
||
"totalBytes": total_bytes,
|
||
"migratedAt": migrated_at,
|
||
"retentionPolicy": "explicit_cleanup_required",
|
||
}
|
||
receipt_hash = "sha256:" + hashlib.sha256(
|
||
_canonical_json(receipt).encode("utf-8")
|
||
).hexdigest()
|
||
try:
|
||
_atomic_write_json(destination / ".migration-receipt.json", receipt)
|
||
migrated_record = self._lease_record(lease, status="migrated")
|
||
migrated_record.update(
|
||
{
|
||
"archiveId": archive_id,
|
||
"archiveSha256": archive_hash,
|
||
"fileCount": file_count,
|
||
"totalBytes": total_bytes,
|
||
"migratedAt": migrated_at,
|
||
"retentionPolicy": "explicit_cleanup_required",
|
||
"migrationReceiptSha256": receipt_hash,
|
||
}
|
||
)
|
||
_atomic_write_json(self._lease_path(lease.vault_id), migrated_record)
|
||
except OSError as exc:
|
||
raise RawVaultError("RAW_MIGRATION_FAILED", "raw 迁移回执写入失败") from exc
|
||
return receipt
|
||
|
||
def recover(self) -> dict[str, list[str]]:
|
||
"""扫描 open lease、缺失 vault 和同命名空间孤儿目录并安全收敛。"""
|
||
|
||
cleaned: list[str] = []
|
||
closed_missing: list[str] = []
|
||
known_vault_ids: set[str] = set()
|
||
for lease_path in sorted(self.leases_root.glob("*.json")):
|
||
if lease_path.is_symlink():
|
||
raise RawVaultError("RAW_LEASE_INVALID", "lease journal 不能是软链接")
|
||
vault_id = lease_path.stem
|
||
record = self._load_record(vault_id)
|
||
known_vault_ids.add(vault_id)
|
||
if record.get("status") in {"migrating", "migrated"}:
|
||
# 迁移中的数据必须保留给人工恢复；已迁移数据的权威副本在受控归档根。
|
||
# recover 不掌握归档根授权，因此绝不能把迁移状态降级成删除动作。
|
||
continue
|
||
if record.get("status") != "open":
|
||
# closed lease 理论上不再有 raw；若崩溃或外部复制留下同名目录，恢复仍立即清理。
|
||
if self._delete_vault_name(self._vault_name(vault_id)):
|
||
cleaned.append(vault_id)
|
||
continue
|
||
lease = self._lease_from_record(record)
|
||
existed = self._delete_vault_name(self._vault_name(vault_id))
|
||
self._close_lease(lease, "recovery_cleaned" if existed else "recovery_missing")
|
||
(cleaned if existed else closed_missing).append(vault_id)
|
||
|
||
cleaned_orphans: list[str] = []
|
||
for entry in sorted(os.listdir(self.vault_root)):
|
||
if not entry.startswith(self.vault_prefix):
|
||
continue
|
||
suffix = entry.removeprefix(self.vault_prefix)
|
||
if suffix not in known_vault_ids:
|
||
self._delete_vault_name(entry)
|
||
cleaned_orphans.append(suffix)
|
||
return {
|
||
"cleanedVaultIds": cleaned,
|
||
"closedMissingVaultIds": closed_missing,
|
||
"cleanedOrphanIds": cleaned_orphans,
|
||
}
|
||
|
||
|
||
__all__ = ["RawVaultError", "RawVaultManager", "VaultLease"]
|