600 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""presence 冗余收口(work8 十组双行)的纯逻辑离线自测。
红线:**不连真实库、不发任何网络/嵌入/LLM 调用**——只使用内存 fake DB 验证
presence 去重的 planner、快照、确认摘要和 preview/execute 事务合同。
跑法:仓库根目录 `.venv/bin/python tests/skills/extract-work-knowledge/test_presence_dedupe.py`;
也支持从其他工作目录通过该文件的绝对路径运行。
"""
import json
import pathlib
import sys
from contextlib import nullcontext
from unittest.mock import patch
from click.testing import CliRunner
# 从原生产 scripts 导入 upgrade;测试替身只隔离数据库连接,不替代生产逻辑。
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
SCRIPT_DIR = PROJECT_ROOT / "muse" / "content" / "entity" / "skills" / "ingest" / "extract-work-knowledge" / "scripts"
sys.path.insert(0, str(SCRIPT_DIR))
import upgrade as pu # noqa: E402
_passed = 0
def check(name, cond, detail=""):
"""单项断言:通过打 [PASS],失败抛 AssertionError(带上下文,令 CI/人工一眼定位)。"""
global _passed
assert cond, f"[FAIL] {name} :: {detail}"
_passed += 1
print(f"[PASS] {name}")
class _CardResult:
"""为 presence 去重离线测试提供最小查询结果对象。"""
def __init__(self, row=None, rows=None):
self.row = row
self.rows = rows or []
def fetchone(self):
"""返回预置的单行结果。"""
return self.row
def fetchall(self):
"""返回预置的多行结果。"""
return self.rows
# ── presence 冗余收口(work8 十组双行:保留 MIN(id) 软删 MAX(id))──
# 离线 fixture 用字符串时间戳代替数据库驱动的 datetime:窄合同只比较同组两行是否相等,
# _sha256_json 计算摘要时统一 default=str 转写,二者行为一致。
_PRESENCE_DEDUPE_BASE_TIME = "2026-07-01 12:00:00"
def _presence_dedupe_rows():
"""构造满足窄合同的 work8 presence 行 fixture。
十组双行:每组 observation 不同、create_time 相同、恰好一个待删 ID 命中 DELETE_IDS、
保留 ID 是组内较小者;另有四个单行,证明快照会跳过不构成冗余的 key。
"""
rows = []
for index, delete_id in enumerate(sorted(pu.PRESENCE_DEDUPE_DELETE_IDS)):
keep_id = 11501 + index
entity_type = "location" if index % 2 == 0 else "item"
name = f"重复实体{index}"
for row_id, observation in ((keep_id, f"观察A{index}"),
(delete_id, f"观察B{index}")):
rows.append((row_id, 8, 57 + index, 403 + index, entity_type, name,
observation, "upgrade", _PRESENCE_DEDUPE_BASE_TIME,
False, pu.TENANT))
for index in range(4):
rows.append((11400 + index, 8, 10 + index, 20 + index, "character",
f"单体实体{index}", f"单体观察{index}",
"upgrade", _PRESENCE_DEDUPE_BASE_TIME, False, pu.TENANT))
return sorted(rows, key=lambda row: row[0])
def _presence_dedupe_state():
return {
"rows": _presence_dedupe_rows(),
"writes": [],
"queries": [],
"commits": 0,
}
def _mutate_presence_row(state, row_id, column, value):
"""替换指定行的单列;元组不可变,整体重建后写回。"""
state["rows"] = [
row[:column] + (value,) + row[column + 1:] if row[0] == row_id else row
for row in state["rows"]
]
class _PresenceDedupeConn:
"""presence 冗余收口 fake DB:只实现本命令的读快照与软删 SQL。
其他任何域(draft/window/alias/card_state/audit/embedding)的 SQL 会落到末尾
AssertionError,因此「零副作用」无需逐条枚举禁写语句即可离线断言。fake 只做
「未软删」粗过滤;列数/work/tenant/deleted 类型等窄合同行像校验正是离线断言对象,
fake 不能代为过滤。
"""
def __init__(self, state):
self.state = state
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def execute(self, query, params=()):
normalized = " ".join(query.split())
self.state.setdefault("queries", []).append(normalized)
if normalized.startswith("SET TRANSACTION") or normalized.startswith("LOCK TABLE"):
return _CardResult()
if normalized.startswith(
"SELECT id, work_id, window_no, chapter_no, entity_type, name, observation,"):
rows = [
row for row in sorted(self.state["rows"], key=lambda item: item[0])
if len(row) > 9 and not row[9]
]
return _CardResult(rows=rows)
if normalized.startswith("UPDATE example_upgrade_presence SET deleted=TRUE"):
tenant_id, work_id, delete_ids = params
assert tenant_id == pu.TENANT
delete_set = set(delete_ids)
hit = []
new_rows = []
for row in self.state["rows"]:
if row[1] == work_id and row[0] in delete_set and not row[9]:
row = row[:9] + (True,) + row[10:]
hit.append((row[0],))
new_rows.append(row)
self.state["rows"] = new_rows
self.state["writes"].append("UPDATE example_upgrade_presence")
return _CardResult(rows=hit)
raise AssertionError(f"presence 冗余收口 fake DB 未覆盖 SQL:{normalized}")
def commit(self):
self.state["commits"] += 1
def _presence_dedupe_cli(state, args):
"""离线执行 repair-presence-duplicates:注入 fake DB 与空放同书锁。"""
conn = _PresenceDedupeConn(state)
with patch.object(pu, "connect", return_value=conn), \
patch.object(pu, "upgrade_work_lock", return_value=nullcontext()):
result = CliRunner().invoke(pu.maintenance_cli, ["repair-presence-duplicates"] + args)
return result, conn
def _presence_dedupe_preview_sha(state):
"""离线取 preview 的 confirmation_sha,作为 execute 的合法输入。"""
result, _ = _presence_dedupe_cli(state, ["--work-id", "8", "--preview"])
assert result.exit_code == 0, result.output
return json.loads(result.output)["confirmation_sha"]
def _presence_snapshot_raises(label, state, work_id=pu.PRESENCE_DEDUPE_WORK_ID):
"""断言快照以 CompensationFenceConflict 拒绝,且没有发出任何写入。"""
conn = _PresenceDedupeConn(state)
try:
pu._presence_dedupe_capture_snapshot(conn, work_id)
except pu.CompensationFenceConflict as exc:
check(f"presence-dedupe-{label}失败关闭", True, detail=str(exc))
else:
check(f"presence-dedupe-{label}失败关闭", False,
detail="未抛 CompensationFenceConflict")
check(
f"presence-dedupe-{label}零副作用",
state["writes"] == [] and state["commits"] == 0,
)
def test_presence_dedupe_actions_planner_guard():
"""planner 级收口守卫:同观察幂等留首条、不同观察失败关闭、非 presence 原样透传。"""
presence_a = ("presence", None, 403, "location", "重复星体", "观察一")
presence_a_dup = ("presence", None, 403, "location", "重复星体", "观察一")
presence_b = ("presence", None, 404, "item", "镜面护盾", "观察二")
alias_action = ("alias", 7, "规范名", "别名", "ai")
new_action = ("new", -1, {"名称": "新实体"}, set(), [])
actions = [presence_a, alias_action, presence_a_dup, new_action, presence_b, ()]
original = list(actions)
deduped = pu._dedupe_presence_actions(actions)
check(
"presence-dedupe-planner同观察去重保留首条",
deduped == [presence_a, alias_action, new_action, presence_b, ()]
and deduped[0] is presence_a,
detail=str(deduped),
)
check("presence-dedupe-重复应用幂等",
pu._dedupe_presence_actions(deduped) == deduped)
check("presence-dedupe-不修改输入列表", actions == original)
conflict = [presence_a,
("presence", None, 403, "location", "重复星体", "另一个观察")]
try:
pu._dedupe_presence_actions(conflict)
except RuntimeError as exc:
check("presence-dedupe-不同观察失败关闭",
"observation 冲突" in str(exc), detail=str(exc))
else:
check("presence-dedupe-不同观察失败关闭", False, detail="未抛 RuntimeError")
for label, bad, keyword in (
("结构缺一元", ("presence", None, 403, "location", "重复星体"),
"结构非法"),
("结构多一元",
("presence", None, 403, "location", "重复星体", "观察一", "extra"),
"结构非法"),
("observation非字符串", ("presence", None, 403, "location", "重复星体", None),
"必须是字符串"),
):
try:
pu._dedupe_presence_actions([bad])
except RuntimeError as exc:
check(f"presence-dedupe-{label}抛错", keyword in str(exc), detail=str(exc))
else:
check(f"presence-dedupe-{label}抛错", False, detail="未抛 RuntimeError")
def test_presence_dedupe_snapshot_success():
"""快照成功路径:恰好 10 组/20 行,待删集合精确等于 DELETE_IDS,保留为组内较小者。"""
state = _presence_dedupe_state()
snapshot = pu._presence_dedupe_capture_snapshot(
_PresenceDedupeConn(state), pu.PRESENCE_DEDUPE_WORK_ID,
)
check(
"presence-dedupe-快照组数行数精确",
len(snapshot["groups"]) == pu.PRESENCE_DEDUPE_GROUP_COUNT
and len(snapshot["rows"]) == pu.PRESENCE_DEDUPE_ROW_COUNT
and snapshot["contract"] == pu.PRESENCE_DEDUPE_CONTRACT
and snapshot["work_id"] == pu.PRESENCE_DEDUPE_WORK_ID,
)
check(
"presence-dedupe-待删集合精确等于DELETE_IDS",
snapshot["delete_ids"] == sorted(pu.PRESENCE_DEDUPE_DELETE_IDS),
detail=str(snapshot["delete_ids"]),
)
check(
"presence-dedupe-保留为组内较小ID",
snapshot["keep_ids"] == list(range(11501, 11511))
and all(group["keep_id"] < group["delete_id"] for group in snapshot["groups"])
and all(len(group["rows"]) == 2 for group in snapshot["groups"]),
detail=str(snapshot["keep_ids"]),
)
check(
"presence-dedupe-行按id排序且排除单体行",
[row[0] for row in snapshot["rows"]]
== sorted(row[0] for row in snapshot["rows"])
and not any(row[0] in range(11400, 11404) for row in snapshot["rows"]),
)
def test_presence_dedupe_snapshot_fail_closed():
"""各类窄合同漂移都必须在写入前以 CompensationFenceConflict 逐条拒绝。"""
# 额外冗余组:组数 11 ≠ 10(新增组复用合同内待删 ID,确保失败点落在组数校验)。
state = _presence_dedupe_state()
state["rows"].extend((
(11581, 8, 90, 500, "event", "额外重复组", "观察C",
"upgrade", _PRESENCE_DEDUPE_BASE_TIME, False, pu.TENANT),
(11582, 8, 90, 500, "event", "额外重复组", "观察D",
"upgrade", _PRESENCE_DEDUPE_BASE_TIME, False, pu.TENANT),
))
_presence_snapshot_raises("额外冗余组", state)
# 三行组不是「恰好两行」。
state = _presence_dedupe_state()
state["rows"].append(
(11601, 8, 57, 403, "location", "重复实体0", "观察E",
"upgrade", _PRESENCE_DEDUPE_BASE_TIME, False, pu.TENANT)
)
_presence_snapshot_raises("三行冗余组", state)
# 缺行:一组只剩单行被跳过 → 组数 9 ≠ 10。
state = _presence_dedupe_state()
state["rows"] = [row for row in state["rows"]
if row[0] != sorted(pu.PRESENCE_DEDUPE_DELETE_IDS)[0]]
_presence_snapshot_raises("缺行", state)
# 同组两行 create_time 漂移。
state = _presence_dedupe_state()
_mutate_presence_row(state, 11501, 8, "2026-07-02 09:30:00")
_presence_snapshot_raises("create_time漂移", state)
# 待删 ID 命中 0:组内两行都不在 DELETE_IDS。
state = _presence_dedupe_state()
_mutate_presence_row(state, 11501, 0, 12001)
_mutate_presence_row(state, 11582, 0, 12002)
_presence_snapshot_raises("待删ID无命中", state)
# 待删 ID 命中 2:组内两行都在 DELETE_IDS。
state = _presence_dedupe_state()
_mutate_presence_row(state, 11501, 0, 11693)
_presence_snapshot_raises("待删ID双命中", state)
# 保留不是组内较小者:待删 ID 反而是组内较小者。
state = _presence_dedupe_state()
_mutate_presence_row(state, 11501, 0, 11900)
_presence_snapshot_raises("保留非组内较小", state)
# 同组两行 observation 相同(byte-exact 重复违反窄合同)。
state = _presence_dedupe_state()
_mutate_presence_row(state, 11582, 6, "观察A0")
_presence_snapshot_raises("observation相同", state)
# work_id ≠ 8 拒绝。
_presence_snapshot_raises("work不符", _presence_dedupe_state(), work_id=9)
# 窄合同行像漂移:列数不足。
state = _presence_dedupe_state()
state["rows"] = [row[:10] if row[0] == 11501 else row for row in state["rows"]]
_presence_snapshot_raises("列数不符", state)
# 窄合同行像漂移:work 列与查询目标不符。
state = _presence_dedupe_state()
_mutate_presence_row(state, 11501, 1, 9)
_presence_snapshot_raises("work列不符", state)
# 窄合同行像漂移:deleted 不是严格 False(整数 0 也必须拒绝)。
state = _presence_dedupe_state()
_mutate_presence_row(state, 11501, 9, 0)
_presence_snapshot_raises("deleted类型不符", state)
# 窄合同行像漂移:tenant 列不符。
state = _presence_dedupe_state()
_mutate_presence_row(state, 11501, 10, "other-tenant")
_presence_snapshot_raises("tenant不符", state)
def test_presence_dedupe_confirmation_sha_binds_rows_and_ids():
"""confirmation_sha 必须确定且绑定二十行完整内容与保留/删除 ID,任一来源漂移即变化。"""
base_snapshot = pu._presence_dedupe_capture_snapshot(
_PresenceDedupeConn(_presence_dedupe_state()), pu.PRESENCE_DEDUPE_WORK_ID,
)
base_sha = pu._presence_dedupe_confirmation_sha(base_snapshot)
again_snapshot = pu._presence_dedupe_capture_snapshot(
_PresenceDedupeConn(_presence_dedupe_state()), pu.PRESENCE_DEDUPE_WORK_ID,
)
check(
"presence-dedupe-confirmation_sha确定性",
base_sha == pu._presence_dedupe_confirmation_sha(again_snapshot)
and len(base_sha) == 64,
detail=base_sha,
)
for label, row_id, column, value in (
("observation", 11582, 6, "被篡改的观察"),
("保留行ID", 11501, 0, 11001),
("creator", 11501, 7, "篡改者"),
):
state = _presence_dedupe_state()
_mutate_presence_row(state, row_id, column, value)
drifted = pu._presence_dedupe_capture_snapshot(
_PresenceDedupeConn(state), pu.PRESENCE_DEDUPE_WORK_ID,
)
check(
f"presence-dedupe-sha感知{label}漂移",
pu._presence_dedupe_confirmation_sha(drifted) != base_sha,
)
def test_presence_dedupe_cli_preview_read_only():
"""preview:RR READ ONLY 输出十组摘要与 SHA,不写任何表,不触嵌入/模型。"""
state = _presence_dedupe_state()
result, _ = _presence_dedupe_cli(state, ["--work-id", "8", "--preview"])
check("presence-dedupe-preview成功", result.exit_code == 0, detail=result.output)
data = json.loads(result.output)
check(
"presence-dedupe-preview摘要完整",
data["mode"] == "preview"
and data["work_id"] == pu.PRESENCE_DEDUPE_WORK_ID
and data["group_count"] == pu.PRESENCE_DEDUPE_GROUP_COUNT
and data["row_count"] == pu.PRESENCE_DEDUPE_ROW_COUNT
and data["delete_ids"] == sorted(pu.PRESENCE_DEDUPE_DELETE_IDS)
and data["keep_ids"] == list(range(11501, 11511))
and len(data["confirmation_sha"]) == 64
and data["deleted_ids"] is None
and data["audit_rows_written"] == 0
and len(data["groups"]) == pu.PRESENCE_DEDUPE_GROUP_COUNT,
detail=result.output[:500],
)
check(
"presence-dedupe-preview不输出observation原文",
"观察A0" not in result.output and "观察B0" not in result.output,
)
check(
"presence-dedupe-preview零副作用",
state["writes"] == [] and state["commits"] == 0
and all(not row[9] for row in state["rows"]),
)
check(
"presence-dedupe-preview首条SQL为RR只读",
state["queries"][0] == "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY",
detail=str(state["queries"][:2]),
)
check(
"presence-dedupe-preview读快照不加行锁",
not any(query.endswith("FOR UPDATE") for query in state["queries"]),
detail=str(state["queries"]),
)
def test_presence_dedupe_cli_execute_soft_deletes_exactly_ten():
"""execute:锁内重算精确匹配后同事务只软删十个精确 ID;其他域零写入。"""
state = _presence_dedupe_state()
sha = _presence_dedupe_preview_sha(_presence_dedupe_state())
result, _ = _presence_dedupe_cli(
state,
["--work-id", "8", "--execute", "--confirmation-sha", sha,
"--confirm-no-live-process"],
)
check("presence-dedupe-execute成功", result.exit_code == 0, detail=result.output)
data = json.loads(result.output)
check(
"presence-dedupe-execute只软删十个精确ID",
data["mode"] == "execute"
and data["deleted_ids"] == sorted(pu.PRESENCE_DEDUPE_DELETE_IDS)
and sorted(row[0] for row in state["rows"] if row[9] is True)
== sorted(pu.PRESENCE_DEDUPE_DELETE_IDS),
detail=result.output[:500],
)
check(
"presence-dedupe-execute保留行与单体行全部留存",
sorted(row[0] for row in state["rows"] if not row[9])
== sorted(list(range(11501, 11511)) + list(range(11400, 11404))),
detail=str(sorted(row[0] for row in state["rows"] if not row[9])),
)
check(
"presence-dedupe-execute同事务单次提交",
state["writes"] == ["UPDATE example_upgrade_presence"] and state["commits"] == 1,
detail=str(state["writes"]),
)
mutating = [
query for query in state["queries"]
if query.startswith(("UPDATE ", "INSERT ", "DELETE "))
]
check(
"presence-dedupe-execute写SQL仅presence软删",
len(mutating) == 1
and mutating[0].startswith("UPDATE example_upgrade_presence SET deleted=TRUE")
and "RETURNING id" in mutating[0],
detail=str(mutating),
)
check(
"presence-dedupe-execute锁内带行锁重算快照",
any(
query.startswith("SELECT id, work_id, window_no, chapter_no")
and query.endswith("FOR UPDATE")
for query in state["queries"]
),
detail=str(state["queries"]),
)
def test_presence_dedupe_execute_second_run_fails_closed():
"""execute 成功后二次运行:软删后每组只剩单行不成十组,必须失败关闭且无新写入。"""
state = _presence_dedupe_state()
sha = _presence_dedupe_preview_sha(_presence_dedupe_state())
args = ["--work-id", "8", "--execute", "--confirmation-sha", sha,
"--confirm-no-live-process"]
first, _ = _presence_dedupe_cli(state, args)
check("presence-dedupe-首次execute成功", first.exit_code == 0,
detail=first.output)
second, _ = _presence_dedupe_cli(state, args)
check(
"presence-dedupe-二次execute失败关闭",
second.exit_code != 0 and "冗余组数量不精确" in second.output,
detail=second.output,
)
check(
"presence-dedupe-二次execute无新写入",
state["writes"] == ["UPDATE example_upgrade_presence"] and state["commits"] == 1,
detail=str(state["writes"]),
)
def test_presence_dedupe_execute_rejects_stale_or_drifted_snapshot():
"""旧 SHA 与 preview/execute 间快照漂移都必须在写入前失败关闭。"""
# 旧/伪 confirmation-sha:锁内重算后精确匹配拒绝。
state = _presence_dedupe_state()
stale, _ = _presence_dedupe_cli(
state,
["--work-id", "8", "--execute", "--confirmation-sha", "0" * 64,
"--confirm-no-live-process"],
)
check("presence-dedupe-旧SHA失败关闭", stale.exit_code != 0,
detail=stale.output)
check(
"presence-dedupe-旧SHA零写入",
state["writes"] == [] and state["commits"] == 0
and all(not row[9] for row in state["rows"]),
)
# preview 与 execute 之间漂移(observation 被改):锁内重算 SHA 不匹配。
sha = _presence_dedupe_preview_sha(_presence_dedupe_state())
drifted = _presence_dedupe_state()
_mutate_presence_row(drifted, 11582, 6, "被篡改的观察")
result, _ = _presence_dedupe_cli(
drifted,
["--work-id", "8", "--execute", "--confirmation-sha", sha,
"--confirm-no-live-process"],
)
check(
"presence-dedupe-锁内漂移失败关闭",
result.exit_code != 0 and "不匹配" in result.output,
detail=result.output,
)
check(
"presence-dedupe-锁内漂移零写入",
drifted["writes"] == [] and drifted["commits"] == 0
and all(not row[9] for row in drifted["rows"]),
)
def test_presence_dedupe_cli_argument_contracts():
"""preview/execute 互斥;execute 必须同时提供 SHA 与无活进程确认。"""
cases = (
("preview与execute同给", ["--work-id", "8", "--preview", "--execute"]),
("两模式都不给", ["--work-id", "8"]),
("execute缺SHA", ["--work-id", "8", "--execute", "--confirm-no-live-process"]),
("execute缺进程确认",
["--work-id", "8", "--execute", "--confirmation-sha", "ab" * 32]),
("work不符", ["--work-id", "9", "--preview"]),
)
for label, args in cases:
state = _presence_dedupe_state()
result, _ = _presence_dedupe_cli(state, args)
check(f"presence-dedupe-{label}拒绝", result.exit_code != 0,
detail=result.output)
check(
f"presence-dedupe-{label}零副作用",
state["writes"] == [] and state["commits"] == 0,
)
def test_presence_dedupe_cli_preview_fail_closed_on_drift():
"""CLI 层各类快照漂移都在 preview 被拒,且零副作用。"""
first_delete_id = sorted(pu.PRESENCE_DEDUPE_DELETE_IDS)[0]
cases = (
("额外冗余组", lambda state: state["rows"].extend((
(11581, 8, 90, 500, "event", "额外重复组", "观察C",
"upgrade", _PRESENCE_DEDUPE_BASE_TIME, False, pu.TENANT),
(11582, 8, 90, 500, "event", "额外重复组", "观察D",
"upgrade", _PRESENCE_DEDUPE_BASE_TIME, False, pu.TENANT),
))),
("observation相同",
lambda state: _mutate_presence_row(state, first_delete_id, 6, "观察A0")),
("缺行", lambda state: state["rows"].remove(
next(row for row in state["rows"] if row[0] == first_delete_id))),
("tenant不符",
lambda state: _mutate_presence_row(state, 11501, 10, "other-tenant")),
)
for label, mutate in cases:
state = _presence_dedupe_state()
mutate(state)
result, _ = _presence_dedupe_cli(state, ["--work-id", "8", "--preview"])
check(f"presence-dedupe-preview{label}拒绝", result.exit_code != 0,
detail=result.output)
check(
f"presence-dedupe-preview{label}零副作用",
state["writes"] == [] and state["commits"] == 0,
)
if __name__ == "__main__":
for fn in (test_presence_dedupe_actions_planner_guard,
test_presence_dedupe_snapshot_success,
test_presence_dedupe_snapshot_fail_closed,
test_presence_dedupe_confirmation_sha_binds_rows_and_ids,
test_presence_dedupe_cli_preview_read_only,
test_presence_dedupe_cli_execute_soft_deletes_exactly_ten,
test_presence_dedupe_execute_second_run_fails_closed,
test_presence_dedupe_execute_rejects_stale_or_drifted_snapshot,
test_presence_dedupe_cli_argument_contracts,
test_presence_dedupe_cli_preview_fail_closed_on_drift):
fn()
print(f"\n全部离线自测通过:{_passed} 项(未发网络/嵌入/LLM 调用)")