From 2a68b95d36a2f8f902190733da4cb9a793772f3a Mon Sep 17 00:00:00 2001 From: zizi Date: Wed, 22 Jul 2026 09:50:30 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D:=20=E5=B0=81=E9=97=AD?= =?UTF-8?q?=E5=8D=87=E6=A0=BC=E9=87=8D=E5=BB=BA=E7=9A=84=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=AE=8C=E6=95=B4=E6=80=A7=E7=BC=BA=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .claude/skills/embed/SKILL.md | 17 +- .claude/skills/embed/scripts/embed_drafts.py | 184 +++- .../scripts/test_embed_drafts_offline.py | 410 +++++++++ .claude/skills/parse-book/SKILL.md | 14 +- .../parse-book/scripts/backup_upgrade_work.py | 1 + .../parse-book/scripts/parse_upgrade.py | 493 +++++++++-- .../parse-book/scripts/reset_upgrade_work.py | 207 ++++- .../test_backup_upgrade_work_offline.py | 2 +- .../scripts/test_parse_upgrade_offline.py | 829 +++++++++++++++++- .../test_reset_upgrade_work_offline.py | 508 ++++++++++- 10 files changed, 2548 insertions(+), 117 deletions(-) create mode 100644 .claude/skills/embed/scripts/test_embed_drafts_offline.py diff --git a/.claude/skills/embed/SKILL.md b/.claude/skills/embed/SKILL.md index bb1ead4..c045240 100644 --- a/.claude/skills/embed/SKILL.md +++ b/.claude/skills/embed/SKILL.md @@ -23,10 +23,21 @@ description: New-API 嵌入封装——Qwen3-Embedding-8B、dimensions=1024、 ## 合同 - **嵌入文本构造**：`【型】名称：一句话摘要\n字段正文摘选`（draft_payload 的 embed_text 字段优先；无则按固定拼接），与检索端 query 语义对齐。 -- **幂等**：sha256(嵌入文本+模型) 为 `content_hash`，已存在则跳过（uk: tenant+hash+model）。 +- **幂等与 owner**：sha256(嵌入文本+模型) 为 `content_hash`（uk: tenant+hash+model）。只有唯一行 `deleted=FALSE`、绑定同一 `draft_id`，且 owner draft 同租户并 `deleted=FALSE` 时才幂等跳过；`entity_id` 非空或其他 active draft owner 明确冲突并失败，绝不迁移 owner。旧 owner draft 已软删时，允许在写前活性重验后把唯一行条件迁到当前 draft。 - **批量**：每请求 ≤16 条文本；失败整批重试 2 次（指数退避），仍失败逐条降级重试，坏行记错并继续（不断批）。 -- **落库**：`example_knowledge_embedding(draft_id, content_hash, embed_text, model, dimensions=1024, embedding)`；draft 确认落 entity 后由 confirm 流程回填 entity_id（嵌入行不动）。 -- 汇报：新嵌 N、跳过（幂等）M、失败 K 及原因。 +- **落库与 reset/confirm/parse 并发**：HTTP 期间不持数据库事务。每个 draft 写入使用独立事务，先 `SELECT ... FOR UPDATE` 锁定 draft 并重验租户、`deleted=FALSE`、`status='pending'`；同时读取当前 `draft_payload`，重构文本与 hash，和 HTTP 前快照任一不一致即跳过。随后 `SELECT ... FOR UPDATE` 锁定该 draft 全部活向量：任一 `entity_id` 非空则冲突失败，同 hash 当前活向量则幂等跳过，其他 hash 的无 entity 旧活向量在 UPSERT 前统一软删，保证每 draft 仅一个活向量。最后锁同 hash 唯一行，执行带 owner 条件的 UPSERT 并用 `RETURNING draft_id` 校验。该 draft 行锁与 reset 的七表 `SHARE ROW EXCLUSIVE` 配合：embed 先锁时 reset 等待且随后能发现快照漂移；reset 先完成时 embed 等待后看到软删并跳过。confirm/parse 先完成时 embed 在锁后看到状态或 payload/hash 漂移并跳过。以上跳过路径均零向量写入。 +- **落库字段**：`example_knowledge_embedding(draft_id, content_hash, embed_text, model, dimensions=1024, embedding)`；draft 确认落 entity 后由 confirm 流程回填 entity_id（嵌入行不动）。 +- 汇报：新嵌 N、跳过 M、失败 K；幂等、失活和冲突原因均输出可追踪明细。 + +## 离线验证 + +```bash +.venv/bin/python .claude/skills/embed/scripts/test_embed_drafts_offline.py +.venv/bin/python -m py_compile .claude/skills/embed/scripts/embed_drafts.py \ + .claude/skills/embed/scripts/test_embed_drafts_offline.py +``` + +离线测试只使用 fake connection 检查并发顺序、SQL 条件和 owner 反例，不连接真实数据库，不调用 embedding 或 reset。 ## 红线 diff --git a/.claude/skills/embed/scripts/embed_drafts.py b/.claude/skills/embed/scripts/embed_drafts.py index 2d8a675..f4769c0 100644 --- a/.claude/skills/embed/scripts/embed_drafts.py +++ b/.claude/skills/embed/scripts/embed_drafts.py @@ -72,6 +72,162 @@ def build_embed_text(payload: dict) -> str: return f"【{t}】{name}：{brief}\n{body}"[:4000] +class EmbeddingOwnershipConflict(RuntimeError): + """同 hash 唯一行已归实体或其他活跃 draft，禁止迁移 owner。""" + + +def _embedding_owner_action(conn, draft_id, content_hash, *, lock=False): + """判断同 hash 唯一行应幂等跳过还是写入；写段可锁行封住预查后的竞态。""" + + lock_clause = " FOR UPDATE OF e" if lock else "" + owner = conn.execute( + """SELECT e.draft_id, e.entity_id, e.deleted, + COALESCE(d.deleted, TRUE), d.tenant_id + FROM example_knowledge_embedding e + LEFT JOIN muse_knowledge_draft d ON d.id=e.draft_id + WHERE e.tenant_id=%s AND e.content_hash=%s AND e.model=%s""" + lock_clause, + (TENANT, content_hash, MODEL), + ).fetchone() + if not owner: + return "write" + + owner_draft_id, owner_entity_id, embedding_deleted, owner_deleted, owner_tenant = owner + # entity owner 是确认后的正式归属，任何 draft 都不得把它降级抢回。 + if owner_entity_id is not None: + raise EmbeddingOwnershipConflict( + f"同 hash 唯一行已归 entity：hash={content_hash}，entity={owner_entity_id}，" + f"candidate={draft_id}" + ) + # 只有当前租户、当前 draft、两侧都 active 才是真正的幂等命中。 + if owner_draft_id == draft_id: + if owner_tenant != TENANT: + raise EmbeddingOwnershipConflict( + f"同 hash 当前 owner 租户不匹配：hash={content_hash}，" + f"owner_tenant={owner_tenant}，candidate_tenant={TENANT}" + ) + if not embedding_deleted and not owner_deleted: + return "skip" + return "write" + # 空 owner、owner 行缺失或 owner draft 已软删时，可由当前活跃 draft 接管唯一行。 + if owner_draft_id is None or owner_deleted: + return "write" + raise EmbeddingOwnershipConflict( + f"同 hash 唯一行已归其他 active draft：hash={content_hash}，" + f"owner={owner_draft_id}，candidate={draft_id}" + ) + + +def _write_embedding(conn, draft_id, content_hash, text, vector): + """在调用方单 draft 事务内锁定活性与 owner，条件写入并校验最终归属。""" + + # 写事务先按固定表顺序取得 ROW EXCLUSIVE 锁，避免与 reset 的多表锁形成交叉等待。 + conn.execute( + "LOCK TABLE muse_knowledge_draft, example_knowledge_embedding IN ROW EXCLUSIVE MODE" + ) + # 取得表锁后再锁 candidate draft：embed 先到时 reset 的七表 SHARE ROW EXCLUSIVE 会等待； + # reset 先到时本查询等待其提交，随后读取 deleted=TRUE 并拒绝陈旧写入。 + candidate = conn.execute( + """SELECT tenant_id, deleted, status, draft_payload FROM muse_knowledge_draft + WHERE id=%s FOR UPDATE""", + (draft_id,), + ).fetchone() + if not candidate: + click.echo(f" [跳过] draft={draft_id} 写前已不存在，未写向量", err=True) + return False + candidate_tenant, candidate_deleted, candidate_status, current_payload = candidate + if candidate_tenant != TENANT: + raise EmbeddingOwnershipConflict( + f"draft 租户不匹配：draft={draft_id}，tenant={candidate_tenant}，expected={TENANT}" + ) + if candidate_deleted: + click.echo(f" [跳过] draft={draft_id} 写前已软删，未写向量", err=True) + return False + if candidate_status != "pending": + click.echo( + f" [跳过] draft={draft_id} 写前 status={candidate_status}，非 pending，未写向量", + err=True, + ) + return False + + # HTTP 期间 payload 可能被 parse/confirm 更新；锁内必须按当前 payload 重构文本与 hash， + # 只要与 HTTP 请求所依据的快照不同，就丢弃陈旧向量，绝不覆盖并发产生的新结果。 + current_text = build_embed_text(current_payload or {}) + current_hash = hashlib.sha256(f"{current_text}|{MODEL}".encode()).hexdigest() + if current_text != text or current_hash != content_hash: + click.echo( + f" [跳过] draft={draft_id} 写前 payload/hash 漂移，" + f"expected_hash={content_hash} current_hash={current_hash}，未写向量", + err=True, + ) + return False + + # 锁定该 draft 的全部活向量，保证 entity 归属和“每 draft 唯一活向量”在同一事务内判定。 + live_embeddings = conn.execute( + """SELECT id, content_hash, entity_id FROM example_knowledge_embedding + WHERE tenant_id=%s AND draft_id=%s AND deleted=FALSE + FOR UPDATE""", + (TENANT, draft_id), + ).fetchall() + entity_rows = [ + (row_id, row_hash, entity_id) + for row_id, row_hash, entity_id in live_embeddings + if entity_id is not None + ] + if entity_rows: + raise EmbeddingOwnershipConflict( + f"draft={draft_id} 存在 entity_id 非空旧活向量，禁止覆盖：{entity_rows}" + ) + if any(row_hash == content_hash for _, row_hash, _ in live_embeddings): + click.echo(f" [跳过] draft={draft_id} 同 hash 活向量已由当前 draft 持有") + return False + + action = _embedding_owner_action(conn, draft_id, content_hash, lock=True) + if action == "skip": + click.echo(f" [跳过] draft={draft_id} 同 hash 活向量已由当前 draft 持有") + return False + + if live_embeddings: + # 当前 payload 已通过锁内 hash 重验，因此其余 hash 均为该 draft 的过期向量； + # 只允许软删 draft owner，entity owner 已在上方失败关闭。 + conn.execute( + """UPDATE example_knowledge_embedding SET deleted=TRUE, updater=%s + WHERE tenant_id=%s AND draft_id=%s AND deleted=FALSE + AND entity_id IS NULL AND content_hash!=%s""", + (ACTOR, TENANT, draft_id, content_hash), + ) + + # 条件 UPSERT 是行锁检查后的第二道防线：当预查时唯一行尚不存在、随后被并发插入时， + # 仅允许当前 owner 或已失活 owner 迁移；entity/其他 active draft 均令 RETURNING 为空。 + upserted = conn.execute( + """INSERT INTO example_knowledge_embedding + (draft_id, content_hash, embed_text, model, dimensions, embedding, + creator, updater, tenant_id) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) + ON CONFLICT (tenant_id, content_hash, model) + DO UPDATE SET draft_id=EXCLUDED.draft_id, + embed_text=EXCLUDED.embed_text, + model=EXCLUDED.model, + dimensions=EXCLUDED.dimensions, + embedding=EXCLUDED.embedding, + deleted=FALSE, + updater=EXCLUDED.updater + WHERE example_knowledge_embedding.entity_id IS NULL + AND (example_knowledge_embedding.draft_id=EXCLUDED.draft_id + OR NOT EXISTS ( + SELECT 1 FROM muse_knowledge_draft owner + WHERE owner.id=example_knowledge_embedding.draft_id + AND owner.deleted=FALSE)) + RETURNING draft_id""", + (draft_id, content_hash, text, MODEL, DIM, json.dumps(vector), + ACTOR, ACTOR, TENANT), + ).fetchone() + if not upserted or upserted[0] != draft_id: + raise EmbeddingOwnershipConflict( + f"同 hash 唯一行未绑定当前 draft：hash={content_hash}，candidate={draft_id}" + ) + return True + + @click.command() @click.option("--work-id", type=int, help="限定拆书批次的 work（draft.work_id=0 为全局行，用 source_id 关联参考书）") @click.option("--limit", type=int, default=0, help="最多处理条数（0=不限）") @@ -109,13 +265,13 @@ def main(work_id, limit, probe): for did, payload in chunk: text = build_embed_text(payload or {}) h = hashlib.sha256(f"{text}|{MODEL}".encode()).hexdigest() - if conn.execute( - "SELECT 1 FROM example_knowledge_embedding WHERE tenant_id=%s AND content_hash=%s AND model=%s", - (TENANT, h, MODEL)).fetchone(): - skip += 1 # 幂等：同文同模型不重嵌 + if _embedding_owner_action(conn, did, h) == "skip": + skip += 1 # 仅当前 active draft 持有 active 唯一行时才幂等跳过。 continue texts.append(text) metas.append((did, h, text)) + # owner 预查只用于避免无效 HTTP；明确提交读事务，HTTP 期间不持数据库事务或表锁。 + conn.commit() if not texts: continue vecs, bad = embed_texts(sess, texts) @@ -123,22 +279,20 @@ def main(work_id, limit, probe): if j in bad: fail += 1 continue - conn.execute( - """INSERT INTO example_knowledge_embedding - (draft_id, content_hash, embed_text, model, dimensions, embedding, - creator, updater, tenant_id) - VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) - ON CONFLICT (tenant_id, content_hash, model) DO NOTHING""", - (did, h, text, MODEL, DIM, json.dumps(vecs[j]), ACTOR, ACTOR, TENANT)) - done += 1 - conn.commit() + # 每个 draft 独立事务：draft 行锁、owner 行锁、条件 UPSERT 同生共死。 + with conn.transaction(): + written = _write_embedding(conn, did, h, text, vecs[j]) + if written: + done += 1 + else: + skip += 1 click.echo(f" 进度 {min(i + BATCH, len(rows))}/{len(rows)}（新嵌{done} 跳过{skip} 失败{fail}）") - click.echo(f"完成：新嵌 {done}、幂等跳过 {skip}、失败 {fail}") + click.echo(f"完成：新嵌 {done}、跳过 {skip}、失败 {fail}") if __name__ == "__main__": try: main() - except (psycopg.Error, requests.RequestException) as e: + except (EmbeddingOwnershipConflict, psycopg.Error, requests.RequestException) as e: click.echo(f"[错误] {type(e).__name__}: {e}", err=True) sys.exit(1) diff --git a/.claude/skills/embed/scripts/test_embed_drafts_offline.py b/.claude/skills/embed/scripts/test_embed_drafts_offline.py new file mode 100644 index 0000000..3ff8da7 --- /dev/null +++ b/.claude/skills/embed/scripts/test_embed_drafts_offline.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python3 +"""embed_drafts 并发与向量 owner 规则的纯离线测试。""" + +import hashlib +import pathlib +import sys +import unittest +from unittest.mock import patch + + +SCRIPT_DIR = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIR)) + +import embed_drafts as embed # noqa: E402 + +TEXT = "文本" +CONTENT_HASH = hashlib.sha256(f"{TEXT}|{embed.MODEL}".encode()).hexdigest() +NEW_TEXT = "新文本" +NEW_HASH = hashlib.sha256(f"{NEW_TEXT}|{embed.MODEL}".encode()).hexdigest() + + +class _Result: + """提供 psycopg 查询结果所需的最小读取接口。""" + + def __init__(self, row=None, rows=None): + self.row = row + self.rows = list(rows or []) + + def fetchone(self): + return self.row + + def fetchall(self): + return list(self.rows) + + +class _EmbeddingConnection: + """模拟 draft 与同 hash 唯一行，记录写事务内的锁和 UPSERT 顺序。""" + + def __init__(self, *, candidate_deleted=False, candidate_status="pending", + candidate_payload=None, draft_live_embeddings=None, owner_draft_id=None, + owner_entity_id=None, owner_embedding_deleted=False, + owner_draft_deleted=None, owner_tenant=embed.TENANT, + inject_active_owner_before_upsert=False): + self.candidate = { + "tenant": embed.TENANT, + "deleted": candidate_deleted, + "status": candidate_status, + "payload": candidate_payload or {"embed_text": TEXT}, + } + self.draft_live_embeddings = [ + {"id": row_id, "content_hash": content_hash, + "entity_id": entity_id, "deleted": False} + for row_id, content_hash, entity_id in (draft_live_embeddings or []) + ] + self.embedding = None + if owner_draft_id is not None or owner_entity_id is not None: + self.embedding = { + "draft_id": owner_draft_id, + "entity_id": owner_entity_id, + "deleted": owner_embedding_deleted, + "owner_deleted": owner_draft_deleted, + "owner_tenant": owner_tenant, + } + self.events = [] + self.sql = [] + self.inject_active_owner_before_upsert = inject_active_owner_before_upsert + + def execute(self, sql, params=None): + normalized = " ".join(sql.split()).lower() + self.sql.append((normalized, params)) + + if normalized.startswith("lock table"): + self.events.append("table_lock") + self.assert_table_lock_sql(normalized, params) + return _Result() + + if normalized.startswith( + "select tenant_id, deleted, status, draft_payload from muse_knowledge_draft"): + self.events.append("candidate_lock") + self.assert_candidate_lock_sql(normalized, params) + return _Result(( + self.candidate["tenant"], self.candidate["deleted"], + self.candidate["status"], self.candidate["payload"], + )) + + if normalized.startswith("select id, content_hash, entity_id from example_knowledge_embedding"): + self.events.append("draft_vectors_lock") + self.assert_draft_vectors_lock_sql(normalized, params) + rows = [ + (row["id"], row["content_hash"], row["entity_id"]) + for row in self.draft_live_embeddings if not row["deleted"] + ] + return _Result(rows=rows) + + if normalized.startswith("select e.draft_id, e.entity_id, e.deleted"): + self.events.append("owner_lock" if "for update of e" in normalized else "owner_read") + self.assert_owner_sql(normalized, params) + if self.embedding is None: + return _Result(None) + row = self.embedding + owner_deleted = row["owner_deleted"] + if owner_deleted is None: + owner_deleted = True + return _Result(( + row["draft_id"], row["entity_id"], row["deleted"], + owner_deleted, row["owner_tenant"], + )) + + if normalized.startswith("insert into example_knowledge_embedding"): + self.events.append("upsert") + self.assert_upsert_sql(normalized, params) + candidate_draft_id = params[0] + # 模拟 owner 预查时尚无唯一行，随后并发事务先插入另一活跃 owner。 + if self.inject_active_owner_before_upsert and self.embedding is None: + self.embedding = { + "draft_id": 90, + "entity_id": None, + "deleted": False, + "owner_deleted": False, + "owner_tenant": embed.TENANT, + } + if self.embedding is not None: + row = self.embedding + owner_is_active = ( + row["draft_id"] is not None + and row["owner_deleted"] is False + ) + can_take = ( + row["entity_id"] is None + and (row["draft_id"] == candidate_draft_id or not owner_is_active) + ) + if not can_take: + return _Result(None) + self.embedding = { + "draft_id": candidate_draft_id, + "entity_id": None, + "deleted": False, + "owner_deleted": False, + "owner_tenant": embed.TENANT, + } + self.draft_live_embeddings.append({ + "id": 999, + "content_hash": params[1], + "entity_id": None, + "deleted": False, + }) + return _Result((candidate_draft_id,)) + + if normalized.startswith("update example_knowledge_embedding set deleted=true"): + self.events.append("old_vectors_soft_delete") + self.assert_old_vectors_update_sql(normalized, params) + for row in self.draft_live_embeddings: + if (not row["deleted"] and row["entity_id"] is None + and row["content_hash"] != params[2]): + row["deleted"] = True + return _Result() + + raise AssertionError(f"未覆盖的离线 SQL：{normalized}") + + @staticmethod + def assert_table_lock_sql(sql, params): + """写事务第一条 SQL 必须按统一顺序取得两表 ROW EXCLUSIVE 锁。""" + + assert sql == ( + "lock table muse_knowledge_draft, example_knowledge_embedding " + "in row exclusive mode" + ) + assert params is None + + @staticmethod + def assert_candidate_lock_sql(sql, params): + """candidate 必须按主键锁行，并在同一快照读取当前 payload。""" + + assert "where id=%s" in sql + assert "for update" in sql + assert params == (101,) + + @staticmethod + def assert_draft_vectors_lock_sql(sql, params): + """必须锁定当前 draft 的全部活向量，而非只看目标 hash。""" + + assert "where tenant_id=%s and draft_id=%s and deleted=false" in sql + assert "for update" in sql + assert params == (embed.TENANT, 101) + + @staticmethod + def assert_old_vectors_update_sql(sql, params): + """只软删当前 draft 的无 entity 旧 hash 活向量。""" + + assert "entity_id is null" in sql + assert "content_hash!=%s" in sql + assert params == (embed.ACTOR, embed.TENANT, 101, CONTENT_HASH) + + @staticmethod + def assert_owner_sql(sql, params): + """hash 查询必须读取 owner 全状态；写段还必须锁住唯一行。""" + + for fragment in ( + "e.draft_id", "e.entity_id", "e.deleted", + "coalesce(d.deleted, true)", "d.tenant_id", + "left join muse_knowledge_draft d on d.id=e.draft_id", + "e.tenant_id=%s", "e.content_hash=%s", "e.model=%s"): + assert fragment in sql + assert params == (embed.TENANT, CONTENT_HASH, embed.MODEL) + + @staticmethod + def assert_upsert_sql(sql, params): + """唯一键冲突只能迁移无 entity 且 owner 已失活的行，并校验返回 owner。""" + + for fragment in ( + "on conflict (tenant_id, content_hash, model)", + "do update set draft_id=excluded.draft_id", + "example_knowledge_embedding.entity_id is null", + "example_knowledge_embedding.draft_id=excluded.draft_id", + "not exists ( select 1 from muse_knowledge_draft owner", + "owner.id=example_knowledge_embedding.draft_id", + "owner.deleted=false", + "returning draft_id"): + assert fragment in sql + assert params[0] == 101 + assert params[1] == CONTENT_HASH + + +class EmbedDraftsOfflineTest(unittest.TestCase): + """覆盖 reset/embed 两种先后顺序与 hash owner 反例。""" + + VECTOR = [0.1, 0.2] + + def test_reset先完成时软删draft零写入且有可追踪输出(self): + conn = _EmbeddingConnection( + candidate_deleted=True, + owner_draft_id=90, + owner_embedding_deleted=True, + owner_draft_deleted=True, + ) + + with patch.object(embed.click, "echo") as echo: + written = embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR) + + self.assertFalse(written) + self.assertEqual(conn.events, ["table_lock", "candidate_lock"]) + self.assertEqual(conn.embedding["draft_id"], 90) + echo.assert_called_once() + self.assertIn("draft=101", echo.call_args.args[0]) + self.assertIn("软删", echo.call_args.args[0]) + + def test_embed先锁draft时先锁后写且写入会造成reset摘要漂移(self): + conn = _EmbeddingConnection() + before = conn.embedding + + with patch.object(embed.click, "echo"): + written = embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR) + + self.assertTrue(written) + self.assertEqual( + conn.events, + ["table_lock", "candidate_lock", "draft_vectors_lock", "owner_lock", "upsert"], + ) + self.assertIsNone(before) + self.assertEqual(conn.embedding["draft_id"], 101) + self.assertFalse(conn.embedding["deleted"]) + + def test_active同did才允许幂等跳过(self): + conn = _EmbeddingConnection( + owner_draft_id=101, + owner_embedding_deleted=False, + owner_draft_deleted=False, + ) + + action = embed._embedding_owner_action(conn, 101, CONTENT_HASH) + + self.assertEqual(action, "skip") + self.assertEqual(conn.events, ["owner_read"]) + + def test软删旧draft_owner允许迁移到新did(self): + conn = _EmbeddingConnection( + owner_draft_id=90, + owner_embedding_deleted=True, + owner_draft_deleted=True, + ) + + self.assertEqual(embed._embedding_owner_action(conn, 101, CONTENT_HASH), "write") + self.assertTrue(embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR)) + self.assertEqual(conn.embedding["draft_id"], 101) + self.assertFalse(conn.embedding["deleted"]) + + def test_entity_owner明确冲突(self): + conn = _EmbeddingConnection( + owner_draft_id=90, + owner_entity_id=9001, + owner_embedding_deleted=True, + owner_draft_deleted=True, + ) + + with self.assertRaisesRegex(embed.EmbeddingOwnershipConflict, "entity"): + embed._embedding_owner_action(conn, 101, CONTENT_HASH) + + def test其它active_draft_owner明确冲突且绝不迁移(self): + conn = _EmbeddingConnection( + owner_draft_id=90, + owner_embedding_deleted=True, + owner_draft_deleted=False, + ) + + with self.assertRaisesRegex(embed.EmbeddingOwnershipConflict, "owner=90"): + embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR) + + self.assertNotIn("upsert", conn.events) + self.assertEqual(conn.embedding["draft_id"], 90) + + def test_owner检查后出现active_owner时条件upsert失败关闭(self): + conn = _EmbeddingConnection(inject_active_owner_before_upsert=True) + + with self.assertRaisesRegex(embed.EmbeddingOwnershipConflict, "未绑定当前 draft"): + embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR) + + self.assertEqual( + conn.events, + ["table_lock", "candidate_lock", "draft_vectors_lock", "owner_lock", "upsert"], + ) + self.assertEqual(conn.embedding["draft_id"], 90) + + def test_candidate租户不匹配时失败且零写入(self): + conn = _EmbeddingConnection() + conn.candidate["tenant"] = 2 + + with self.assertRaisesRegex(embed.EmbeddingOwnershipConflict, "租户"): + embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR) + + self.assertEqual(conn.events, ["table_lock", "candidate_lock"]) + + def test_confirm先完成时非pending_draft可追踪跳过且零写入(self): + conn = _EmbeddingConnection(candidate_status="confirmed") + + with patch.object(embed.click, "echo") as echo: + written = embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR) + + self.assertFalse(written) + self.assertEqual(conn.events, ["table_lock", "candidate_lock"]) + self.assertIsNone(conn.embedding) + echo.assert_called_once() + self.assertIn("status=confirmed", echo.call_args.args[0]) + + def test_HTTP期间payload变化时重算hash后跳过且零写入(self): + conn = _EmbeddingConnection(candidate_payload={"embed_text": NEW_TEXT}) + + with patch.object(embed.click, "echo") as echo: + written = embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR) + + self.assertFalse(written) + self.assertEqual(conn.events, ["table_lock", "candidate_lock"]) + self.assertIsNone(conn.embedding) + self.assertIn("payload", echo.call_args.args[0]) + self.assertIn("漂移", echo.call_args.args[0]) + + def test_parse并发写入新hash时陈旧结果不软删新向量(self): + conn = _EmbeddingConnection( + candidate_payload={"embed_text": NEW_TEXT}, + draft_live_embeddings=[(701, NEW_HASH, None)], + ) + + with patch.object(embed.click, "echo"): + written = embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR) + + self.assertFalse(written) + self.assertEqual(conn.events, ["table_lock", "candidate_lock"]) + self.assertFalse(conn.draft_live_embeddings[0]["deleted"]) + + def test_entity旧活向量阻断写入且绝不软删(self): + conn = _EmbeddingConnection( + draft_live_embeddings=[(701, "old-hash", 9001)], + ) + + with self.assertRaisesRegex(embed.EmbeddingOwnershipConflict, "entity"): + embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR) + + self.assertEqual(conn.events, ["table_lock", "candidate_lock", "draft_vectors_lock"]) + self.assertFalse(conn.draft_live_embeddings[0]["deleted"]) + + def test其它hash的draft_owner旧活向量先软删再写入(self): + conn = _EmbeddingConnection( + draft_live_embeddings=[(701, "old-hash", None)], + ) + + written = embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR) + + self.assertTrue(written) + self.assertEqual( + conn.events, + ["table_lock", "candidate_lock", "draft_vectors_lock", "owner_lock", + "old_vectors_soft_delete", "upsert"], + ) + active = [row for row in conn.draft_live_embeddings if not row["deleted"]] + self.assertEqual([(row["content_hash"], row["entity_id"]) for row in active], + [(CONTENT_HASH, None)]) + + def test写前同hash当前活向量幂等跳过(self): + conn = _EmbeddingConnection( + draft_live_embeddings=[(701, CONTENT_HASH, None)], + ) + + with patch.object(embed.click, "echo"): + written = embed._write_embedding(conn, 101, CONTENT_HASH, TEXT, self.VECTOR) + + self.assertFalse(written) + self.assertEqual(conn.events, ["table_lock", "candidate_lock", "draft_vectors_lock"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.claude/skills/parse-book/SKILL.md b/.claude/skills/parse-book/SKILL.md index 6f26385..420350c 100644 --- a/.claude/skills/parse-book/SKILL.md +++ b/.claude/skills/parse-book/SKILL.md @@ -58,11 +58,21 @@ disable-model-invocation: true **窗行陷阱（放量首日实测）**：`--window` 参数变化后重切,旧窗行会按 from_order 占位,新的大窗被「已有大纲跳过」→ 中间章域永远漏出卡（验收期 1–3 章小窗占住 from_order=1,放量 1–34 章大窗被跳过）。**换窗参数重切前必须先删该书全部窗行**（窗行是可再生中间产物;卡挂「窗起」,cards 重出时按窗软删重出）。 -**作品面升格执行器 `scripts/parse_upgrade.py`（与上面范式拆书管线并行的另一条线,命令 `windows`/`run`/`status`）**：把参考书正文按窗抽成「会随剧情长大的实体卡」(升格卡,`source_type=upgrade_book`),设计见 `docs/2026-07-16-升格卡改造设计.md`。里程碑除真实章号外，模型输出必须临时携带所标章节正文短引；系统机械核验后删除证据，错章、缺证据或改写证据均拒收入库并留审计。模型给出的顶层出场章也必须由实体规范名或合法别名在对应章节正文中的实际出现机械证明；合法别名集合同时取卡内 payload 与独立 alias 表，单字规范名禁作章证据，规范名和别名均禁止通用称谓/关系称呼（如队长、舰长）作证据，常规二至四字专名保持精确子串命中。未命中章不得参与立卡、登场兜底或既有卡追加，无实证章时不补登场里程碑；既有卡每窗新增出场章必须审计完整旧值，redo/undo 精确恢复后再按新正文重算。显式 redo **仅允许当前 active 末窗**：执行前机械校验窗口号连续、章域合法且首尾相接，并校验目标窗逐章正文齐全且非空；任一失败必须在快照、undo 与任何写入前非零退出。历史窗修正必须全书前滚重建；后缀级联重算属于 P1，当前不支持。合法末窗 redo 仍须在撤销事务中清除所有 active `upgrade_book` 卡的本窗章域，保留窗外章并为历史无审计数据补旧值审计；redo 前建立完整恢复点（卡 payload/revision/deleted、别名、presence、水位、审计与窗状态），首次失败重新清理再试，最终失败完整恢复且重试初建卡软删。`--max-calls` 不得截断已经开始的即时重试，非显式 redo 不做该全局清理。`run` 默认**不发嵌入**;加 `--semantic-dedup` 开语义判重(治改名/跨型漏并)时,每窗按**读→算→写三段式短连接**跑——观察/近邻召回/M3 终判都在**无长连接**段发 LLM 与嵌入 HTTP(不再持窗级连接跨调用存活),写段只查预判结果落库;并**边抽边嵌**：本窗新建/更新卡在窗事务提交后增量嵌入落库(软删旧向量+upsert 新行),**后窗即可语义召回前窗刚长成的卡**,不再依赖"同书须预先全量 embed"。嵌段失败只告警、不牵连已 done 的窗。 +**作品面升格执行器 `scripts/parse_upgrade.py`（与上面范式拆书管线并行的另一条线,命令 `windows`/`run`/`status`）**：把参考书正文按窗抽成「会随剧情长大的实体卡」(升格卡,`source_type=upgrade_book`),设计见 `docs/2026-07-16-升格卡改造设计.md`。里程碑除真实章号外，模型输出必须临时携带所标章节正文短引；系统机械核验后删除证据，错章、缺证据或改写证据均拒收入库并留审计。模型给出的顶层出场章也必须由实体规范名或合法别名在对应章节正文中的实际出现机械证明；合法别名集合同时取卡内 payload 与独立 alias 表，单字规范名禁作章证据，规范名和别名均禁止通用称谓/关系称呼（如队长、舰长）作证据，常规二至四字专名保持精确子串命中。未命中章不得参与立卡、登场兜底或既有卡追加，无实证章时不补登场里程碑；机械登场兜底只允许用 debut 章正文存在性与实体名称/型构造中性台阶，禁止读取跨章摘要。既有卡每窗新增出场章必须审计完整旧值，redo/undo 精确恢复后再按新正文重算。显式 redo **仅允许当前 active 末窗**：执行前机械校验窗口号连续、章域合法且首尾相接，并校验目标窗逐章正文齐全且非空；任一失败必须在快照、undo 与任何写入前非零退出。历史窗修正必须全书前滚重建；后缀级联重算属于 P1，当前不支持。合法末窗 redo 仍须在撤销事务中清除所有 active `upgrade_book` 卡的本窗章域，保留窗外章并为历史无审计数据补旧值审计；redo 前建立完整恢复点（卡 payload/revision/deleted、别名、presence、水位、审计与窗状态），首次失败重新清理再试，最终失败完整恢复且重试初建卡软删。`--max-calls` 不得截断已经开始的即时重试，非显式 redo 不做该全局清理。`run` 默认**不发嵌入**;加 `--semantic-dedup` 开语义判重(治改名/跨型漏并)时,每窗按**读→算→写三段式短连接**跑——观察/近邻召回/M3 终判都在**无长连接**段发 LLM 与嵌入 HTTP(不再持窗级连接跨调用存活),写段只查预判结果落库;实体卡与关系卡的新建/更新都必须同步 state、逐字段审计并进入 touched；并**边抽边嵌**：本窗新建/更新卡在窗事务提交后增量嵌入落库(软删旧向量+upsert 新行),**后窗即可语义召回前窗刚长成的卡**,不再依赖"同书须预先全量 embed"。嵌段失败只告警、不牵连已 done 的窗。 + +**登场中性台阶名称边界**：名称只能取 debut 章正文唯一实际命中的合法规范名或别名；若只命中旧别名就用旧别名，多个合法名称同时命中、仅命中通用称谓或无法唯一确定时退化为“人物登场/物件登场”等仅类型描述，禁止泄漏未来才形成的规范名。`new_card` 规范化名称后必须把实际 canonical 与合法 aliases 立即登记到本窗判重索引，不能继续使用模型原始括号名。 + +**嵌入唯一键冲突失败关闭**：前述“嵌段失败只告警”仅指普通网络或服务异常；owner 预检必须在 `embed_texts` 前覆盖全部 todo hash。写连接第一条 SQL 必须按固定顺序以 `ROW EXCLUSIVE` 锁 `muse_knowledge_draft, example_knowledge_embedding`；随后按 did 锁 draft 行，机械确认 tenant、deleted=false、status=pending，并以当前 payload+MODEL 重算 hash 与 ready hash 一致，再按 did 锁全部活向量、按 hash 以 `FOR UPDATE` 二次校验 owner。若 HTTP 期间 payload 漂移，`(tenant_id, content_hash, model)` 唯一键被两个活跃 draft 的当前 payload 同时声明，唯一行 `entity_id` 非空，或同 draft 任一旧活向量已有 `entity_id`，必须在任何向量写入前硬停、把窗置 failed 并令命令非零；禁止迁移 entity owner、清空 `entity_id`、软删实体向量或打印窗口完成。仅当唯一行没有 entity owner 且命中旧软删 draft 时，才把行迁到当前 draft并刷新当前嵌入语义列；冲突更新及旧 hash 软删条件都必须保留 `entity_id IS NULL` 防并发竞态。 **同书命令互斥**：`parse_upgrade.py run/windows`、`reset_upgrade_work.py` 的预览/执行，以及 `backup_upgrade_work.py backup/rehearse/restore` 均须先取得 `scripts/upgrade_work_lock.py` 的同租户同作品 PostgreSQL session advisory lock；失败必须在任何业务 SQL、文件 verify/写入、嵌入或 LLM 调用前非零退出。锁由独立 autocommit 连接持有到命令结束，该连接只执行加锁/解锁 SQL；`status` 只读且不取锁。所有调用方必须导入同一个 `upgrade_work_lock(...)` context manager，禁止另造不兼容锁键。 -**升格 work reset 前备份（只读）**：任何 `reset_upgrade_work.py --execute` 前，先用单个 `REPEATABLE READ READ ONLY` 事务导出该作品七域状态；目录必须是 `/private/tmp` 下尚不存在的新目录。input 摘要在同一快照中绑定不含密码的数据库 identity、Canonical 章节/block 内容、active 窗边界、升格七型 active 字段合同，以及计划参数 `MiniMax-M3 + semantic-dedup=true`；同时绑定本次实际执行的 `backup_upgrade_work.py`、`parse_upgrade.py`、`llm.py` 和本 SKILL 的 fileSha。`gitCommit` 只记录 HEAD；当代码未提交时不得把它当执行代码身份，确认以 manifest 的 `inputSha + codeFiles + confirmationSha` 为准。SHA 仅用于发现传输、落盘或误操作造成的意外损坏，不宣称抵抗能同时改写工件和摘要的恶意篡改。命令默认不写数据库、不调用模型，生成后会关闭数据库连接并从磁盘独立复验；任一失败删除本次新建目录并非零退出。 +**升格 work reset 前备份（只读）**：任何 `reset_upgrade_work.py --execute` 前，先用单个 `REPEATABLE READ READ ONLY` 事务导出该作品七域状态；目录必须是 `/private/tmp` 下尚不存在的新目录。input 摘要在同一快照中绑定不含密码的数据库 identity、Canonical 章节/block 内容、active 窗边界、升格七型 active 字段合同，以及计划参数 `MiniMax-M3 + semantic-dedup=true`；同时绑定本次实际执行的 `backup_upgrade_work.py`、`reset_upgrade_work.py`、`parse_upgrade.py`、`parse_llm.py`、`embed_drafts.py`、`upgrade_work_lock.py`、`llm.py` 和本 SKILL 的 fileSha。`gitCommit` 只记录 HEAD；当代码未提交时不得把它当执行代码身份，确认以 manifest 的 `inputSha + codeFiles + confirmationSha` 为准。SHA 仅用于发现传输、落盘或误操作造成的意外损坏，不宣称抵抗能同时改写工件和摘要的恶意篡改。命令默认不写数据库、不调用模型，生成后会关闭数据库连接并从磁盘独立复验；任一失败删除本次新建目录并非零退出。 + +reset preview 不要求备份且只读；execute 必须同时提供 `--backup-dir`、与 manifest 精确匹配的 `--backup-id` 和 `--confirmation-sha`。取得统一同书 advisory lock 后，execute 须在同一业务事务内先以 `SHARE ROW EXCLUSIVE` 锁住 drafts/windows/aliases/presence/card_state/audits/embeddings 七表，再复用 `backup_upgrade_work` 的七域读取与摘要规则，将当前 `rowCount/primaryKeySha/contentSha` 和已离线复验 manifest 逐域 exact 比较；任一漂移必须在写入前失败关闭。 + +reset execute 还必须在同书 advisory lock 内、创建业务连接前重新调用 `capture_code_identity`：manifest `codeFiles` 必须包含 `reset_upgrade_work.py`，并与当前实际完整 `codeFiles` 逐项一致；reset/parse/embed/llm/skill 任一 fileSha 漂移或集合变化均拒绝。旧备份仍可离线 `verify`，但缺少 reset fileSha 时不得用于 execute。 + +目标 `upgrade_book` draft 的任一向量只要 `entity_id` 非空，execute 必须失败关闭；reset 只允许软删 `entity_id IS NULL` 的活向量，统一写入 `updater='upgrade-reset'`，禁止物删。向量范围不要求关联卡当前 active，以便幂等收口旧版 reset 遗留向量；其它作品、其它来源和其它租户不在边界内。提交前必须机械断言 active 升格卡为 0、active 窗全部 pending、alias/presence/state/audit 为 0、目标 draft 活向量为 0；任一失败回滚整个事务。七域备份/恢复合同不变。 ```bash # 创建 work 8 七域一致性备份；expected 值进入 input 摘要并机械核对 594 章/116 窗 diff --git a/.claude/skills/parse-book/scripts/backup_upgrade_work.py b/.claude/skills/parse-book/scripts/backup_upgrade_work.py index c7794bc..f3914ca 100644 --- a/.claude/skills/parse-book/scripts/backup_upgrade_work.py +++ b/.claude/skills/parse-book/scripts/backup_upgrade_work.py @@ -375,6 +375,7 @@ def capture_code_identity(git_commit: str | None = None) -> dict[str, Any]: root = _repository_root() files = { "backup_upgrade_work.py": pathlib.Path(__file__).resolve(), + "reset_upgrade_work.py": here / "reset_upgrade_work.py", "parse_upgrade.py": here / "parse_upgrade.py", "parse_llm.py": here / "parse_llm.py", "embed_drafts.py": here.parents[1] / "embed" / "scripts" / "embed_drafts.py", diff --git a/.claude/skills/parse-book/scripts/parse_upgrade.py b/.claude/skills/parse-book/scripts/parse_upgrade.py index d2824f8..b52f0dd 100644 --- a/.claude/skills/parse-book/scripts/parse_upgrade.py +++ b/.claude/skills/parse-book/scripts/parse_upgrade.py @@ -73,6 +73,19 @@ LIFECYCLE = ("登场", "成长", "高光", "退场", "结局") DEDUP_SIM_THRESHOLD = 0.60 CHAP_BIG = 10 ** 9 # 章号缺失/待人工的里程碑，排序时排到最后 STEP_MAX = 80 # 里程碑「台阶」机械字数上限（洞③）：提示词目标 ≤40 字，机械上限 80，双层防跑飞长文 +# 登场机械兜底只允许使用名称与型的中性描述，禁止把跨章摘要事实绑定到首章。 +DEBUT_TYPE_LABELS = { + "character": "人物", + "location": "地点", + "item": "物件", + "faction": "组织", + "power_system": "能力体系", + "event": "事件", +} + + +class EmbeddingOwnershipConflict(RuntimeError): + """同一嵌入唯一键被两个活跃 draft 的当前 payload 同时声明。""" # ── 库内合同（元数据驱动公理：prompt 与守卫同源，禁手写合同）── @@ -460,6 +473,10 @@ GENERIC_ADDRESS_ALIASES = frozenset({ "叔叔", "阿姨", "爷爷", "奶奶", }) TOP_LEVEL_APPEARANCE_AUDIT_FIELD = "顶层:出场章" +# 关系卡顶层字段与卡水位不在 payload["字段"] 内，必须用明确 sentinel 让 undo 精确落回原位置。 +TOP_LEVEL_RELATION_TYPE_AUDIT_FIELD = "顶层:关系类型" +TOP_LEVEL_RELATION_EVOLUTION_AUDIT_FIELD = "顶层:演变轨迹" +CARD_STATE_WATERMARK_AUDIT_FIELD = "状态:watermark_window" def _filter_entity_chapters(name, aliases, chaps, chapter_texts): @@ -541,6 +558,144 @@ def _append_verified_appearance_chapters( return True +def _audit_relation_change(conn, draft_id, win_no, field_name, old_value, new_value): + """记录关系卡一个真实落点的完整旧/新 JSON；值未变化时不制造空审计。""" + + if old_value == new_value: + return False + conn.execute( + """INSERT INTO example_upgrade_audit + (draft_id, window_no, field_name, old_value, new_value, tenant_id) + VALUES (%s,%s,%s,%s,%s,%s)""", + ( + draft_id, + win_no, + field_name, + json.dumps(old_value, ensure_ascii=False) if old_value is not None else None, + json.dumps(new_value, ensure_ascii=False) if new_value is not None else None, + TENANT, + ), + ) + return True + + +def _update_relation_card_state(conn, draft_id, work_id, win_no): + """推进关系卡水位并审计旧值；undo 据 sentinel 恢复旧水位或删除本窗新 state。""" + + row = conn.execute( + "SELECT watermark_window FROM example_upgrade_card_state WHERE draft_id=%s", + (draft_id,), + ).fetchone() + old_watermark = row[0] if row else None + _audit_relation_change( + conn, + draft_id, + win_no, + CARD_STATE_WATERMARK_AUDIT_FIELD, + old_watermark, + max(old_watermark or 0, win_no), + ) + conn.execute( + """INSERT INTO example_upgrade_card_state (draft_id, work_id, watermark_window, tenant_id) + VALUES (%s,%s,%s,%s) + ON CONFLICT (draft_id) DO UPDATE + SET watermark_window=GREATEST(example_upgrade_card_state.watermark_window, + EXCLUDED.watermark_window), + update_time=now()""", + (draft_id, work_id, win_no, TENANT), + ) + + +def _update_relation_card(conn, work_id, win_no, draft_id, payload, relation): + """按真实字段落点更新关系卡，并为顶层、逐字段、演变数组和水位分别留可撤销审计。""" + + payload = deepcopy(payload) + fields = payload.setdefault("字段", {}) + + # 历史数据可能把演变轨迹放在顶层；迁移本身也必须可撤销，不能只恢复字段侧。 + old_field_evolution = deepcopy(fields.get("演变轨迹")) if "演变轨迹" in fields else None + legacy_present = "演变轨迹" in payload + legacy_evolution = deepcopy(payload.get("演变轨迹")) if legacy_present else None + if legacy_present: + _audit_relation_change( + conn, + draft_id, + win_no, + TOP_LEVEL_RELATION_EVOLUTION_AUDIT_FIELD, + legacy_evolution, + None, + ) + payload.pop("演变轨迹", None) + + evolution = fields.get("演变轨迹") + evolution = list(evolution) if isinstance(evolution, list) else ([evolution] if evolution else []) + if legacy_present: + evolution.extend(legacy_evolution if isinstance(legacy_evolution, list) else [legacy_evolution]) + evolution_core = _strip_prefix(relation.get("本窗演变", "")) + if evolution_core and evolution_core not in {_strip_prefix(item) for item in evolution}: + evolution.append(f"[窗{win_no}] {evolution_core}") + if (legacy_present or evolution_core or "演变轨迹" in fields) \ + and old_field_evolution != evolution: + _audit_relation_change( + conn, draft_id, win_no, "演变轨迹", old_field_evolution, evolution + ) + fields["演变轨迹"] = evolution + + # 关系合同的其他字段逐字段覆写，每个字段都记录完整旧/新值，供即时重试精确还原。 + for field_name, new_value in (relation.get("其他字段") or {}).items(): + if field_name == "演变轨迹": + continue + old_value = deepcopy(fields.get(field_name)) if field_name in fields else None + if _audit_relation_change( + conn, draft_id, win_no, field_name, old_value, new_value): + fields[field_name] = new_value + + new_relation_type = relation.get("关系类型") + if new_relation_type: + old_relation_type = payload.get("关系类型") if "关系类型" in payload else None + if _audit_relation_change( + conn, + draft_id, + win_no, + TOP_LEVEL_RELATION_TYPE_AUDIT_FIELD, + old_relation_type, + new_relation_type): + payload["关系类型"] = new_relation_type + + conn.execute( + """UPDATE muse_knowledge_draft SET draft_payload=%s, + revision=revision+1, updater='upgrade' WHERE id=%s""", + (json.dumps(payload, ensure_ascii=False), draft_id), + ) + _update_relation_card_state(conn, draft_id, work_id, win_no) + return draft_id + + +def _insert_relation_card(conn, work_id, win_no, payload): + """新建关系卡并返回 draft id；逐字段审计和 state 任一步失败都会让窗事务回滚。""" + + draft_id = conn.execute( + """INSERT INTO muse_knowledge_draft + (work_id, draft_type, draft_payload, status, source_type, + source_id, creator, updater, tenant_id) + VALUES (%s,'entity',%s,'pending',%s,%s,'upgrade','upgrade',%s) + RETURNING id""", + (work_id, json.dumps(payload, ensure_ascii=False), SOURCE_TYPE, work_id, TENANT), + ).fetchone()[0] + _audit_relation_change( + conn, + draft_id, + win_no, + TOP_LEVEL_RELATION_TYPE_AUDIT_FIELD, + None, + payload.get("关系类型"), + ) + for field_name, value in (payload.get("字段") or {}).items(): + _audit_relation_change(conn, draft_id, win_no, field_name, None, value) + _update_relation_card_state(conn, draft_id, work_id, win_no) + return draft_id + + def _milestone_evidence_matches(item, chapter_texts): """验证短原文证据确实存在于里程碑声明的章节正文。""" @@ -704,11 +859,31 @@ def _merge_material(ent, rest_limit=600): return "；".join(seg) -def _debut_milestone(milestones, brief, chaps, win_no): +def _debut_evidence_name(entity_name, aliases, chapter_text): + """从 debut 正文选择唯一实际命中的合法名称；歧义或无命中时返回 None。""" + + canonical = str(entity_name or "").strip() + candidates = [] + if len(canonical) >= 2 and canonical not in GENERIC_ADDRESS_ALIASES: + candidates.append(canonical) + for raw_alias in aliases or []: + alias = _clean_alias(raw_alias) + if alias and alias not in GENERIC_ADDRESS_ALIASES and alias not in candidates: + candidates.append(alias) + normalized_text = unicodedata.normalize("NFKC", str(chapter_text or "")) + matched = [ + name for name in candidates + if unicodedata.normalize("NFKC", name) in normalized_text + ] + return matched[0] if len(matched) == 1 else None + + +def _debut_milestone(milestones, entity_name, entity_type, aliases, chaps, win_no, chapter_texts): """登场兜底（洞② 机械那一保险，纯函数便于离线自测）：里程碑列表里若没有任何 周期=登场 的条目， 在头部补一条兜底登场里程碑；已有登场则原样返回（提示词软约束 + 此机械兜底=双保险）。 - 章：min(正文实证出场章)（仅取整数章号）；无实证章则不补，真实性优先； - - 台阶："登场："+一句话摘要截 40 字，再过 STEP_MAX 守卫（与 _clean_milestone 同上限）； + - 台阶：名称只可取 debut 正文唯一命中的合法规范名/别名；无唯一命中则退化为仅类型； + - 证据：debut 章必须存在非空正文；缺正文时不补，禁止把后续章事实倒灌到最早章； - _win：盖当前窗号——带 _win 才能被同窗撤销（undo_window 按 _win 删本窗新增）识别，防重跑 double-append。 根治病象：实体首现走新名字路径时模型倾向只填当前态、漏建登场里程碑，机械补一条保成长线起点不缺。""" if any(isinstance(m, dict) and m.get("周期") == "登场" for m in (milestones or [])): @@ -716,8 +891,20 @@ def _debut_milestone(milestones, brief, chaps, win_no): ch_ints = sorted(_int_chaps(chaps)) if not ch_ints: return list(milestones or []) - debut = {"台阶": ("登场：" + (brief or "")[:40])[:STEP_MAX], "周期": "登场", "_win": win_no} - debut["章"] = ch_ints[0] + debut_chapter = ch_ints[0] + if not isinstance(chapter_texts, dict) or not str(chapter_texts.get(debut_chapter) or "").strip(): + return list(milestones or []) + name = str(entity_name or "").strip() + label = DEBUT_TYPE_LABELS.get(entity_type) + if not name or not label: + return list(milestones or []) + evidence_name = _debut_evidence_name(name, aliases, chapter_texts[debut_chapter]) + debut = { + "章": debut_chapter, + "台阶": (f"{label}「{evidence_name}」登场" if evidence_name else f"{label}登场")[:STEP_MAX], + "周期": "登场", + "_win": win_no, + } return [debut] + list(milestones or []) @@ -962,7 +1149,8 @@ def new_card( # 模型倾向只填当前态、漏建登场，此为「提示词硬约束 + 机械兜底」双保险里的机械那一保险。 if milestone_types and ent.get("型") in milestone_types: fields0["演变历程"] = _debut_milestone( - fields0.get("演变历程") or [], ent.get("一句话摘要", ""), chaps_int, win_no) + fields0.get("演变历程") or [], raw, ent.get("型"), aliases, + chaps_int, win_no, chapter_texts) payload = {"type": ent["型"], "名称": raw, "别名": aliases, "一句话摘要": ent.get("一句话摘要", ""), @@ -997,7 +1185,7 @@ def new_card( """INSERT INTO example_upgrade_card_state (draft_id, work_id, watermark_window, tenant_id) VALUES (%s,%s,%s,%s) ON CONFLICT (draft_id) DO NOTHING""", (did, work_id, win_no, TENANT)) - return did + return did, raw, tuple(payload["别名"]) def _snapshot_redo_window(conn, work_id, win_no): @@ -1145,6 +1333,21 @@ def undo_window(conn, work_id, win_no, *, from_chapter=None, to_chapter=None): WHERE a.tenant_id=%s AND d.work_id=%s AND a.window_no=%s ORDER BY a.id DESC""", (TENANT, work_id, win_no)).fetchall() for did, fname, old in rows: + if fname == CARD_STATE_WATERMARK_AUDIT_FIELD: + # 水位是独立表状态，不得误还原进 payload["字段"]；新关系旧值为空时直接删 state。 + if old is None: + conn.execute("DELETE FROM example_upgrade_card_state WHERE draft_id=%s", (did,)) + else: + watermark = json.loads(old) if isinstance(old, str) else old + conn.execute( + """INSERT INTO example_upgrade_card_state + (draft_id, work_id, watermark_window, tenant_id) + VALUES (%s,%s,%s,%s) + ON CONFLICT (draft_id) DO UPDATE + SET watermark_window=EXCLUDED.watermark_window, update_time=now()""", + (did, work_id, watermark, TENANT), + ) + continue payload = conn.execute("SELECT draft_payload FROM muse_knowledge_draft WHERE id=%s", (did,)).fetchone()[0] if fname == TOP_LEVEL_APPEARANCE_AUDIT_FIELD: @@ -1153,6 +1356,18 @@ def undo_window(conn, work_id, win_no, *, from_chapter=None, to_chapter=None): payload.pop("出场章", None) else: payload["出场章"] = json.loads(old) if isinstance(old, str) else old + elif fname == TOP_LEVEL_RELATION_TYPE_AUDIT_FIELD: + # 关系类型住 payload 顶层；普通字段恢复会错误写到 payload["字段"]。 + if old is None: + payload.pop("关系类型", None) + else: + payload["关系类型"] = json.loads(old) if isinstance(old, str) else old + elif fname == TOP_LEVEL_RELATION_EVOLUTION_AUDIT_FIELD: + # 兼容历史关系卡顶层演变轨迹迁移，undo 时恢复到原顶层位置。 + if old is None: + payload.pop("演变轨迹", None) + else: + payload["演变轨迹"] = json.loads(old) if isinstance(old, str) else old elif old is None: payload.get("字段", {}).pop(fname, None) else: @@ -1249,6 +1464,30 @@ def _classify_new_name(ent, name_map, presence): return "presence", None # 单章龙套 → 留档 +def _resolve_canonical_name(name_map, aliases_by_draft, matched_name): + """把命中的规范名或别名解析为该 draft 当前真实规范名。 + + ``name_map`` 同时以规范名和别名建索引，不能把命中的键直接当 ``canonical_name`` 落库。 + 真实规范名必须是同 draft 下不属于合法别名集合的活跃卡名称；若底册不完整到无法唯一解析， + 直接失败关闭当前窗，避免写入 alias token 后仍把窗口标成完成。""" + + target = name_map.get(matched_name) + if not target: + raise RuntimeError(f"判重命中项不在底册：{matched_name}") + draft_id = target[0] + aliases = set(aliases_by_draft.get(draft_id, set())) + canonical_names = [ + name + for name, value in name_map.items() + if value[0] == draft_id and name not in aliases + ] + if len(canonical_names) != 1: + raise RuntimeError( + f"draft={draft_id} 无法唯一解析真实规范名：候选={canonical_names}，命中={matched_name}" + ) + return canonical_names[0] + + # ── 语义判重（P1，设计稿 §8.2）：打开 v6 已设计、暂时关着的「嵌入近邻 + M3 终判」那级 ── # 治病根 4：机械判重只比名字字符串，改名（"影杀者"→"IV代纯机械机甲·影杀者"）/跨型指代就漏并。 # 默认关（--semantic-dedup 开启）：连接三段式（洞①）——嵌入/召回/终判在预判段(prejudge_semantic)无长 @@ -1385,16 +1624,92 @@ def prejudge_semantic(obs, name_map, presence, embed_sess, work_id, call): return verdicts, merge_n, chain_n +def _assert_embedding_owner_available(conn, draft_id, content_hash, *, lock=False): + """校验同 hash 唯一行可安全归当前 draft；写段可加行锁防预检后的并发竞态。""" + + lock_clause = " FOR UPDATE OF e" if lock else "" + conflict = conn.execute( + """SELECT e.draft_id, e.entity_id, e.deleted, + COALESCE(d.deleted, TRUE), d.draft_payload + FROM example_knowledge_embedding e + LEFT JOIN muse_knowledge_draft d ON d.id=e.draft_id + WHERE e.tenant_id=%s AND e.content_hash=%s AND e.model=%s""" + lock_clause, + (TENANT, content_hash, EMBED_MODEL), + ).fetchone() + if not conflict: + return + owner_draft_id, owner_entity_id, _, owner_deleted, owner_payload = conflict + # 已确认实体拥有的向量不可重新降级归 draft；无论 draft 是否软删都必须失败关闭。 + if owner_entity_id is not None: + raise EmbeddingOwnershipConflict( + f"同 hash 唯一行已归 entity：hash={content_hash}，entity={owner_entity_id}，" + f"candidate={draft_id}" + ) + if owner_draft_id in (None, draft_id) or owner_deleted: + return + owner_text = build_embed_text(owner_payload or {}) + owner_hash = hashlib.sha256(f"{owner_text}|{EMBED_MODEL}".encode()).hexdigest() + if owner_hash == content_hash: + raise EmbeddingOwnershipConflict( + f"活跃 draft 同 hash 争用：hash={content_hash}，" + f"owner={owner_draft_id}，candidate={draft_id}" + ) + + +def _assert_draft_live_embeddings_mutable(conn, draft_id): + """写前锁定同 draft 全部活向量；任一已归 entity 都禁止被 draft 更新链软删。""" + + rows = conn.execute( + """SELECT id, content_hash, entity_id FROM example_knowledge_embedding + WHERE tenant_id=%s AND draft_id=%s AND deleted=FALSE + FOR UPDATE""", + (TENANT, draft_id), + ).fetchall() + entity_rows = [(row_id, content_hash, entity_id) for row_id, content_hash, entity_id in rows + if entity_id is not None] + if entity_rows: + raise EmbeddingOwnershipConflict( + f"draft={draft_id} 的旧活向量已归 entity，禁止软删：{entity_rows}" + ) + + +def _assert_ready_draft_current(conn, draft_id, expected_hash): + """锁定待写 draft，复验可信状态与当前 payload hash，阻断嵌入 HTTP 期间漂移。""" + + row = conn.execute( + """SELECT tenant_id, deleted, status, draft_payload + FROM muse_knowledge_draft WHERE id=%s FOR UPDATE""", + (draft_id,), + ).fetchone() + if not row: + raise EmbeddingOwnershipConflict(f"待写 draft 不存在：draft={draft_id}") + tenant_id, deleted, status, payload = row + if tenant_id != TENANT or deleted or status != "pending": + raise EmbeddingOwnershipConflict( + f"待写 draft 状态非法：draft={draft_id}，tenant={tenant_id}，" + f"deleted={deleted}，status={status}" + ) + current_text = build_embed_text(payload or {}) + current_hash = hashlib.sha256(f"{current_text}|{EMBED_MODEL}".encode()).hexdigest() + if current_hash != expected_hash: + raise EmbeddingOwnershipConflict( + f"HTTP 期间 draft payload 已漂移：draft={draft_id}，" + f"expected_hash={expected_hash}，current_hash={current_hash}" + ) + + def embed_touched_cards(sess, work_id, touched): """嵌段（洞① 三段式连接）：窗事务 commit **之后**，把本窗新建/更新的卡增量嵌入落库， 让后窗语义判重能召回前窗刚长成的卡（治「起了对不上正文的名→机械预扫认不出→成长线静默断」）。 读（短连接）：取 touched 卡最新 payload + 各卡现存活嵌入行的 content_hash 集；关连接。 算（无连接）：build_embed_text 构文（与检索端同源同构）→ sha256(text|MODEL) 同 embed_drafts 公式 → 同哈希已有活行的卡跳过（内容未变，幂等）→ 批量 embed_texts。 - 写（短连接单事务）：该卡其他活行 UPDATE deleted=TRUE（软删红线，绝不物理 DELETE）→ 新行 upsert - （ON CONFLICT DO UPDATE deleted=FALSE：内容回退到旧版本时复活软删旧行）。 - 失败语义：本段任何异常只 stderr 告警、**不 fail 窗**（窗已 commit done）——判重是增益非必需， - 向量缺口可由后续批量补嵌（embed skill）兜住。返回本窗实际新嵌条数。""" + 写（短连接单事务）：固定顺序锁表后复验 draft 状态与当前 payload hash，再锁定同 draft + 全部活向量与同 hash 唯一行，拒绝任何 entity owner； + 再软删该卡其他仍属 draft 的旧活行； + upsert 会把旧软删卡占用的 hash 迁到当前 draft，并刷新当前嵌入全部语义列。 + 失败语义：普通网络/服务异常只 stderr 告警、不 fail 已提交窗；确定性的活跃 owner 冲突向 caller + 重抛，由 caller 把窗改为 failed 并非零退出，确保下次 run 会 undo 后重跑。返回本窗实际新嵌条数。""" try: # 读段：短连接取 payload + 各卡现存活嵌入哈希，读完即关 payloads, live_hashes = {}, {} @@ -1418,31 +1733,86 @@ def embed_touched_cards(sess, work_id, touched): todo.append((did, text, h)) if not todo: return 0 + # owner 预检必须覆盖全部 todo，放在 embed_texts 之前；即使服务随后把候选标 bad， + # entity/活跃 draft 冲突也已经硬停，不能因 ready 为空而返回假成功。 + with psycopg.connect(DSN) as conn: + for did, _, content_hash in todo: + _assert_embedding_owner_available(conn, did, content_hash) + # 唯一键是 tenant+content_hash+model；同批两个活跃 draft 文本完全相同时，数据库无法同时表达 + # 两个 owner。必须在发嵌入和写库前失败关闭，禁止后处理者按循环顺序抢走前者的唯一行。 + hash_owners = {} + for did, _, content_hash in todo: + hash_owners.setdefault(content_hash, []).append(did) + duplicate_hashes = { + content_hash: draft_ids + for content_hash, draft_ids in hash_owners.items() + if len(draft_ids) > 1 + } + if duplicate_hashes: + raise EmbeddingOwnershipConflict(f"活跃 touched 卡出现同 hash 争用：{duplicate_hashes}") vecs, bad = embed_texts(sess, [t for _, t, _ in todo]) - # 写段：短连接单事务——淘汰旧活行（软删）+ upsert 新行 + ready = [ + (did, text, content_hash, vecs[index]) + for index, (did, text, content_hash) in enumerate(todo) + if index not in bad and index < len(vecs) and vecs[index] is not None + ] + if not ready: + return 0 + # 写段：短连接单事务——先锁定并校验唯一 owner，再淘汰旧活行（软删）+ upsert 新行。 done = 0 with psycopg.connect(DSN) as conn: - for j, (did, text, h) in enumerate(todo): - if j in bad or j >= len(vecs) or vecs[j] is None: - continue # 嵌入失败的卡跳过（向量缺口由后续批量补嵌兜住） + # 写事务先按固定表顺序取得 ROW EXCLUSIVE 锁，再执行任何行锁或写入。 + conn.execute( + "LOCK TABLE muse_knowledge_draft, example_knowledge_embedding IN ROW EXCLUSIVE MODE" + ) + for did, _, content_hash, _ in sorted(ready, key=lambda item: item[0]): + _assert_ready_draft_current(conn, did, content_hash) + for did in sorted({draft_id for draft_id, _, _, _ in ready}): + _assert_draft_live_embeddings_mutable(conn, did) + for did, _, content_hash, _ in sorted(ready, key=lambda item: (item[2], item[0])): + _assert_embedding_owner_available(conn, did, content_hash, lock=True) + for did, text, h, vector in ready: # 该卡其他活行软删（内容已变、旧向量过期）——软删红线：绝不物理 DELETE conn.execute( """UPDATE example_knowledge_embedding SET deleted=TRUE, updater=%s - WHERE tenant_id=%s AND draft_id=%s AND deleted=FALSE AND content_hash!=%s""", + WHERE tenant_id=%s AND draft_id=%s AND deleted=FALSE + AND entity_id IS NULL AND content_hash!=%s""", (EMBED_ACTOR, TENANT, did, h)) - # 新行 upsert：内容回退到旧版本（content_hash 命中软删旧行）时复活，列结构照抄 embed_drafts - conn.execute( + # 同 hash 旧行可能仍绑定全重抽前的软删 draft；冲突时必须迁移 owner 并刷新全部当前语义列。 + upserted = conn.execute( """INSERT INTO example_knowledge_embedding (draft_id, content_hash, embed_text, model, dimensions, embedding, creator, updater, tenant_id) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (tenant_id, content_hash, model) - DO UPDATE SET deleted=FALSE, updater=EXCLUDED.updater""", - (did, h, text, EMBED_MODEL, EMBED_DIM, json.dumps(vecs[j]), - EMBED_ACTOR, EMBED_ACTOR, TENANT)) + DO UPDATE SET draft_id=EXCLUDED.draft_id, + embed_text=EXCLUDED.embed_text, + model=EXCLUDED.model, + dimensions=EXCLUDED.dimensions, + embedding=EXCLUDED.embedding, + deleted=FALSE, + updater=EXCLUDED.updater + WHERE example_knowledge_embedding.entity_id IS NULL + AND (example_knowledge_embedding.draft_id=EXCLUDED.draft_id + OR example_knowledge_embedding.deleted=TRUE + OR NOT EXISTS ( + SELECT 1 FROM muse_knowledge_draft owner + WHERE owner.id=example_knowledge_embedding.draft_id + AND owner.deleted=FALSE)) + RETURNING draft_id""", + (did, h, text, EMBED_MODEL, EMBED_DIM, json.dumps(vector), + EMBED_ACTOR, EMBED_ACTOR, TENANT)).fetchone() + if not upserted or upserted[0] != did: + raise EmbeddingOwnershipConflict( + f"同 hash 唯一行未绑定当前 draft：hash={h}，candidate={did}" + ) done += 1 conn.commit() return done + except EmbeddingOwnershipConflict as e: + # 这是确定性数据所有权冲突，不是可降级的网络抖动；必须令命令非零且禁止打印窗完成假绿。 + print(f"[嵌段阻断] work={work_id} {e}", file=sys.stderr) + raise except Exception as e: # 嵌段失败不牵连窗（窗已 commit done）：判重是增益、向量缺口后续批量补嵌可兜，只告警可审计 print(f"[嵌段告警] work={work_id} 增量嵌入异常（不 fail 窗，向量缺口后续补嵌兜住）: " @@ -1621,6 +1991,7 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): continue if kind == "alias": # 疑似别名：初卡转观察材料（G3；洞②材料不截里程碑） did = name_map[key][0] + canonical_name = _resolve_canonical_name(name_map, aliases_by_draft, key) to_update.setdefault(did, []).append( f"（别名「{nm}」并入）初卡材料：{_merge_material(ent)}") conn.execute( @@ -1628,7 +1999,7 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): (work_id, canonical_name, alias, evidence_window, verdict_by, tenant_id) VALUES (%s,%s,%s,%s,'ai',%s) ON CONFLICT (tenant_id, work_id, alias) DO NOTHING""", - (work_id, key, nm, win_no, TENANT)) + (work_id, canonical_name, nm, win_no, TENANT)) legal_alias = _clean_alias(nm) if legal_alias: aliases_by_draft.setdefault(did, set()).add(legal_alias) @@ -1636,6 +2007,7 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): if kind == "substr": # 同型名称互为子串（「果子」vs「开心果子」）： # 不直接立卡，转观察材料并入既有卡由更新步 AI 甄别（软防护，留复核标记；洞②材料不截里程碑） did = name_map[key][0] + canonical_name = _resolve_canonical_name(name_map, aliases_by_draft, key) to_update.setdefault(did, []).append( f"（名称疑似同一实体「{nm}」≈「{key}」，请甄别后再并入）" f"初卡材料：{_merge_material(ent)}") @@ -1644,7 +2016,7 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): (work_id, canonical_name, alias, evidence_window, verdict_by, tenant_id) VALUES (%s,%s,%s,%s,'substr',%s) ON CONFLICT (tenant_id, work_id, alias) DO NOTHING""", - (work_id, key, nm, win_no, TENANT)) + (work_id, canonical_name, nm, win_no, TENANT)) legal_alias = _clean_alias(nm) if legal_alias: aliases_by_draft.setdefault(did, set()).add(legal_alias) @@ -1672,7 +2044,7 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): aliases_by_draft.setdefault(did0, set()).add(legal_alias) continue if verdict == "chain": # 前身后继：仍立卡，串链关系记候选审计 - did = new_card( + did, canonical_name, card_aliases = new_card( conn, work_id, win_no, @@ -1682,7 +2054,11 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): known_chapters=hist, ) new_ids.append(did) - name_map[nm] = (did, ent.get("型", ""), ent.get("一句话摘要", "")) + target = (did, ent.get("型", ""), ent.get("一句话摘要", "")) + name_map[canonical_name] = target + for card_alias in card_aliases: + name_map[card_alias] = target + aliases_by_draft.setdefault(did, set()).add(card_alias) # 只记候选提示、不自动写「前身/后继」字段——避免误串，链接由人工/后续确认落字段 for did2, rel, nm2 in vd: conn.execute( @@ -1693,7 +2069,7 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): json.dumps({"对方卡号": did2, "对方名称": nm2}, ensure_ascii=False), TENANT)) continue - did = new_card( + did, canonical_name, card_aliases = new_card( conn, work_id, win_no, @@ -1703,7 +2079,11 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): known_chapters=hist, ) new_ids.append(did) - name_map[nm] = (did, ent.get("型", ""), ent.get("一句话摘要", "")) + target = (did, ent.get("型", ""), ent.get("一句话摘要", "")) + name_map[canonical_name] = target + for card_alias in card_aliases: + name_map[card_alias] = target + aliases_by_draft.setdefault(did, set()).add(card_alias) else: # 单章龙套 → 留档（G4）；无正文实证章则不造留档 for ch in chaps: conn.execute( @@ -1761,6 +2141,7 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): appearance_chapters=pres_add.pop(u["draft_id"], set()), known_aliases=aliases_by_draft.get(u["draft_id"], set())) # ⑤ 关系增量（核心角色=本窗有更新的 character + 在场 character，≤8） + relation_touched = set() char_cards = [] seen = set() for did in list(to_update.keys()): @@ -1795,32 +2176,10 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): key = tuple(sorted((ja, yi))) evo_core = _strip_prefix(r.get("本窗演变", "")) if key in exist: - # 落点统一（抽检#3）：其他字段与演变轨迹全并进 rp["字段"]， - # 顶层遗留的演变轨迹迁移进字段后删除，避免双份分裂 rid, rp = exist[key] - f2 = rp.setdefault("字段", {}) - if "演变轨迹" in rp: # 存量顶层迁移 - legacy = rp.pop("演变轨迹") - base = f2.get("演变轨迹") or [] - f2["演变轨迹"] = (base if isinstance(base, list) else [base]) + \ - (legacy if isinstance(legacy, list) else [legacy]) - for k2, v2 in (r.get("其他字段") or {}).items(): - if k2 == "演变轨迹": - continue # 演变只走本窗演变通道 - f2[k2] = v2 # 覆写（当前状态等保最新） - if evo_core: - lst = f2.setdefault("演变轨迹", []) - if not isinstance(lst, list): - lst = [lst] - f2["演变轨迹"] = lst - if evo_core not in {_strip_prefix(x) for x in lst}: - lst.append(f"[窗{win_no}] {evo_core}") - if r.get("关系类型"): - rp["关系类型"] = r["关系类型"] - conn.execute( - """UPDATE muse_knowledge_draft SET draft_payload=%s, - revision=revision+1 WHERE id=%s""", - (json.dumps(rp, ensure_ascii=False), rid)) + relation_touched.add( + _update_relation_card(conn, work_id, win_no, rid, rp, r) + ) else: f2 = dict(r.get("其他字段") or {}) f2["演变轨迹"] = [f"[窗{win_no}] {evo_core}"] if evo_core else [] @@ -1833,13 +2192,9 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): "来源": f"升格@窗{win_no}", "状态": "草稿", "目标库": "本书作品库", "可见范围": "本书私有", "_work_id": work_id} - conn.execute( - """INSERT INTO muse_knowledge_draft - (work_id, draft_type, draft_payload, status, source_type, - source_id, creator, updater, tenant_id) - VALUES (%s,'entity',%s,'pending',%s,%s,'upgrade','upgrade',%s)""", - (work_id, json.dumps(rp, ensure_ascii=False), SOURCE_TYPE, - work_id, TENANT)) + relation_touched.add( + _insert_relation_card(conn, work_id, win_no, rp) + ) # ⑥ 机械收尾：出场章统一并入（纯出场 + 被更新卡，抽检#2）+ 窗置 done for it in obs.get("纯出场", []): nm = (it.get("名称") or "").strip() @@ -1871,9 +2226,8 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): updater='upgrade' WHERE tenant_id=%s AND work_id=%s AND window_no=%s""", (TENANT, work_id, win_no)) conn.commit() - # 写段成功：收集本窗触达卡（洞①：new_card 新 id + to_update 键）供嵌段增量嵌入；关系卡走 - # 独立 INSERT/UPDATE 不经这两处、天然不嵌，由后续批量补嵌（embed skill）覆盖。 - touched = set(new_ids) | set(to_update.keys()) + # 写段成功：实体与关系的新建/更新 id 一并进入提交后嵌段，确保当前 payload 只有一个活向量。 + touched = set(new_ids) | set(to_update.keys()) | relation_touched except SensitiveHardStop as e: with psycopg.connect(DSN) as conn: if redo_snapshot is not None: @@ -1923,11 +2277,22 @@ def _run(work_id, max_windows, max_calls, model, redo_window, semantic_on): ) return # ── 嵌段（仅 semantic_on 且 touched 非空；窗 commit 之后；三段式）+ 窗完成汇报 ── - # 成功路径放在 try 之外：绝不触发上面的 failed/undo（窗已 commit done）；嵌段内部吞异常、 - # 只告警，不会把已 done 的窗拖回 failed（判重是增益、向量缺口后续批量补嵌可兜）。 + # 普通服务异常仍由嵌段告警降级；确定性的唯一 owner 冲突必须把已提交窗补偿标为 failed， + # 让下次 run 先 undo 后重跑，并在打印窗完成勾号前非零退出。 embed_n = 0 if semantic_on and touched: - embed_n = embed_touched_cards(embed_sess, work_id, touched) + try: + embed_n = embed_touched_cards(embed_sess, work_id, touched) + except EmbeddingOwnershipConflict as exc: + error_message = f"嵌入唯一 owner 冲突：{exc}"[:500] + with psycopg.connect(DSN) as conn: + conn.execute( + """UPDATE example_upgrade_window SET status='failed', error_message=%s, + updater='upgrade' WHERE tenant_id=%s AND work_id=%s AND window_no=%s""", + (error_message, TENANT, work_id, win_no), + ) + conn.commit() + raise click.ClickException(error_message) from exc new_n = len(obs.get('新名字', [])) extra = f" 判重:并{dedup_merge_n}/链{dedup_chain_n} 嵌入:{embed_n}" if semantic_on else "" click.echo(f" 窗{win_no}✓ ({a}-{b}章) 在场{len(onstage)} 新名字{new_n} " diff --git a/.claude/skills/parse-book/scripts/reset_upgrade_work.py b/.claude/skills/parse-book/scripts/reset_upgrade_work.py index 7f72ef0..f3d5141 100644 --- a/.claude/skills/parse-book/scripts/reset_upgrade_work.py +++ b/.claude/skills/parse-book/scripts/reset_upgrade_work.py @@ -14,33 +14,108 @@ · example_upgrade_card_state → 硬删该书行（卡水位挂旧卡 draft_id，新卡新水位） · example_upgrade_audit → 硬删该书行——undo_window 的还原 JOIN **不过滤 d.deleted**， 留着旧审计行，重抽期间任何同窗撤销都会把旧代软删卡错还原（污染） - · example_knowledge_embedding → **不动**：旧卡向量行靠判重召回 SQL 的 JOIN d.deleted=FALSE 天然排除， - 软删状态保留可回溯；重抽边抽边嵌写新卡向量 + · example_knowledge_embedding → **软删**该书升格卡的全部活向量（不物删，保留恢复能力）；同时覆盖 + 上次旧版 reset 遗留在已软删卡上的活向量，避免 HNSW 候选持续膨胀 用法： reset_upgrade_work.py --work-id N 预览（只读统计，不写库） - reset_upgrade_work.py --work-id N --execute 真清（单事务，全清或全不清） + reset_upgrade_work.py --work-id N --execute --backup-dir DIR --backup-id ID \ + --confirmation-sha SHA 真清（锁内验备份，单事务全清或全不清） """ import sys import pathlib import click import psycopg +from psycopg.rows import dict_row # 复用管线的连接与租户常量（与 parse_upgrade 同源） sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import backup_upgrade_work as backup # noqa: E402 from parse_llm import DSN, TENANT # noqa: E402 from upgrade_work_lock import UpgradeWorkLockUnavailable, upgrade_work_lock # noqa: E402 SOURCE_TYPE = "upgrade_book" +# execute 在读取备份快照前锁住七域表，阻断未接统一 advisory lock 的 embed/confirm 并发写入。 +RESET_TABLE_LOCK_SQL = """LOCK TABLE + muse_knowledge_draft, + example_upgrade_window, + example_upgrade_alias, + example_upgrade_presence, + example_upgrade_card_state, + example_upgrade_audit, + example_knowledge_embedding + IN SHARE ROW EXCLUSIVE MODE""" -def _reset(work_id, execute): + +def _assert_snapshot_matches_manifest(manifest, current_domains): + """复用备份模块摘要规则，逐域核对当前事务快照与已验证 manifest。""" + + for name, spec in backup.DOMAIN_SPECS.items(): + rows = backup.sort_rows(name, current_domains[name]) + actual = { + "rowCount": len(rows), + "primaryKeySha": backup._primary_key_sha(spec, rows), + "contentSha": backup._content_sha(rows), + } + expected = manifest["domains"][name] + for field, value in actual.items(): + if expected.get(field) != value: + raise click.ClickException( + f"备份七域快照不一致：{name}.{field} 已漂移，拒绝 reset" + ) + + +def _require_execute_backup(backup_dir, backup_id, confirmation_sha): + """execute 的三项备份确认缺一不可；preview 不受此约束。""" + + supplied = { + "--backup-dir": backup_dir, + "--backup-id": backup_id, + "--confirmation-sha": confirmation_sha, + } + missing = [option for option, value in supplied.items() if not value] + if missing: + raise click.ClickException( + f"--execute 必须提供备份确认参数：{', '.join(missing)}" + ) + + +def _assert_code_identity_matches_manifest(manifest, current_identity): + """要求 execute 使用的完整代码 fileSha 与 confirmation 绑定值逐项一致。""" + + manifest_files = manifest.get("input", {}).get("codeFiles") + if not isinstance(manifest_files, dict) or "reset_upgrade_work.py" not in manifest_files: + raise click.ClickException( + "备份 manifest.input.codeFiles 缺少 reset_upgrade_work.py，拒绝 reset" + ) + current_files = current_identity.get("codeFiles") + if not isinstance(current_files, dict): + raise click.ClickException("无法读取当前完整 codeFiles 身份，拒绝 reset") + if manifest_files != current_files: + changed = sorted( + name for name in set(manifest_files) | set(current_files) + if manifest_files.get(name) != current_files.get(name) + ) + raise click.ClickException( + f"当前代码 fileSha 与备份不一致：{', '.join(changed)}" + ) + + +def _reset(work_id, execute, manifest=None): """在调用方已持有同书锁时预览或执行重抽清理。""" with psycopg.connect(DSN) as conn: - title = conn.execute("SELECT title FROM muse_content_work WHERE id=%s", - (work_id,)).fetchone()[0] + if execute: + # 表锁、当前七域快照和后续 reset 全部位于同一事务，摘要核对后不存在写竞态窗口。 + conn.execute(RESET_TABLE_LOCK_SQL) + with conn.cursor(row_factory=dict_row) as cursor: + title, current_domains = backup._read_snapshot(cursor, work_id, TENANT) + _assert_snapshot_matches_manifest(manifest, current_domains) + else: + title = conn.execute("SELECT title FROM muse_content_work WHERE id=%s", + (work_id,)).fetchone()[0] # 预览统计：各表将被处理的行数 cards = conn.execute( """SELECT count(*) FROM muse_knowledge_draft @@ -60,19 +135,52 @@ def _reset(work_id, execute): aud = conn.execute( """SELECT count(*) FROM example_upgrade_audit a WHERE a.tenant_id=%s AND a.draft_id IN - (SELECT id FROM muse_knowledge_draft WHERE work_id=%s AND source_type=%s)""", - (TENANT, work_id, SOURCE_TYPE)).fetchone()[0] + (SELECT id FROM muse_knowledge_draft + WHERE tenant_id=%s AND work_id=%s AND source_type=%s)""", + (TENANT, TENANT, work_id, SOURCE_TYPE)).fetchone()[0] + # 不限制 d.deleted：除本轮活卡外，也要收口旧版 reset 已软删卡遗留的活向量。 + embeddings = conn.execute( + """SELECT count(*) FROM example_knowledge_embedding e + WHERE e.tenant_id=%s AND e.deleted=FALSE AND e.entity_id IS NULL AND EXISTS ( + SELECT 1 FROM muse_knowledge_draft d + WHERE d.id=e.draft_id AND d.tenant_id=%s + AND d.work_id=%s AND d.source_type=%s + )""", + (TENANT, TENANT, work_id, SOURCE_TYPE)).fetchone()[0] click.echo(f"《{title}》work={work_id} 重抽准备{'（执行）' if execute else '（预览）'}：") - click.echo(f" 软删活升格卡 {cards} ｜ 重置窗 {wins[0]}(其中done {wins[1]}) ｜ " + click.echo(f" 软删活升格卡 {cards} ｜ 软删活向量 {embeddings} ｜ " + f"重置窗 {wins[0]}(其中done {wins[1]}) ｜ " f"硬删 别名{ali} 留档{pres} 卡水位{cs} 审计{aud}") if not execute: click.echo(" （预览模式未写库；加 --execute 真清）") return + # 目标 draft 只要曾绑定确认实体就失败关闭，避免把已确认实体向量降格为 reset 产物处理。 + confirmed_vectors = conn.execute( + """SELECT count(*) FROM example_knowledge_embedding e + WHERE e.tenant_id=%s AND e.entity_id IS NOT NULL AND EXISTS ( + SELECT 1 FROM muse_knowledge_draft d + WHERE d.id=e.draft_id AND d.tenant_id=%s + AND d.work_id=%s AND d.source_type=%s + )""", + (TENANT, TENANT, work_id, SOURCE_TYPE)).fetchone()[0] + if confirmed_vectors: + raise click.ClickException( + f"目标升格 draft 存在 {confirmed_vectors} 条 entity_id 非空向量，拒绝 reset" + ) # 单事务执行：全清或全不清（中途失败自动回滚，不留半清状态） conn.execute( """UPDATE muse_knowledge_draft SET deleted=TRUE, updater='upgrade-reset' WHERE tenant_id=%s AND work_id=%s AND source_type=%s AND deleted=FALSE""", (TENANT, work_id, SOURCE_TYPE)) + # 只软删目标作品升格卡的活向量；双租户边界避免软引用串租户时误伤。 + conn.execute( + """UPDATE example_knowledge_embedding e SET deleted=TRUE, updater='upgrade-reset' + WHERE e.tenant_id=%s AND e.deleted=FALSE AND e.entity_id IS NULL AND EXISTS ( + SELECT 1 FROM muse_knowledge_draft d + WHERE d.id=e.draft_id AND d.tenant_id=%s + AND d.work_id=%s AND d.source_type=%s + )""", + (TENANT, TENANT, work_id, SOURCE_TYPE)) conn.execute( """UPDATE example_upgrade_window SET status='pending', error_message=NULL, updater='upgrade-reset' WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE""", @@ -81,16 +189,16 @@ def _reset(work_id, execute): conn.execute( """DELETE FROM example_upgrade_audit WHERE tenant_id=%s AND draft_id IN - (SELECT id FROM muse_knowledge_draft WHERE work_id=%s AND source_type=%s)""", - (TENANT, work_id, SOURCE_TYPE)) + (SELECT id FROM muse_knowledge_draft + WHERE tenant_id=%s AND work_id=%s AND source_type=%s)""", + (TENANT, TENANT, work_id, SOURCE_TYPE)) conn.execute("DELETE FROM example_upgrade_alias WHERE tenant_id=%s AND work_id=%s", (TENANT, work_id)) conn.execute("DELETE FROM example_upgrade_presence WHERE tenant_id=%s AND work_id=%s", (TENANT, work_id)) conn.execute("DELETE FROM example_upgrade_card_state WHERE tenant_id=%s AND work_id=%s", (TENANT, work_id)) - conn.commit() - # 清后复核：底册应为空、窗应全 pending + # 提交前复核：任一断言失败都由同一业务事务回滚，不留下半清状态。 left = conn.execute( """SELECT count(*) FROM muse_knowledge_draft WHERE tenant_id=%s AND work_id=%s AND source_type=%s AND deleted=FALSE""", @@ -98,20 +206,87 @@ def _reset(work_id, execute): pend = conn.execute( """SELECT count(*) FILTER (WHERE status='pending'), count(*) FROM example_upgrade_window WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE""", (TENANT, work_id)).fetchone() - click.echo(f" ✅ 清后复核：活升格卡 {left}（应0）｜ 窗 pending {pend[0]}/{pend[1]}（应全 pending）") + aliases_left = conn.execute( + "SELECT count(*) FROM example_upgrade_alias WHERE tenant_id=%s AND work_id=%s", + (TENANT, work_id)).fetchone()[0] + presence_left = conn.execute( + "SELECT count(*) FROM example_upgrade_presence WHERE tenant_id=%s AND work_id=%s", + (TENANT, work_id)).fetchone()[0] + state_left = conn.execute( + "SELECT count(*) FROM example_upgrade_card_state WHERE tenant_id=%s AND work_id=%s", + (TENANT, work_id)).fetchone()[0] + audit_left = conn.execute( + """SELECT count(*) FROM example_upgrade_audit a + WHERE a.tenant_id=%s AND a.draft_id IN + (SELECT id FROM muse_knowledge_draft + WHERE tenant_id=%s AND work_id=%s AND source_type=%s)""", + (TENANT, TENANT, work_id, SOURCE_TYPE)).fetchone()[0] + vectors_left = conn.execute( + """SELECT count(*) FROM example_knowledge_embedding e + WHERE e.tenant_id=%s AND e.deleted=FALSE AND EXISTS ( + SELECT 1 FROM muse_knowledge_draft d + WHERE d.id=e.draft_id AND d.tenant_id=%s + AND d.work_id=%s AND d.source_type=%s + )""", + (TENANT, TENANT, work_id, SOURCE_TYPE)).fetchone()[0] + failures = [] + if left: + failures.append(f"active升格卡={left}") + if pend[0] != pend[1]: + failures.append(f"windows pending={pend[0]}/{pend[1]}") + for label, value in ( + ("alias", aliases_left), ("presence", presence_left), + ("state", state_left), ("audit", audit_left), + ("目标draft活向量", vectors_left)): + if value: + failures.append(f"{label}={value}") + if failures: + raise click.ClickException( + "清后复核失败：" + "；".join(failures) + ) + conn.commit() + click.echo(f" ✅ 七域清理断言全部通过：活升格卡0 ｜ " + f"窗pending {pend[0]}/{pend[1]} ｜ alias/presence/state/audit 0 ｜ " + f"目标draft活向量0") @click.command() @click.option("--work-id", type=int, required=True) @click.option("--execute", is_flag=True, help="真执行（默认只预览统计）") -def main(work_id, execute): +@click.option("--backup-dir", type=click.Path(path_type=pathlib.Path, exists=True, + file_okay=False, resolve_path=False)) +@click.option("--backup-id") +@click.option("--confirmation-sha") +def main(work_id, execute, backup_dir, backup_id, confirmation_sha): """先获取同书会话锁，再预览或执行升格全量重抽准备。""" + if execute: + _require_execute_backup(backup_dir, backup_id, confirmation_sha) try: with upgrade_work_lock(DSN, TENANT, work_id): - return _reset(work_id, execute) + manifest = None + if execute: + manifest = backup.verify_backup(backup_dir) + backup.validate_execute_confirmation( + True, + backup_id, + manifest["backup_id"], + confirmation_sha, + manifest["confirmationSha"], + ) + if manifest.get("tenant") != TENANT or manifest.get("work") != work_id: + raise click.ClickException("备份 manifest 的 tenant/work 与 reset 目标不一致") + # 仍在同书 advisory lock 内，且尚未创建业务连接或执行任何 reset SQL。 + _assert_code_identity_matches_manifest( + manifest, backup.capture_code_identity() + ) + return _reset(work_id, execute, manifest) except UpgradeWorkLockUnavailable as exc: raise click.ClickException(str(exc)) from exc + except click.ClickException: + raise + except Exception as exc: + raise click.ClickException(str(exc)) from exc if __name__ == "__main__": diff --git a/.claude/skills/parse-book/scripts/test_backup_upgrade_work_offline.py b/.claude/skills/parse-book/scripts/test_backup_upgrade_work_offline.py index b81c3d1..41d0b4f 100644 --- a/.claude/skills/parse-book/scripts/test_backup_upgrade_work_offline.py +++ b/.claude/skills/parse-book/scripts/test_backup_upgrade_work_offline.py @@ -592,7 +592,7 @@ def test_expected_scope_code_dependencies_and_lock_fail_closed(): identity = backup.capture_code_identity("HEAD") _check("代码身份-补齐实际依赖SHA", { - "parse_llm.py", "embed_drafts.py", "upgrade_work_lock.py" + "reset_upgrade_work.py", "parse_llm.py", "embed_drafts.py", "upgrade_work_lock.py" }.issubset(identity["codeFiles"])) business_calls = {"count": 0} diff --git a/.claude/skills/parse-book/scripts/test_parse_upgrade_offline.py b/.claude/skills/parse-book/scripts/test_parse_upgrade_offline.py index f5420a6..10e4664 100644 --- a/.claude/skills/parse-book/scripts/test_parse_upgrade_offline.py +++ b/.claude/skills/parse-book/scripts/test_parse_upgrade_offline.py @@ -3,7 +3,7 @@ 红线：**不连库、不发任何网络/嵌入/LLM 调用**——只覆盖纯函数与提示词构造器： · 机械判重分类 _classify_new_name（各分支） - · 登场兜底 _debut_milestone（有/无登场、无出场章、摘要超长） + · 登场兜底 _debut_milestone（有/无登场、无出场章、跨章事实隔离） · 里程碑台阶守卫 _clean_milestone（超 80 字截断） · 归并材料 _merge_material（演变历程完整不截、其余字段截断） · embed_drafts.build_embed_text 对 type 键的型修正 @@ -14,6 +14,7 @@ 跑法：仓根 `.venv/bin/python .claude/skills/parse-book/scripts/test_parse_upgrade_offline.py` """ import json +import hashlib import pathlib import sys from contextlib import nullcontext @@ -72,34 +73,107 @@ def test_classify_new_name(): name_map, presence) == ("presence", None)) +def test_resolve_canonical_name(): + """name_map 的 alias 键命中时，落库正名仍必须取同 draft 的真实活跃卡名称。""" + + target = (701, "item", "一代机械外骨骼") + name_map = { + "一代机械外骨骼·铁壳": target, + "铁壳": target, + "机械铁壳": target, + } + aliases_by_draft = {701: {"铁壳", "机械铁壳"}} + check( + "canonical-alias-token解析为真实卡名", + pu._resolve_canonical_name(name_map, aliases_by_draft, "铁壳") == "一代机械外骨骼·铁壳", + ) + check( + "canonical-substring命中alias仍解析为真实卡名", + pu._resolve_canonical_name(name_map, aliases_by_draft, "机械铁壳") == "一代机械外骨骼·铁壳", + ) + try: + pu._resolve_canonical_name({"铁壳": target}, {701: {"铁壳"}}, "铁壳") + except RuntimeError as exc: + check("canonical-找不到真实卡名失败关闭", "真实规范名" in str(exc)) + else: + check("canonical-找不到真实卡名失败关闭", False, detail="未抛 RuntimeError") + + # ── ② 登场兜底 _debut_milestone（洞②机械那一保险）── def test_debut_milestone(): # 已有登场 → 原样返回，不重复补 has = [{"章": 10, "台阶": "训练机登场", "周期": "登场"}, {"章": 50, "台阶": "进化", "周期": "成长"}] - out = pu._debut_milestone(has, "摘要", [8], 3) + out = pu._debut_milestone(has, "训练机", "item", [], [8], 3, {8: "训练机进入机库。"}) check("debut-已有登场不补", len(out) == 2 and out[0]["台阶"] == "训练机登场") - # 无登场 + 有出场章 → 头部补登场，章=min(出场章)，_win 盖窗号 + # 无登场 + 有正文实证出场章 → 只补中性名称/型描述，章=min(出场章)，_win 盖窗号 no_debut = [{"章": 50, "台阶": "V代编队", "周期": "成长"}] - out = pu._debut_milestone(no_debut, "4级训练机", [15, 3, 8], 7) + out = pu._debut_milestone( + no_debut, + "4级训练机", + "item", + [], + [15, 3, 8], + 7, + {3: "4级训练机被推入测试场。", 8: "4级训练机完成返航。", 15: "4级训练机封存。"}, + ) check("debut-无登场则补一条", len(out) == 2 and out[0]["周期"] == "登场" and out[0]["章"] == 3 and out[0]["_win"] == 7) - check("debut-台阶取摘要", out[0]["台阶"].startswith("登场：4级训练机")) + check("debut-台阶仅用中性名称与型", out[0]["台阶"] == "物件「4级训练机」登场") check("debut-进化台阶仍在尾部", out[1]["台阶"] == "V代编队") # 无正文实证出场章 → 不补登场里程碑（真实性优先，不能生成无证据台阶） - out = pu._debut_milestone([], "无章摘要", [], 5) + out = pu._debut_milestone([], "无章实体", "item", [], [], 5, {}) check("debut-无真实章不补", out == []) # milestones=None 也当空处理 - out = pu._debut_milestone(None, "空列表摘要", [2], 5) + out = pu._debut_milestone(None, "空列表实体", "item", [], [2], 5, {2: "空列表实体出现。"}) check("debut-None当空补登场", len(out) == 1 and out[0]["周期"] == "登场" and out[0]["章"] == 2) - # 摘要超长 → 台阶截 40 字摘要 + 过 STEP_MAX 守卫（总长 ≤ STEP_MAX） - out = pu._debut_milestone([], "甲" * 100, [2], 9) - step = out[0]["台阶"] - check("debut-摘要超长台阶受守卫", len(step) <= pu.STEP_MAX and step.startswith("登场：") and "甲" * 40 in step) - check("debut-摘要只取前40字", "甲" * 41 not in step) + # 反例：实体第57章首现，但当前窗后续章材料含第59/60章事实；兜底绝不能把未来事实倒灌到57章。 + future_fact = "第59章升级二代装甲，第60章击毁母舰" + out = pu._debut_milestone( + [], + "一代机械外骨骼·铁壳", + "item", + ["铁壳"], + [57, 59, 60], + 8, + { + 57: "一代机械外骨骼·铁壳第一次走出维修架。", + 59: "铁壳升级成二代装甲。", + 60: "铁壳击毁母舰。", + }, + ) + check("debut-57章台阶不含59与60章事实", + out[0]["章"] == 57 and future_fact not in out[0]["台阶"] + and "升级" not in out[0]["台阶"] and "母舰" not in out[0]["台阶"]) + alias_only = pu._debut_milestone( + [], + "二代机械外骨骼·铁壳", + "item", + ["铁壳"], + [57], + 8, + {57: "铁壳第一次走出维修架。"}, + ) + check("debut-规范名未出现时只用首章真实别名", alias_only[0]["台阶"] == "物件「铁壳」登场") + ambiguous = pu._debut_milestone( + [], + "二代机械外骨骼·铁壳", + "item", + ["铁壳", "老铁"], + [57], + 8, + {57: "铁壳又被队员叫作老铁。"}, + ) + check("debut-多名称命中退化为仅类型", ambiguous[0]["台阶"] == "物件登场") + generic_only = pu._debut_milestone( + [], "未来规范名", "character", ["队长"], [57], 8, {57: "队长首次出现。"} + ) + check("debut-通用称谓不得进入台阶", generic_only[0]["台阶"] == "人物登场") + check("debut-缺少首章正文宁可不补", + pu._debut_milestone([], "铁壳", "item", [], [57], 8, {59: "铁壳升级。"}) == []) # ── ③ 里程碑台阶守卫 _clean_milestone（洞③：超 80 字截断）── @@ -289,6 +363,152 @@ class _CardConn: """模拟事务提交；离线测试中的状态已在内存立即生效。""" +class _RelationConn: + """关系卡持久化与 undo 的内存数据库，覆盖 payload、审计、水位和软删。""" + + def __init__(self, drafts=None, states=None): + self.drafts = { + did: {"payload": deepcopy(payload), "deleted": False} + for did, payload in (drafts or {}).items() + } + self.states = dict(states or {}) + self.audits = [] + self.next_draft_id = max(self.drafts, default=700) + 1 + + def execute(self, query, params=()): + """按生产 SQL 语义更新内存状态，测试不建立任何真实连接。""" + + normalized = " ".join(query.split()) + if normalized.startswith("SELECT watermark_window FROM example_upgrade_card_state"): + watermark = self.states.get(params[0]) + return _CardResult((watermark,) if watermark is not None else None) + if normalized.startswith("SELECT a.draft_id, a.field_name, a.old_value"): + window_no = params[1] + rows = [ + (item["draft_id"], item["field_name"], item["old_value"]) + for item in reversed(self.audits) + if item["window_no"] == window_no + ] + return _CardResult(rows=rows) + if normalized.startswith("SELECT draft_payload FROM muse_knowledge_draft WHERE id=%s"): + return _CardResult((self.drafts[params[0]]["payload"],)) + if normalized.startswith("SELECT id, draft_payload FROM muse_knowledge_draft"): + return _CardResult(rows=[ + (did, row["payload"]) + for did, row in self.drafts.items() + if not row["deleted"] + ]) + if normalized.startswith("INSERT INTO example_upgrade_audit"): + self.audits.append({ + "draft_id": params[0], + "window_no": params[1], + "field_name": params[2], + "old_value": params[3], + "new_value": params[4], + }) + return _CardResult() + if normalized.startswith("INSERT INTO muse_knowledge_draft"): + did = self.next_draft_id + self.next_draft_id += 1 + self.drafts[did] = {"payload": json.loads(params[1]), "deleted": False} + return _CardResult((did,)) + if normalized.startswith("UPDATE muse_knowledge_draft SET draft_payload=%s"): + self.drafts[params[1]]["payload"] = json.loads(params[0]) + return _CardResult() + if normalized.startswith("UPDATE muse_knowledge_draft SET deleted=TRUE"): + self.drafts[params[0]]["deleted"] = True + return _CardResult() + if normalized.startswith("INSERT INTO example_upgrade_card_state"): + did, _, watermark = params[:3] + if "SET watermark_window=EXCLUDED.watermark_window" in normalized: + self.states[did] = watermark + else: + self.states[did] = max(self.states.get(did, watermark), watermark) + return _CardResult() + if normalized.startswith("UPDATE example_upgrade_card_state SET watermark_window=%s"): + self.states[params[1]] = params[0] + return _CardResult() + if normalized.startswith("DELETE FROM example_upgrade_card_state"): + self.states.pop(params[0], None) + return _CardResult() + if normalized.startswith("DELETE FROM example_upgrade_audit"): + window_no = params[1] + self.audits = [item for item in self.audits if item["window_no"] != window_no] + return _CardResult() + return _CardResult(rows=[]) + + +def test_relation_update_and_new_card_undo_boundaries(): + """关系更新按真实落点审计并可完整 undo；本窗新关系软删且清除 state。""" + + old_payload = { + "type": pu.RELATION_TYPE, + "名称": "甲×乙", + "甲方draft": 11, + "乙方draft": 12, + "关系类型": "盟友", + "字段": {"当前状态": "合作", "演变轨迹": ["[窗3] 初次合作"]}, + "来源": "升格@窗3", + "_work_id": 8, + } + conn = _RelationConn({701: old_payload}, {701: 3}) + pu._update_relation_card( + conn, + 8, + 8, + 701, + deepcopy(old_payload), + { + "关系类型": "对手", + "本窗演变": "公开决裂", + "其他字段": {"当前状态": "敌对", "信任基础": "破裂"}, + }, + ) + updated = conn.drafts[701]["payload"] + check("relation-update-顶层关系类型落正确位置", updated["关系类型"] == "对手") + check("relation-update-其他字段逐字段落库", + updated["字段"]["当前状态"] == "敌对" and updated["字段"]["信任基础"] == "破裂") + check("relation-update-演变轨迹带窗号", updated["字段"]["演变轨迹"][-1] == "[窗8] 公开决裂") + audit_by_name = {item["field_name"]: item for item in conn.audits} + check("relation-update-关系类型使用顶层sentinel", + pu.TOP_LEVEL_RELATION_TYPE_AUDIT_FIELD in audit_by_name) + check("relation-update-其他字段完整旧新审计", + json.loads(audit_by_name["当前状态"]["old_value"]) == "合作" + and json.loads(audit_by_name["当前状态"]["new_value"]) == "敌对" + and audit_by_name["信任基础"]["old_value"] is None) + check("relation-update-演变轨迹完整旧新审计", + json.loads(audit_by_name["演变轨迹"]["old_value"]) == ["[窗3] 初次合作"] + and json.loads(audit_by_name["演变轨迹"]["new_value"])[-1] == "[窗8] 公开决裂") + check("relation-update-state升到本窗且留旧水位审计", + conn.states[701] == 8 + and json.loads(audit_by_name[pu.CARD_STATE_WATERMARK_AUDIT_FIELD]["old_value"]) == 3) + + pu.undo_window(conn, 8, 8) + check("relation-update-undo完整恢复payload", conn.drafts[701]["payload"] == old_payload) + check("relation-update-undo恢复旧水位", conn.states[701] == 3) + + new_payload = { + "type": pu.RELATION_TYPE, + "名称": "甲×丙", + "甲方draft": 11, + "乙方draft": 13, + "关系类型": "师徒", + "字段": {"当前状态": "建立", "演变轨迹": ["[窗8] 首次指点"]}, + "来源": "升格@窗8", + "_work_id": 8, + } + new_id = pu._insert_relation_card(conn, 8, 8, new_payload) + new_audits = [item for item in conn.audits if item["draft_id"] == new_id] + check("relation-new-取得RETURNING-id并建state", new_id in conn.drafts and conn.states[new_id] == 8) + check("relation-new-不是单条占位审计", + {item["field_name"] for item in new_audits} + >= {pu.TOP_LEVEL_RELATION_TYPE_AUDIT_FIELD, "当前状态", "演变轨迹", + pu.CARD_STATE_WATERMARK_AUDIT_FIELD}) + pu.undo_window(conn, 8, 8) + check("relation-new-undo软删", conn.drafts[new_id]["deleted"] is True) + check("relation-new-undo清除state", new_id not in conn.states) + + def test_entity_chapter_evidence_filter(): """模型出场章只能保留规范名或合法别名在对应正文真实出现的章节。""" @@ -589,9 +809,21 @@ class _RunConn: if normalized.startswith("SELECT window_no FROM example_upgrade_window"): return _CardResult(rows=[]) if normalized.startswith("SELECT draft_payload FROM muse_knowledge_draft"): - return _CardResult((self.state["payload"],)) + payload = self.state.get("payloads", {}).get(params[0], self.state.get("payload")) + return _CardResult((payload,)) + if normalized.startswith("INSERT INTO muse_knowledge_draft"): + draft_id = self.state.setdefault("next_draft_id", 101) + self.state["next_draft_id"] = draft_id + 1 + self.state.setdefault("payloads", {})[draft_id] = json.loads(params[1]) + return _CardResult((draft_id,)) if normalized.startswith("UPDATE muse_knowledge_draft SET draft_payload=%s"): - self.state["payload"] = json.loads(params[0]) + payload = json.loads(params[0]) + if "payloads" in self.state: + self.state["payloads"][params[1]] = payload + else: + self.state["payload"] = payload + if normalized.startswith("INSERT INTO example_upgrade_alias"): + self.state.setdefault("alias_rows", []).append(params) if normalized.startswith("UPDATE example_upgrade_window SET status='failed'"): self.state["window_status"] = "failed" if normalized.startswith("UPDATE example_upgrade_window SET status='done'"): @@ -605,6 +837,563 @@ class _RunConn: self.commits += 1 +def test_run_alias_paths_store_real_canonical_name(): + """疑似别名与 substring 命中 alias 键时，实际 INSERT 均写活跃卡真实名称。""" + + canonical = "一代机械外骨骼·铁壳" + target = (701, "item", "一代机械外骨骼") + state = { + "payload": { + "type": "item", + "名称": canonical, + "别名": ["铁壳"], + "字段": {}, + "_work_id": 8, + }, + } + + def fake_m3_json(prompt, model, need_keys, system=None): + if need_keys == ("新名字", "已知实体新信息", "纯出场"): + return ({ + "新名字": [ + {"型": "item", "名称": "新铁壳", "疑似别名指向": "铁壳", + "出场章": [57], "字段": {}}, + {"型": "item", "名称": "机械铁壳", "出场章": [58], "字段": {}}, + ], + "已知实体新信息": [], + "纯出场": [], + }, {}) + if need_keys == ("更新",): + return ({"更新": []}, {}) + raise AssertionError(f"不应出现的离线模型调用：{need_keys}") + + name_map = {canonical: target, "铁壳": target} + with patch.object(pu.psycopg, "connect", side_effect=lambda *_: _RunConn(state)), \ + patch.object(pu, "load_entity_contracts", return_value=_fake_contracts()), \ + patch.object(pu, "load_window_material", return_value=( + "新铁壳出现，机械铁壳随后启动。", + {57: "新铁壳出现。", 58: "机械铁壳随后启动。"}, + )), \ + patch.object(pu, "load_known", return_value=(name_map, {}, {701: {"铁壳"}})), \ + patch.object(pu, "m3_json", side_effect=fake_m3_json): + pu._run( + work_id=8, + max_windows=0, + max_calls=0, + model="MiniMax-M3", + redo_window=0, + semantic_on=False, + ) + check("alias-run-两条alias路径均执行", len(state.get("alias_rows", [])) == 2) + check("alias-run-canonical_name始终是真实卡名", + {params[1] for params in state["alias_rows"]} == {canonical}, detail=str(state["alias_rows"])) + + +def test_run_new_card_registers_normalized_name_and_aliases_same_window(): + """括号名立卡后，同窗后续 alias/substr 必须命中新 canonical 并写真实规范名。""" + + state = {"payloads": {}} + + def fake_m3_json(prompt, model, need_keys, system=None): + if need_keys == ("新名字", "已知实体新信息", "纯出场"): + return ({ + "新名字": [ + {"型": "item", "名称": "白色游魂（无名侦察兵）", "出场章": [70, 71], + "字段": {}, "别名": []}, + {"型": "item", "名称": "侦察兵甲", "疑似别名指向": "无名侦察兵", + "出场章": [70], "字段": {}}, + {"型": "item", "名称": "无名侦察兵甲", "出场章": [71], "字段": {}}, + ], + "已知实体新信息": [], + "纯出场": [], + }, {}) + if need_keys == ("更新",): + return ({"更新": []}, {}) + raise AssertionError(f"不应出现的离线模型调用：{need_keys}") + + with patch.object(pu.psycopg, "connect", side_effect=lambda *_: _RunConn(state)), \ + patch.object(pu, "load_entity_contracts", return_value=_fake_contracts()), \ + patch.object(pu, "load_window_material", return_value=( + "白色游魂现身，侦察兵甲靠近。无名侦察兵甲随后出现。", + {70: "白色游魂现身，侦察兵甲靠近。", 71: "无名侦察兵甲随后出现。"}, + )), \ + patch.object(pu, "load_known", return_value=({}, {}, {})), \ + patch.object(pu, "m3_json", side_effect=fake_m3_json): + pu._run( + work_id=8, + max_windows=0, + max_calls=0, + model="MiniMax-M3", + redo_window=0, + semantic_on=False, + ) + candidate_rows = [ + params for params in state.get("alias_rows", []) + if params[2] in {"侦察兵甲", "无名侦察兵甲"} + ] + check("new-card-index-同窗alias与substr均命中", len(candidate_rows) == 2, detail=str(state)) + check("new-card-index-canonical_name均为规范化名称", + {params[1] for params in candidate_rows} == {"白色游魂"}, detail=str(candidate_rows)) + + +class _RelationRunConn: + """复用关系内存库执行一个完整窗，验证关系 draft id 真进入提交后 touched。""" + + def __init__(self, db): + self.db = db + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, query, params=()): + normalized = " ".join(query.split()) + if normalized.startswith("SELECT title FROM muse_content_work"): + return _CardResult(("离线关系书",)) + if normalized.startswith("SELECT window_no, from_chapter, to_chapter, status"): + return _CardResult(rows=[(8, 57, 60, "pending")]) + if normalized.startswith("SELECT window_no FROM example_upgrade_window"): + return _CardResult(rows=[]) + if normalized.startswith("SELECT id, draft_payload FROM muse_knowledge_draft") \ + and "draft_payload->>'type'=%s" in normalized: + return _CardResult(rows=[ + (did, row["payload"]) + for did, row in self.db.drafts.items() + if not row["deleted"] and row["payload"].get("type") == pu.RELATION_TYPE + ]) + return self.db.execute(query, params) + + def commit(self): + """内存写即时生效，无需额外动作。""" + + +def test_run_relation_cards_enter_touched_embedding(): + """关系更新与新建都必须进入窗提交后的 touched，复用唯一活向量刷新链路。""" + + character_payloads = { + 11: {"type": "character", "名称": "甲", "字段": {}, "_work_id": 8}, + 12: {"type": "character", "名称": "乙", "字段": {}, "_work_id": 8}, + 13: {"type": "character", "名称": "丙", "字段": {}, "_work_id": 8}, + } + relation_payload = { + "type": pu.RELATION_TYPE, + "名称": "甲×乙", + "甲方draft": 11, + "乙方draft": 12, + "关系类型": "盟友", + "字段": {"演变轨迹": ["[窗3] 初次合作"]}, + "来源": "升格@窗3", + "_work_id": 8, + } + db = _RelationConn({**character_payloads, 701: relation_payload}, {701: 3}) + embedded = [] + + def fake_m3_json(prompt, model, need_keys, system=None): + if need_keys == ("新名字", "已知实体新信息", "纯出场"): + return ({"新名字": [], "已知实体新信息": [], "纯出场": []}, {}) + if need_keys == ("关系",): + return ({ + "关系": [ + {"甲方": 11, "乙方": 12, "关系类型": "对手", "本窗演变": "公开决裂", + "其他字段": {"当前状态": "敌对"}}, + {"甲方": 11, "乙方": 13, "关系类型": "师徒", "本窗演变": "首次指点", + "其他字段": {"当前状态": "建立"}}, + ] + }, {}) + raise AssertionError(f"不应出现的离线模型调用：{need_keys}") + + name_map = { + "甲": (11, "character", ""), + "乙": (12, "character", ""), + "丙": (13, "character", ""), + } + with patch.object(pu.psycopg, "connect", side_effect=lambda *_: _RelationRunConn(db)), \ + patch.object(pu, "load_entity_contracts", return_value=_fake_contracts()), \ + patch.object(pu, "load_window_material", return_value=( + "甲与乙决裂，随后甲开始指点丙。", + {57: "甲与乙共同出现。", 60: "甲开始指点丙。"}, + )), \ + patch.object(pu, "load_known", return_value=(name_map, {}, {})), \ + patch.object(pu, "m3_json", side_effect=fake_m3_json), \ + patch.object(pu, "_embed_session", return_value=object()), \ + patch.object(pu, "prejudge_semantic", return_value=({}, 0, 0)), \ + patch.object( + pu, + "embed_touched_cards", + side_effect=lambda sess, work_id, touched: embedded.append(set(touched)) or len(touched), + ): + pu._run( + work_id=8, + max_windows=0, + max_calls=0, + model="MiniMax-M3", + redo_window=0, + semantic_on=True, + ) + check("relation-run-新关系取得id", 702 in db.drafts) + check("relation-run-更新与新建关系均进入touched", embedded == [{701, 702}], detail=str(embedded)) + + +def test_run_embedding_owner_conflict_marks_window_failed(): + """当前新 hash 无 owner 时，同 draft 旧活向量的 entity owner 仍必须阻断且保持活跃。""" + + state = { + "window_status": "pending", + "payload": { + "type": "character", + "名称": "安若雪", + "别名": [], + "字段": {}, + "_work_id": 8, + }, + } + + def fake_m3_json(prompt, model, need_keys, system=None): + if need_keys == ("新名字", "已知实体新信息", "纯出场"): + return ({ + "新名字": [], + "已知实体新信息": [{"名称": "安若雪", "观察点": "状态变化", "出场章": [70]}], + "纯出场": [], + }, {}) + if need_keys == ("更新",): + return ({"更新": []}, {}) + raise AssertionError(f"不应出现的离线模型调用：{need_keys}") + + embed_text = "当前新向量文本" + old_embed_text = "同 draft 旧向量文本" + old_content_hash = hashlib.sha256(f"{old_embed_text}|{pu.EMBED_MODEL}".encode()).hexdigest() + embedding_state = { + "drafts": { + 701: {"payload": deepcopy(state["payload"]), "deleted": False}, + }, + "embeddings": [{ + "draft_id": 701, + "entity_id": 9001, + "content_hash": old_content_hash, + "embed_text": old_embed_text, + "model": pu.EMBED_MODEL, + "dimensions": pu.EMBED_DIM, + "embedding": "[0.3]", + "deleted": False, + }], + } + embed_calls = [] + real_embed_touched_cards = pu.embed_touched_cards + + def run_real_embed(sess, work_id, touched): + with patch.object(pu.psycopg, "connect", side_effect=lambda *_: _EmbeddingConn(embedding_state)), \ + patch.object(pu, "build_embed_text", return_value=embed_text), \ + patch.object( + pu, + "embed_texts", + side_effect=lambda embed_sess, texts: embed_calls.append(list(texts)) or ([[0.9]], set()), + ): + return real_embed_touched_cards(sess, work_id, touched) + + with patch.object(pu, "upgrade_work_lock", return_value=nullcontext()), \ + patch.object(pu.psycopg, "connect", side_effect=lambda *_: _RunConn(state)), \ + patch.object(pu, "load_entity_contracts", return_value=_fake_contracts()), \ + patch.object( + pu, + "load_window_material", + return_value=("安若雪状态变化。", {70: "安若雪状态变化。", 71: "正文。"}), + ), \ + patch.object( + pu, + "load_known", + return_value=({"安若雪": (701, "character", "旧摘要")}, {}, {}), + ), \ + patch.object(pu, "m3_json", side_effect=fake_m3_json), \ + patch.object(pu, "_embed_session", return_value=object()), \ + patch.object(pu, "prejudge_semantic", return_value=({}, 0, 0)), \ + patch.object( + pu, + "embed_touched_cards", + side_effect=run_real_embed, + ): + result = CliRunner().invoke(pu.cli, ["run", "--work-id", "8", "--semantic-dedup"]) + check("embed-conflict-run-命令非零", result.exit_code != 0, detail=result.output) + check("embed-conflict-run-status补偿为failed", state["window_status"] == "failed") + check("embed-conflict-run-不打印窗完成", "窗7✓" not in result.output, detail=result.output) + check("embed-old-entity-owner-新向量计算已发生", embed_calls == [[embed_text]]) + check("embed-old-entity-owner-保持活跃与绑定", + embedding_state["embeddings"][0]["draft_id"] == 701 + and embedding_state["embeddings"][0]["entity_id"] == 9001 + and embedding_state["embeddings"][0]["deleted"] is False) + + +def test_run_embedding_payload_drift_marks_window_failed_without_vector_writes(): + """HTTP 期间 payload 漂移时，写段必须锁行复验并在任何向量 UPDATE/UPSERT 前阻断。""" + + state = { + "window_status": "pending", + "payload": { + "type": "character", "名称": "安若雪", "别名": [], "字段": {}, "_work_id": 8, + }, + } + embedding_state = { + "drafts": {701: { + "payload": deepcopy(state["payload"]), "deleted": False, + "tenant_id": pu.TENANT, "status": "pending", + }}, + "embeddings": [], + } + real_embed_touched_cards = pu.embed_touched_cards + + def fake_m3_json(prompt, model, need_keys, system=None): + if need_keys == ("新名字", "已知实体新信息", "纯出场"): + return ({"新名字": [], "已知实体新信息": [ + {"名称": "安若雪", "观察点": "状态变化", "出场章": [70]} + ], "纯出场": []}, {}) + if need_keys == ("更新",): + return ({"更新": []}, {}) + raise AssertionError(f"不应出现的离线模型调用：{need_keys}") + + def run_real_embed(sess, work_id, touched): + def drift_during_http(embed_sess, texts): + embedding_state["drafts"][701]["payload"]["字段"]["HTTP后漂移"] = "新值" + return ([[0.7]], set()) + + with patch.object(pu.psycopg, "connect", side_effect=lambda *_: _EmbeddingConn(embedding_state)), \ + patch.object(pu, "embed_texts", side_effect=drift_during_http): + return real_embed_touched_cards(sess, work_id, touched) + + with patch.object(pu, "upgrade_work_lock", return_value=nullcontext()), \ + patch.object(pu.psycopg, "connect", side_effect=lambda *_: _RunConn(state)), \ + patch.object(pu, "load_entity_contracts", return_value=_fake_contracts()), \ + patch.object(pu, "load_window_material", return_value=( + "安若雪状态变化。", {70: "安若雪状态变化。", 71: "正文。"}, + )), \ + patch.object(pu, "load_known", return_value=( + {"安若雪": (701, "character", "旧摘要")}, {}, {}, + )), \ + patch.object(pu, "m3_json", side_effect=fake_m3_json), \ + patch.object(pu, "_embed_session", return_value=object()), \ + patch.object(pu, "prejudge_semantic", return_value=({}, 0, 0)), \ + patch.object(pu, "embed_touched_cards", side_effect=run_real_embed): + result = CliRunner().invoke(pu.cli, ["run", "--work-id", "8", "--semantic-dedup"]) + + vector_writes = [sql for sql in embedding_state.get("sql_order", []) + if sql.startswith("UPDATE example_knowledge_embedding") + or sql.startswith("INSERT INTO example_knowledge_embedding")] + check("embed-payload-drift-命令非零且窗failed", + result.exit_code != 0 and state["window_status"] == "failed", detail=result.output) + check("embed-payload-drift-无窗完成", "窗7✓" not in result.output, detail=result.output) + check("embed-payload-drift-零向量写入", not vector_writes, detail=str(vector_writes)) + + +class _EmbeddingConn: + """增量嵌入 fake DB：复现旧软删 draft 占用同 hash 唯一键的冲突行。""" + + def __init__(self, state): + self.state = state + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, query, params=()): + normalized = " ".join(query.split()) + self.state.setdefault("sql_order", []).append(normalized) + if normalized.startswith("SELECT tenant_id, deleted, status, draft_payload"): + draft = self.state["drafts"].get(params[0]) + if not draft: + return _CardResult(None) + return _CardResult(( + draft.get("tenant_id", pu.TENANT), + draft["deleted"], + draft.get("status", "pending"), + draft["payload"], + )) + self.state.setdefault("sql_order", []).append(normalized) + if normalized.startswith("SELECT draft_payload FROM muse_knowledge_draft"): + draft = self.state["drafts"].get(params[0]) + if not draft or draft["deleted"]: + return _CardResult(None) + return _CardResult((draft["payload"],)) + if normalized.startswith("SELECT content_hash FROM example_knowledge_embedding"): + rows = [ + (row["content_hash"],) + for row in self.state["embeddings"] + if row["draft_id"] == params[1] and not row["deleted"] + ] + return _CardResult(rows=rows) + if normalized.startswith("SELECT id, content_hash, entity_id FROM example_knowledge_embedding"): + rows = [ + (index + 1, row["content_hash"], row.get("entity_id")) + for index, row in enumerate(self.state["embeddings"]) + if row["draft_id"] == params[1] and not row["deleted"] + ] + return _CardResult(rows=rows) + if normalized.startswith("SELECT e.draft_id"): + content_hash = params[1] + row = next( + (item for item in self.state["embeddings"] if item["content_hash"] == content_hash), + None, + ) + if row is None: + return _CardResult(None) + owner = self.state["drafts"].get(row["draft_id"]) + owner_values = ( + row["draft_id"], + row["deleted"], + owner["deleted"] if owner else True, + owner["payload"] if owner else None, + ) + if "e.entity_id" in normalized: + owner_values = (owner_values[0], row.get("entity_id"), *owner_values[1:]) + return _CardResult(owner_values) + if normalized.startswith("UPDATE example_knowledge_embedding SET deleted=TRUE"): + for row in self.state["embeddings"]: + if row["draft_id"] == params[2] and not row["deleted"] \ + and row["content_hash"] != params[3] \ + and ("entity_id IS NULL" not in normalized or row.get("entity_id") is None): + row["deleted"] = True + return _CardResult() + if normalized.startswith("INSERT INTO example_knowledge_embedding"): + draft_id, content_hash, embed_text, model, dimensions, embedding = params[:6] + row = next( + (item for item in self.state["embeddings"] if item["content_hash"] == content_hash), + None, + ) + is_new = row is None + if is_new: + row = {"entity_id": None} + self.state["embeddings"].append(row) + previous_owner = row.get("draft_id") + previous_draft = self.state["drafts"].get(previous_owner) + guarded = "WHERE example_knowledge_embedding.draft_id=EXCLUDED.draft_id" in normalized + can_update = is_new or previous_owner == draft_id or row.get("deleted") \ + or not previous_draft or previous_draft["deleted"] + if "example_knowledge_embedding.entity_id IS NULL" in normalized: + can_update = can_update and row.get("entity_id") is None + if guarded and not can_update: + return _CardResult(None) + # 按 SQL 文本模拟真实 ON CONFLICT：旧实现没有迁移 draft_id，测试应先 RED。 + if is_new or "draft_id=EXCLUDED.draft_id" in normalized: + row["draft_id"] = draft_id + row.update({ + "content_hash": content_hash, + "embed_text": embed_text if "embed_text=EXCLUDED.embed_text" in normalized or is_new else row.get("embed_text"), + "model": model, + "dimensions": dimensions if "dimensions=EXCLUDED.dimensions" in normalized or is_new else row.get("dimensions"), + "embedding": embedding if "embedding=EXCLUDED.embedding" in normalized or is_new else row.get("embedding"), + "deleted": False, + }) + return _CardResult((row["draft_id"],)) + return _CardResult() + + def commit(self): + """fake DB 写入即时生效。""" + + +def test_embed_touched_migrates_soft_deleted_hash_owner(): + """全重抽新 draft 命中旧软删同 hash 时，唯一活向量必须迁到新 draft 并刷新当前列。""" + + payload = {"type": pu.RELATION_TYPE, "名称": "甲×乙", "字段": {"演变轨迹": ["首次合作"]}} + embed_text = "当前关系卡嵌入文本" + content_hash = hashlib.sha256(f"{embed_text}|{pu.EMBED_MODEL}".encode()).hexdigest() + state = { + "drafts": { + 701: {"payload": deepcopy(payload), "deleted": True}, + 902: {"payload": deepcopy(payload), "deleted": False}, + }, + "embeddings": [{ + "draft_id": 701, + "content_hash": content_hash, + "embed_text": "旧文本", + "model": pu.EMBED_MODEL, + "dimensions": pu.EMBED_DIM, + "embedding": "[0.1]", + "deleted": True, + }], + } + current_vector = [0.9, 0.8] + with patch.object(pu.psycopg, "connect", side_effect=lambda *_: _EmbeddingConn(state)), \ + patch.object(pu, "build_embed_text", return_value=embed_text), \ + patch.object(pu, "embed_texts", return_value=([current_vector], set())): + done = pu.embed_touched_cards(object(), 8, {902}) + active = [row for row in state["embeddings"] if not row["deleted"]] + check("embed-conflict-只保留唯一活向量", done == 1 and len(active) == 1) + check("embed-conflict-同hash迁到新draft", active[0]["draft_id"] == 902, detail=str(active)) + check("embed-conflict-同步当前嵌入列", + active[0]["embed_text"] == embed_text + and active[0]["dimensions"] == pu.EMBED_DIM + and json.loads(active[0]["embedding"]) == current_vector) + sql_order = state.get("sql_order", []) + table_lock = next((i for i, sql in enumerate(sql_order) + if sql.startswith("LOCK TABLE muse_knowledge_draft, example_knowledge_embedding")), None) + row_locks = [i for i, sql in enumerate(sql_order) if "FOR UPDATE" in sql] + check("embed-write-固定表锁早于draft与hash行锁", + table_lock is not None and row_locks and table_lock < min(row_locks), detail=str(sql_order)) + + +def test_embed_touched_rejects_two_active_drafts_with_same_hash(): + """两个活跃 touched draft 当前文本同 hash 时必须失败关闭，禁止按循环顺序抢占唯一行。""" + + state = { + "drafts": { + 801: {"payload": {"名称": "重复卡"}, "deleted": False}, + 802: {"payload": {"名称": "重复卡"}, "deleted": False}, + }, + "embeddings": [], + } + embed_calls = [] + try: + with patch.object(pu.psycopg, "connect", side_effect=lambda *_: _EmbeddingConn(state)), \ + patch.object(pu, "build_embed_text", return_value="完全相同文本"), \ + patch.object( + pu, + "embed_texts", + side_effect=lambda sess, texts: embed_calls.append(list(texts)) or ([[0.1], [0.1]], set()), + ): + pu.embed_touched_cards(object(), 8, {801, 802}) + except pu.EmbeddingOwnershipConflict: + check("embed-duplicate-active-失败关闭", True) + else: + check("embed-duplicate-active-失败关闭", False, detail="同 hash 活跃 owner 冲突未抛硬错误") + check("embed-duplicate-active-不调用嵌入不落错误owner", not embed_calls and not state["embeddings"]) + + +def test_embed_touched_rejects_existing_active_hash_owner(): + """单个新候选撞到另一活跃 draft 的当前 hash 时，也不得迁移或复活成错误 owner。""" + + embed_text = "跨窗完全相同文本" + content_hash = hashlib.sha256(f"{embed_text}|{pu.EMBED_MODEL}".encode()).hexdigest() + state = { + "drafts": { + 811: {"payload": {"名称": "重复卡"}, "deleted": False}, + 812: {"payload": {"名称": "重复卡"}, "deleted": False}, + }, + "embeddings": [{ + "draft_id": 811, + "content_hash": content_hash, + "embed_text": embed_text, + "model": pu.EMBED_MODEL, + "dimensions": pu.EMBED_DIM, + "embedding": "[0.2]", + "deleted": False, + }], + } + try: + with patch.object(pu.psycopg, "connect", side_effect=lambda *_: _EmbeddingConn(state)), \ + patch.object(pu, "build_embed_text", return_value=embed_text), \ + patch.object(pu, "embed_texts", return_value=([[0.8]], set())): + pu.embed_touched_cards(object(), 8, {812}) + except pu.EmbeddingOwnershipConflict: + check("embed-existing-active-失败关闭", True) + else: + check("embed-existing-active-失败关闭", False, detail="已有活跃 owner 未阻断") + check("embed-existing-active-owner保持不变", + len(state["embeddings"]) == 1 + and state["embeddings"][0]["draft_id"] == 811 + and state["embeddings"][0]["deleted"] is False) + + def test_run_redo_preflight_rejects_unsafe_targets(): """所有 redo 非法输入都必须在快照、撤销与写入前失败。""" @@ -1022,10 +1811,20 @@ def test_prompts_disciplines(): if __name__ == "__main__": - for fn in (test_classify_new_name, test_debut_milestone, test_clean_milestone_guard, + for fn in (test_classify_new_name, test_resolve_canonical_name, + test_debut_milestone, test_clean_milestone_guard, test_merge_material, test_build_embed_text_type_fix, test_int_chaps, + test_relation_update_and_new_card_undo_boundaries, test_entity_chapter_evidence_filter, test_new_card_chapter_evidence_chain, test_merge_card_chapter_evidence, test_redo_cleans_legacy_appearance_chapters, + test_run_alias_paths_store_real_canonical_name, + test_run_new_card_registers_normalized_name_and_aliases_same_window, + test_run_relation_cards_enter_touched_embedding, + test_run_embedding_owner_conflict_marks_window_failed, + test_run_embedding_payload_drift_marks_window_failed_without_vector_writes, + test_embed_touched_migrates_soft_deleted_hash_owner, + test_embed_touched_rejects_two_active_drafts_with_same_hash, + test_embed_touched_rejects_existing_active_hash_owner, test_run_redo_preflight_rejects_unsafe_targets, test_run_rejects_negative_limits_before_redo_side_effects, test_run_redo_max_calls_finishes_active_retry, diff --git a/.claude/skills/parse-book/scripts/test_reset_upgrade_work_offline.py b/.claude/skills/parse-book/scripts/test_reset_upgrade_work_offline.py index f96e414..d63b455 100644 --- a/.claude/skills/parse-book/scripts/test_reset_upgrade_work_offline.py +++ b/.claude/skills/parse-book/scripts/test_reset_upgrade_work_offline.py @@ -2,6 +2,7 @@ """升格命令锁接线的纯离线测试：锁失败时禁止任何业务调用。""" import contextlib +import copy import pathlib import sys import unittest @@ -14,16 +15,298 @@ SCRIPT_DIR = pathlib.Path(__file__).resolve().parent sys.path.insert(0, str(SCRIPT_DIR)) import parse_upgrade as parse # noqa: E402 +import backup_upgrade_work as backup # noqa: E402 import reset_upgrade_work as reset # noqa: E402 from upgrade_work_lock import UpgradeWorkLockUnavailable # noqa: E402 +BACKUP_ID = "11111111-1111-4111-8111-111111111111" +CONFIRMATION_SHA = "c" * 64 +BACKUP_ARGS = [ + "--backup-dir", "/private/tmp", + "--backup-id", BACKUP_ID, + "--confirmation-sha", CONFIRMATION_SHA, +] + + def _lock_failure(*_args, **_kwargs): """模拟同书已有命令持锁。""" raise UpgradeWorkLockUnavailable(tenant_id=1, work_id=8) +class _Result: + """模拟 psycopg 查询结果，提供 reset 与备份快照使用的读取接口。""" + + def __init__(self, row=None, rows=None): + self.row = row + self.rows = list(rows or []) + + def fetchone(self): + return self.row + + def fetchall(self): + return list(self.rows) + + +class _CursorContext: + """记录 dict_row 游标请求；快照读取本身由备份模块替身返回。""" + + def __init__(self, conn): + self.conn = conn + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + +class _ResetConnection: + """有状态 fake DB：验证向量 SQL 边界，并保留不应被 reset 误伤的反例。""" + + def __init__(self, *, postcheck_failure=None, confirmed_draft_id=None): + self.postcheck_failure = postcheck_failure + self.commits = 0 + self.executions = [] + self.cursor_row_factories = [] + self.drafts = { + # 目标作品本轮活卡，以及上次 reset 已软删但向量仍活跃的旧卡。 + 101: {"tenant": 1, "work": 8, "source": "upgrade_book", "deleted": False}, + 102: {"tenant": 1, "work": 8, "source": "upgrade_book", "deleted": True}, + # 三个反例：其它作品、其它来源、其它租户均不得被本次 reset 误伤。 + 201: {"tenant": 1, "work": 9, "source": "upgrade_book", "deleted": False}, + 202: {"tenant": 1, "work": 8, "source": "parse_book", "deleted": False}, + 203: {"tenant": 2, "work": 8, "source": "upgrade_book", "deleted": False}, + } + self.embeddings = { + draft_id: { + "deleted": False, + "updater": "", + "entity_id": 9001 if draft_id == confirmed_draft_id else None, + } + for draft_id in self.drafts + } + self.window_statuses = ["done", "pending"] + self.alias_count = 3 + self.presence_count = 4 + self.card_state_count = 2 + self.audit_count = 5 + + def __enter__(self): + # 进入连接事务时保留快照，异常退出须像 psycopg 一样回滚全部已执行写入。 + self._transaction_snapshot = ( + copy.deepcopy(self.drafts), + copy.deepcopy(self.embeddings), + list(self.window_statuses), + self.alias_count, + self.presence_count, + self.card_state_count, + self.audit_count, + ) + return self + + def __exit__(self, exc_type, exc, traceback): + if exc_type is not None: + (self.drafts, self.embeddings, self.window_statuses, + self.alias_count, self.presence_count, + self.card_state_count, self.audit_count) = self._transaction_snapshot + return False + + def commit(self): + self.commits += 1 + + def cursor(self, *, row_factory=None): + """只允许实现代码显式请求 dict_row 快照游标。""" + + self.cursor_row_factories.append(row_factory) + return _CursorContext(self) + + @staticmethod + def _is_target_draft(draft): + return (draft["tenant"], draft["work"], draft["source"]) == (1, 8, "upgrade_book") + + def _active_target_vectors(self, *, unconfirmed_only=False): + matched = [] + for draft_id, embedding in self.embeddings.items(): + draft = self.drafts[draft_id] + if embedding["deleted"] or not self._is_target_draft(draft): + continue + if unconfirmed_only and embedding["entity_id"] is not None: + continue + matched.append(draft_id) + return matched + + @staticmethod + def _assert_vector_boundary(sql, params, *, require_active=True, require_unconfirmed=False): + """SQL 必须同时约束向量租户、卡租户、作品和来源。""" + + assert "e.tenant_id=%s" in sql + if require_active: + assert "e.deleted=false" in sql + assert "d.id=e.draft_id" in sql + assert "d.tenant_id=%s" in sql + assert "d.work_id=%s" in sql + assert "d.source_type=%s" in sql + assert params == (1, 1, 8, "upgrade_book") + if require_unconfirmed: + assert "e.entity_id is null" in sql + + def execute(self, sql, params=None): + normalized = " ".join(sql.split()).lower() + self.executions.append((normalized, params)) + if normalized.startswith("lock table"): + assert "share row exclusive mode" in normalized + for table in ( + "muse_knowledge_draft", "example_upgrade_window", + "example_upgrade_alias", "example_upgrade_presence", + "example_upgrade_card_state", "example_upgrade_audit", + "example_knowledge_embedding"): + assert table in normalized + return _Result() + if normalized.startswith("select title from muse_content_work"): + return _Result(("离线测试书",)) + if normalized.startswith("select count(*) from muse_knowledge_draft"): + count = sum( + not draft["deleted"] and self._is_target_draft(draft) + for draft in self.drafts.values() + ) + return _Result((count,)) + if (normalized.startswith("select count(*), count(*) filter") + or normalized.startswith("select count(*) filter")): + done = sum(status == "done" for status in self.window_statuses) + pending = sum(status == "pending" for status in self.window_statuses) + if "where status='done'" in normalized: + return _Result((len(self.window_statuses), done)) + return _Result((pending, len(self.window_statuses))) + if normalized.startswith("select count(*) from example_upgrade_alias"): + return _Result((self.alias_count,)) + if normalized.startswith("select count(*) from example_upgrade_presence"): + return _Result((self.presence_count,)) + if normalized.startswith("select count(*) from example_upgrade_card_state"): + return _Result((self.card_state_count,)) + if normalized.startswith("select count(*) from example_upgrade_audit"): + return _Result((self.audit_count,)) + if normalized.startswith("select count(*) from example_knowledge_embedding e"): + if "e.entity_id is not null" in normalized: + self._assert_vector_boundary(normalized, params, require_active=False) + count = sum( + embedding["entity_id"] is not None + for draft_id, embedding in self.embeddings.items() + if self._is_target_draft(self.drafts[draft_id]) + ) + return _Result((count,)) + self._assert_vector_boundary(normalized, params) + unconfirmed_only = "e.entity_id is null" in normalized + count = len(self._active_target_vectors(unconfirmed_only=unconfirmed_only)) + return _Result((count,)) + if normalized.startswith("update muse_knowledge_draft"): + for draft_id, draft in self.drafts.items(): + if self._is_target_draft(draft) and not draft["deleted"]: + if self.postcheck_failure == "cards" and draft_id == 101: + continue + draft["deleted"] = True + return _Result(None) + if normalized.startswith("update example_knowledge_embedding e"): + self._assert_vector_boundary(normalized, params, require_unconfirmed=True) + assert "set deleted=true, updater='upgrade-reset'" in normalized + targets = self._active_target_vectors(unconfirmed_only=True) + if self.postcheck_failure == "vectors": + targets = targets[:-1] + for draft_id in targets: + self.embeddings[draft_id].update(deleted=True, updater="upgrade-reset") + return _Result(None) + if normalized.startswith("update example_upgrade_window"): + self.window_statuses = ["pending"] * len(self.window_statuses) + if self.postcheck_failure == "windows": + self.window_statuses[-1] = "done" + return _Result(None) + if normalized.startswith("delete from example_upgrade_audit"): + self.audit_count = 1 if self.postcheck_failure == "audit" else 0 + return _Result(None) + if normalized.startswith("delete from example_upgrade_alias"): + self.alias_count = 1 if self.postcheck_failure == "alias" else 0 + return _Result(None) + if normalized.startswith("delete from example_upgrade_presence"): + self.presence_count = 1 if self.postcheck_failure == "presence" else 0 + return _Result(None) + if normalized.startswith("delete from example_upgrade_card_state"): + self.card_state_count = 1 if self.postcheck_failure == "state" else 0 + return _Result(None) + raise AssertionError(f"未覆盖的离线 SQL：{normalized}") + + +def _snapshot_domains(*, entity_id=None): + """构造七域最小一致快照；摘要全部调用备份模块既有规则。""" + + return { + "drafts": [ + {"id": 101, "tenant_id": 1, "work_id": 8, + "source_type": "upgrade_book", "deleted": False}, + {"id": 102, "tenant_id": 1, "work_id": 8, + "source_type": "upgrade_book", "deleted": True}, + ], + "windows": [ + {"id": 301, "tenant_id": 1, "work_id": 8, + "window_no": 1, "from_chapter": 1, "to_chapter": 2, + "status": "done", "deleted": False}, + ], + "aliases": [{"id": 401, "tenant_id": 1, "work_id": 8}], + "presence": [{"id": 501, "tenant_id": 1, "work_id": 8}], + "card_state": [{"draft_id": 101, "tenant_id": 1, "work_id": 8}], + "audits": [{"id": 601, "tenant_id": 1, "draft_id": 101}], + "embeddings": [ + {"id": 701, "tenant_id": 1, "draft_id": 101, + "entity_id": entity_id, "deleted": False}, + ], + } + + +def _code_files(): + """固定当前代码身份；reset execute 测试只改变待验证的单个 fileSha。""" + + return { + "backup_upgrade_work.py": "b" * 64, + "reset_upgrade_work.py": "r" * 64, + "parse_upgrade.py": "u" * 64, + "parse_llm.py": "p" * 64, + "embed_drafts.py": "e" * 64, + "upgrade_work_lock.py": "k" * 64, + "llm.py": "l" * 64, + "parse-book/SKILL.md": "s" * 64, + } + + +def _manifest_for(domains, *, code_files=None): + """按备份模块的排序、主键摘要和内容摘要构造已离线 verify 的清单替身。""" + + domain_manifest = {} + normalized = {} + for name, spec in backup.DOMAIN_SPECS.items(): + rows = backup.sort_rows(name, domains[name]) + normalized[name] = rows + domain_manifest[name] = { + "rowCount": len(rows), + "primaryKeySha": backup._primary_key_sha(spec, rows), + "contentSha": backup._content_sha(rows), + } + return { + "backup_id": BACKUP_ID, + "confirmationSha": CONFIRMATION_SHA, + "work": 8, + "tenant": 1, + "input": {"codeFiles": dict(code_files or _code_files())}, + "domains": domain_manifest, + }, normalized + + +@contextlib.contextmanager +def _lock_success(*_args, **_kwargs): + """离线成功锁，不连接 PostgreSQL。""" + + yield + + class UpgradeCommandLockOfflineTest(unittest.TestCase): """验证锁位于全部业务 SQL、写入和模型调用之前。""" @@ -66,7 +349,7 @@ class UpgradeCommandLockOfflineTest(unittest.TestCase): self._assert_reset_lock_failure_skips_database(execute=True) def _assert_reset_lock_failure_skips_database(self, *, execute): - args = ["--work-id", "8"] + (["--execute"] if execute else []) + args = ["--work-id", "8"] + (["--execute", *BACKUP_ARGS] if execute else []) with patch.object(reset, "upgrade_work_lock", side_effect=_lock_failure) as lock, \ patch.object(reset.psycopg, "connect") as connect: result = self.runner.invoke(reset.main, args) @@ -97,6 +380,229 @@ class UpgradeCommandLockOfflineTest(unittest.TestCase): self.assertEqual(result.exit_code, 0, result.output) self.assertEqual(events, ["lock-enter", "business", "lock-exit"]) + def _invoke_reset_execute(self, conn, *, manifest=None, current_domains=None, + backup_id=BACKUP_ID, confirmation_sha=CONFIRMATION_SHA, + lock_context=_lock_success, current_code_files=None): + """在锁、磁盘 verify 与同事务快照均为离线替身时调用 execute。""" + + default_manifest, default_domains = _manifest_for(_snapshot_domains()) + manifest = manifest or default_manifest + current_domains = current_domains or default_domains + args = [ + "--work-id", "8", "--execute", + "--backup-dir", "/private/tmp", + "--backup-id", backup_id, + "--confirmation-sha", confirmation_sha, + ] + with patch.object(reset, "upgrade_work_lock", lock_context), \ + patch.object(reset.psycopg, "connect", return_value=conn), \ + patch.object(backup, "verify_backup", return_value=manifest), \ + patch.object(backup, "capture_code_identity", return_value={ + "codeFiles": dict(current_code_files or manifest["input"]["codeFiles"]), + }), \ + patch.object(backup, "_read_snapshot", + return_value=("离线测试书", current_domains)): + return self.runner.invoke(reset.main, args) + + def test_reset_execute_requires_complete_backup_confirmation(self): + """execute 缺少任一备份确认参数时必须在取锁和连接数据库前失败。""" + + required = { + "--backup-dir": "/private/tmp", + "--backup-id": BACKUP_ID, + "--confirmation-sha": CONFIRMATION_SHA, + } + for missing in required: + args = ["--work-id", "8", "--execute"] + for option, value in required.items(): + if option != missing: + args.extend((option, value)) + with self.subTest(missing=missing), \ + patch.object(reset, "upgrade_work_lock") as lock, \ + patch.object(reset.psycopg, "connect") as connect: + result = self.runner.invoke(reset.main, args) + self.assertNotEqual(result.exit_code, 0) + self.assertIn(missing, result.output) + lock.assert_not_called() + connect.assert_not_called() + + def test_reset_preview_reports_all_target_active_vectors_without_writes(self): + """预览包含旧 reset 遗留向量，但不得执行任何 UPDATE/DELETE。""" + + conn = _ResetConnection() + with patch.object(reset, "upgrade_work_lock", _lock_success), \ + patch.object(reset.psycopg, "connect", return_value=conn): + result = self.runner.invoke(reset.main, ["--work-id", "8"]) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("软删活向量 2", result.output) + self.assertFalse(any(sql.startswith(("update ", "delete ")) + for sql, _params in conn.executions)) + self.assertEqual(conn.commits, 0) + + def test_reset_execute_soft_deletes_only_target_upgrade_vectors(self): + """执行只软删当前租户、作品、升格来源向量，并用明确 actor 留痕。""" + + conn = _ResetConnection() + result = self._invoke_reset_execute(conn) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("七域清理断言全部通过", result.output) + self.assertEqual(conn.commits, 1) + self.assertTrue(conn.executions[0][0].startswith("lock table")) + self.assertEqual(len(conn.cursor_row_factories), 1) + for draft_id in (101, 102): + self.assertTrue(conn.embeddings[draft_id]["deleted"]) + self.assertEqual(conn.embeddings[draft_id]["updater"], "upgrade-reset") + for draft_id in (201, 202, 203): + self.assertFalse(conn.embeddings[draft_id]["deleted"]) + self.assertEqual(conn.embeddings[draft_id]["updater"], "") + vector_writes = [sql for sql, _params in conn.executions + if "example_knowledge_embedding" in sql + and sql.startswith(("update ", "delete "))] + self.assertEqual(len(vector_writes), 1) + self.assertTrue(vector_writes[0].startswith("update ")) + self.assertIn("e.entity_id is null", vector_writes[0]) + + def test_reset_execute_verifies_backup_inside_advisory_lock(self): + """磁盘 verify 必须发生在同书 advisory lock 内，且早于业务连接。""" + + events = [] + + @contextlib.contextmanager + def tracked_lock(*_args, **_kwargs): + events.append("lock-enter") + try: + yield + finally: + events.append("lock-exit") + + manifest, domains = _manifest_for(_snapshot_domains()) + conn = _ResetConnection() + + def verified_manifest(_path): + self.assertEqual(events, ["lock-enter"]) + events.append("backup-verified") + return manifest + + with patch.object(reset, "upgrade_work_lock", tracked_lock), \ + patch.object(reset.psycopg, "connect", return_value=conn), \ + patch.object(backup, "verify_backup", side_effect=verified_manifest), \ + patch.object(backup, "capture_code_identity", return_value={ + "codeFiles": _code_files(), + }), \ + patch.object(backup, "_read_snapshot", + return_value=("离线测试书", domains)): + result = self.runner.invoke( + reset.main, ["--work-id", "8", "--execute", *BACKUP_ARGS] + ) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(events, ["lock-enter", "backup-verified", "lock-exit"]) + + def test_reset_execute_rejects_snapshot_drift_before_writes(self): + """七域任一摘要漂移必须在所有 reset 写语句前失败。""" + + manifest, domains = _manifest_for(_snapshot_domains()) + drifted = copy.deepcopy(domains) + drifted["aliases"].append({"id": 999, "tenant_id": 1, "work_id": 8}) + conn = _ResetConnection() + result = self._invoke_reset_execute( + conn, manifest=manifest, current_domains=drifted, + ) + + self.assertNotEqual(result.exit_code, 0) + self.assertIn("aliases", result.output) + self.assertEqual(conn.commits, 0) + self.assertEqual(len(conn.executions), 1) + self.assertTrue(conn.executions[0][0].startswith("lock table")) + + def test_reset_execute_rejects_missing_or_drifted_code_identity_before_database(self): + """七域不变时，旧 manifest 缺 reset 或 reset/parse 任一 fileSha 漂移仍须拒绝。""" + + for case in ("missing-reset", "reset-drift", "parse-drift"): + with self.subTest(case=case): + manifest_files = _code_files() + current_files = _code_files() + expected_name = "reset_upgrade_work.py" + if case == "missing-reset": + manifest_files.pop("reset_upgrade_work.py") + elif case == "reset-drift": + current_files["reset_upgrade_work.py"] = "x" * 64 + else: + current_files["parse_upgrade.py"] = "x" * 64 + expected_name = "parse_upgrade.py" + manifest, domains = _manifest_for( + _snapshot_domains(), code_files=manifest_files, + ) + conn = _ResetConnection() + result = self._invoke_reset_execute( + conn, + manifest=manifest, + current_domains=domains, + current_code_files=current_files, + ) + + self.assertNotEqual(result.exit_code, 0) + self.assertIn(expected_name, result.output) + self.assertEqual(conn.executions, []) + self.assertEqual(conn.commits, 0) + + def test_reset_execute_rejects_confirmed_entity_vectors_before_writes(self): + """目标 draft 任一 entity_id 非空时失败关闭，不能软删已确认实体向量。""" + + manifest, domains = _manifest_for(_snapshot_domains(entity_id=9001)) + conn = _ResetConnection(confirmed_draft_id=101) + result = self._invoke_reset_execute( + conn, manifest=manifest, current_domains=domains, + ) + + self.assertNotEqual(result.exit_code, 0) + self.assertIn("entity_id", result.output) + self.assertEqual(conn.commits, 0) + self.assertFalse(any(sql.startswith(("update ", "delete ")) + for sql, _params in conn.executions)) + self.assertFalse(conn.embeddings[101]["deleted"]) + + def test_reset_execute_rejects_confirmation_mismatch_before_database(self): + """backup_id 或 confirmationSha 不匹配时不得创建业务连接。""" + + manifest, _domains = _manifest_for(_snapshot_domains()) + with patch.object(reset, "upgrade_work_lock", _lock_success), \ + patch.object(reset.psycopg, "connect") as connect, \ + patch.object(backup, "verify_backup", return_value=manifest): + result = self.runner.invoke( + reset.main, + ["--work-id", "8", "--execute", "--backup-dir", "/private/tmp", + "--backup-id", BACKUP_ID, "--confirmation-sha", "wrong"], + ) + + self.assertNotEqual(result.exit_code, 0) + self.assertIn("confirmation-sha", result.output) + connect.assert_not_called() + + def test_reset_execute_rolls_back_when_any_postcheck_fails(self): + """七类提交前断言任一失败都必须回滚全部已执行 reset 写入。""" + + for failure in ("cards", "windows", "alias", "presence", "state", "audit", "vectors"): + with self.subTest(failure=failure): + conn = _ResetConnection(postcheck_failure=failure) + result = self._invoke_reset_execute(conn) + self.assertNotEqual(result.exit_code, 0) + self.assertIn("清后复核失败", result.output) + self.assertEqual(conn.commits, 0) + self.assertFalse(conn.drafts[101]["deleted"]) + self.assertTrue(conn.drafts[102]["deleted"]) + self.assertEqual(conn.window_statuses, ["done", "pending"]) + self.assertEqual( + (conn.alias_count, conn.presence_count, + conn.card_state_count, conn.audit_count), + (3, 4, 2, 5), + ) + for embedding in conn.embeddings.values(): + self.assertFalse(embedding["deleted"]) + self.assertEqual(embedding["updater"], "") + if __name__ == "__main__": unittest.main()