130 lines
6.0 KiB
Python
130 lines
6.0 KiB
Python
#!/usr/bin/env python3
|
||
"""存量回洗（一次性，抽检 2026-07-15 后）：
|
||
① 追加字段前缀堆叠/[本窗]残留清洗 + 同文去重（34+9 张卡）
|
||
② 关系卡顶层演变轨迹迁移进 字段.演变轨迹（双份分裂修复）
|
||
③ 出场章全量重建：卡名+别名 机械预扫全书正文（零 AI，比增量准）
|
||
④ 脏别名清理：备忘录行/括号注释/错认两行（橡树妖/雷行天下）
|
||
⑤ 占位值清理：字段值含「续写不可见」的删字段
|
||
"""
|
||
import json
|
||
import re
|
||
import sys
|
||
|
||
import psycopg
|
||
|
||
DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example"
|
||
"?keepalives=1")
|
||
TENANT = 1
|
||
PREFIX_RE = re.compile(r"^(?:\[[窗本][^\]]{0,6}\]\s*)+") # 含 [窗本窗] 等模型自造变体
|
||
WIN_NUMS = re.compile(r"\[窗(\d+)\]")
|
||
|
||
# 7 型合同合法 key（越合同 key 清洗用，运行时从库加载）
|
||
VALID_KEYS = {}
|
||
|
||
|
||
def load_valid_keys(conn):
|
||
for (sk, snap) in conn.execute(
|
||
"""SELECT s.schema_key, v.field_contract_snapshot FROM muse_meta_schema s
|
||
JOIN muse_meta_schema_version v ON v.id=s.active_version_id
|
||
WHERE s.tenant_id=1 AND s.schema_key IN
|
||
('character','location','item','faction','power_system','event','character_relation')"""):
|
||
VALID_KEYS[sk] = {f["key"] for f in snap.get("特有字段", [])} | {"一句话摘要", "演变轨迹"}
|
||
|
||
|
||
def clean_list(items):
|
||
"""剥堆叠前缀+同文去重；窗号取原条目最内层（最后一个），无窗号保持裸条目。"""
|
||
out, seen = [], set()
|
||
for it in items:
|
||
s = str(it)
|
||
core = PREFIX_RE.sub("", s).strip()
|
||
if not core or core in seen:
|
||
continue
|
||
seen.add(core)
|
||
nums = WIN_NUMS.findall(s[:len(s) - len(core)])
|
||
out.append(f"[窗{nums[-1]}] {core}" if nums else core)
|
||
return out
|
||
|
||
|
||
def rebuild_presence(chapters, names):
|
||
"""全书逐章扫描名字集合 → 出场章号列表。"""
|
||
hits = set()
|
||
for order_no, text in chapters:
|
||
if any(nm in text for nm in names):
|
||
hits.add(order_no)
|
||
return sorted(hits)
|
||
|
||
|
||
def main():
|
||
stats = {"前缀清洗卡": 0, "关系迁移卡": 0, "出场章重建卡": 0, "删别名行": 0, "删占位字段": 0}
|
||
with psycopg.connect(DSN) as conn:
|
||
load_valid_keys(conn)
|
||
for work_id in (4, 8):
|
||
chapters = conn.execute(
|
||
"""SELECT c.order_no, b.content_text FROM muse_content_chapter c
|
||
JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE
|
||
WHERE c.tenant_id=%s AND c.work_id=%s AND c.deleted=FALSE
|
||
ORDER BY c.order_no""", (TENANT, work_id)).fetchall()
|
||
cards = conn.execute(
|
||
"""SELECT id, draft_payload FROM muse_knowledge_draft
|
||
WHERE tenant_id=%s AND work_id=%s AND source_type='upgrade_book'
|
||
AND deleted=FALSE""", (TENANT, work_id)).fetchall()
|
||
for did, p in cards:
|
||
changed = False
|
||
fields = p.get("字段") or {}
|
||
# ② 关系卡顶层演变迁移
|
||
if p.get("type") == "character_relation" and "演变轨迹" in p:
|
||
legacy = p.pop("演变轨迹")
|
||
base = fields.get("演变轨迹") or []
|
||
fields["演变轨迹"] = (base if isinstance(base, list) else [base]) + \
|
||
(legacy if isinstance(legacy, list) else [legacy])
|
||
p["字段"] = fields
|
||
changed = True
|
||
stats["关系迁移卡"] += 1
|
||
# ①⑤ 数组前缀清洗 + 占位字段删除 + 越合同畸形 key 清洗（窗29黄蜂针实测）
|
||
vk = VALID_KEYS.get(p.get("type"))
|
||
for k in list(fields.keys()):
|
||
if vk is not None and k not in vk:
|
||
del fields[k]
|
||
changed = True
|
||
stats["删越合同key"] = stats.get("删越合同key", 0) + 1
|
||
continue
|
||
v = fields[k]
|
||
if isinstance(v, str) and "续写不可见" in v:
|
||
del fields[k]
|
||
changed = True
|
||
stats["删占位字段"] += 1
|
||
elif isinstance(v, list):
|
||
nv = clean_list(v)
|
||
if nv != v:
|
||
fields[k] = nv
|
||
changed = True
|
||
stats["前缀清洗卡"] += 1
|
||
# ③ 出场章全量重建（实体卡；关系卡无出场章语义）
|
||
if p.get("type") != "character_relation":
|
||
names = {p.get("名称", "")} | set(p.get("别名") or [])
|
||
names = {n for n in names if n and len(n) >= 2} # 单字名误命中太多，跳过
|
||
if names:
|
||
nv = rebuild_presence(chapters, names)
|
||
if nv and nv != p.get("出场章"):
|
||
p["出场章"] = nv
|
||
changed = True
|
||
stats["出场章重建卡"] += 1
|
||
if changed:
|
||
conn.execute(
|
||
"UPDATE muse_knowledge_draft SET draft_payload=%s, updater='rewash' WHERE id=%s",
|
||
(json.dumps(p, ensure_ascii=False), did))
|
||
# ④ 脏别名：备忘录/括号/超长 + 抽检定点两行错认
|
||
r = conn.execute(
|
||
"""UPDATE example_upgrade_alias SET deleted=TRUE, updater='rewash'
|
||
WHERE tenant_id=%s AND deleted=FALSE AND (
|
||
alias ~ '[（()）。，：]' OR length(alias) > 12
|
||
OR canonical_name ~ '[（()）。，：]'
|
||
OR alias IN ('橡树妖','雷行天下'))""", (TENANT,))
|
||
stats["删别名行"] = r.rowcount
|
||
conn.commit()
|
||
print(json.dumps(stats, ensure_ascii=False))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|