feat(parse-book): 范式卡回溯聚类去重(recluster)——同型近义卡嵌入初筛+M3终判归并母卡,全库7045→6327(并718/~10%)

recluster 子命令:连接三段(LLM期不持DB连接)/贪心最近母卡(避union-find巨簇)/软删可逆/dry-run门控;母卡累积实例+回溯归并审计(被并卡全文留档可回滚)。前向修复 cards():SIM_MERGE 0.85→0.80、最近邻≥0.75即送 merge_judge(堵住跨书近义卡从不判定的病根)。

五型放量:combat886→770/emotion1151→1027/craft1445→1326/scene_pattern1753→1578/trope1810→1626,并718。真实归并率~10%(向量投影74%多为同型词汇假近,merge_judge剪掉85-95%;创始人知情拍板全量0.75)。可逆性/完整性/软删/质量四项主代理亲验。

llm.py:睡窗日志边界显示 typo 修复(secs取整落在04:59:59被%H显示成非法边界04:00,+1s归整为05:00)。
This commit is contained in:
zizi 2026-07-17 06:10:04 +08:00
parent 9a513e79b8
commit 9c87f0dee9
3 changed files with 217 additions and 9 deletions

View File

@ -277,9 +277,11 @@ def chat_governed(prompt, model=DEFAULT_MODEL, system=None, max_tokens=512000,
if calls >= WINDOW_CALL_CAP: if calls >= WINDOW_CALL_CAP:
now2 = _now() now2 = _now()
secs = seconds_to_next_window(now2) secs = seconds_to_next_window(now2)
boundary = now2 + timedelta(seconds=secs) # 目标窗边界:secs 经 int() 截断可能落在边界前 <1s(如 04:59:59),+1s 归整到整分,
# 否则 %H 会把 04:59:59 显示成上一整点"04"、误导成非法边界(窗边界只有 00/05/10/15/20)
wake = (now2 + timedelta(seconds=secs + 1)).replace(second=0, microsecond=0)
print(f"[llm] 本窗 {wk} 已达 {calls} 次调用上限(≥{WINDOW_CALL_CAP})," print(f"[llm] 本窗 {wk} 已达 {calls} 次调用上限(≥{WINDOW_CALL_CAP}),"
f"睡 {secs // 60} 分钟到下一窗 {boundary:%H:00} 续跑", file=sys.stderr) f"睡 {secs // 60} 分钟到下一窗 {wake:%H:%M} 续跑", file=sys.stderr)
time.sleep(secs) time.sleep(secs)
continue # 醒来重读账本:跨过窗边界后是新窗,calls 归 0 continue # 醒来重读账本:跨过窗边界后是新窗,calls 归 0
# 2) 预算耗尽:本窗改用非 MiniMax 链;否则用全链 # 2) 预算耗尽:本窗改用非 MiniMax 链;否则用全链

View File

@ -41,8 +41,8 @@ IP_LEAK_WORDS = ("GN粒子", "太阳炉", "扎古", "高达", "夏亚", "阿姆
"脑量子波", "GN-Bit", "影印人", "殖装", "战功点", "宇宙世纪", "脑量子波", "GN-Bit", "影印人", "殖装", "战功点", "宇宙世纪",
"爆种", "次元兽", "伪造物主", "幽畸", "锐眼") "爆种", "次元兽", "伪造物主", "幽畸", "锐眼")
EMBED_MODEL = "Qwen/Qwen3-Embedding-8B" EMBED_MODEL = "Qwen/Qwen3-Embedding-8B"
SIM_MERGE = 0.85 # 嵌入判重:≥该值触发 M3 归并判定(阈值未校准,实战收集中) SIM_MERGE = 0.80 # 强候选线:≥该值=高置信近义(前向修复后 0.75+ 均送 merge_judge,本值只作档位标注)
SIM_MARK = 0.75 # [MARK, MERGE) 区间只标记不判定(校准带宽,审核环可见) SIM_MARK = 0.75 # 候选线:≥该值即送 merge_judge 终判(cards 前向判重 + recluster 回溯聚类 同一判据)
def norm_scaffold(data: dict) -> dict: def norm_scaffold(data: dict) -> dict:
@ -454,7 +454,10 @@ def cards(work_id, from_order, file_, model):
WHERE e.tenant_id=%s AND e.deleted=FALSE AND d.deleted=FALSE WHERE e.tenant_id=%s AND e.deleted=FALSE AND d.deleted=FALSE
AND d.source_type='parse_book' AND d.source_type='parse_book'
ORDER BY score DESC LIMIT 1""", (json.dumps(vec), TENANT)).fetchone() ORDER BY score DESC LIMIT 1""", (json.dumps(vec), TENANT)).fetchone()
if top and top[2] >= SIM_MERGE: # 前向修复(2026-07-17):不再"只有 ≥SIM_MERGE 才判、0.75-0.85 仅打标记"——
# 最近邻 ≥SIM_MARK(0.75) 即送 merge_judge,与 recluster 回溯聚类同一判据。
# WHY:跨书 0.75-0.85 的近义卡此前从不真正判定、只挂"未达判定线",是重复卡沉积的病根。
if top and top[2] >= SIM_MARK:
old_id, old_payload, score = top old_id, old_payload, score = top
verdict, why = merge_judge(payload, old_payload, model) verdict, why = merge_judge(payload, old_payload, model)
if verdict == "merge": if verdict == "merge":
@ -468,11 +471,10 @@ def cards(work_id, from_order, file_, model):
(Jsonb(old_payload), ACTOR, old_id)) (Jsonb(old_payload), ACTOR, old_id))
merged.append(f"《{payload['名称']}》并入已有卡#{old_id}《{old_payload.get('名称')}》({score:.2f}) {why}") merged.append(f"《{payload['名称']}》并入已有卡#{old_id}《{old_payload.get('名称')}》({score:.2f}) {why}")
continue continue
# keep 留痕;档位区分强候选(≥SIM_MERGE)/扩展候选(≥SIM_MARK),供审核判读近似度
payload["判重"] = {"相似卡": old_payload.get("名称"), "相似度": round(float(score), 4), payload["判重"] = {"相似卡": old_payload.get("名称"), "相似度": round(float(score), 4),
"判定": "keep", "依据": why} "判定": "keep", "档": "强候选" if score >= SIM_MERGE else "扩展候选",
elif top and top[2] >= SIM_MARK: "依据": why}
payload["判重"] = {"相似卡": top[1].get("名称"), "相似度": round(float(top[2]), 4),
"判定": "未达判定线"}
cid = f"parse-{work_id}-w{from_order}-{i}-" + hashlib.sha256( cid = f"parse-{work_id}-w{from_order}-{i}-" + hashlib.sha256(
json.dumps(payload, ensure_ascii=False, sort_keys=True).encode()).hexdigest()[:8] json.dumps(payload, ensure_ascii=False, sort_keys=True).encode()).hexdigest()[:8]
row = conn.execute( row = conn.execute(
@ -508,6 +510,203 @@ def cards(work_id, from_order, file_, model):
click.echo(f" [拒] {r['卡']}: {'; '.join(r['原因'])}") click.echo(f" [拒] {r['卡']}: {'; '.join(r['原因'])}")
def _judge_view(payload):
"""送 merge_judge 的"手法身份"视图:只留 型/名称/摘要/字段(判"是否同一手法"的全部依据)。
WHY 裁剪:审核(三角色判词)/实例/出处/判重 是噪声——尤其母卡累积实例与「回溯归并审计」后
payload 会塞进多张别的卡全文,直接喂 merge_judge 既爆 token 又把判定带偏;裁到身份四件套后
判定输入稳定(母卡吸并再多,身份不变)、便宜、聚焦手法本身。"""
return {"型": payload.get("型"), "名称": payload.get("名称"),
"一句话摘要": payload.get("一句话摘要"), "字段": payload.get("字段") or {}}
def _mother_key(payload, cid):
"""母卡优先序(best-first 排序键):审核判定优 > 均分高 > 实例多 > id 小。
WHY:让存活下来的母卡是该手法的最佳代表——下游审核/导出以母卡为规范样本;实例多者是更厚的
证据锚点;id 兜底保证确定性可复现。审核缺失(他型可能未审)排在最后,不抢占母卡位。"""
r = payload.get("审核") or {}
rank = {"pass": 0, "revise": 1, "reject": 2}.get(r.get("判定"), 3)
return (rank, -(r.get("均分") or 0), -len(payload.get("实例") or []), cid)
def _absorb(mother, loser, sim, why):
"""把输家并入母卡(内存态,写库延后到写段统一落):输家实例(带书名归属)追加进母卡实例数组
+ 记「回溯归并审计」(相似度/判定依据/被归并卡全文——全文留痕保证归并可逆)。"""
book = (loser.get("出处") or {}).get("书名") or ""
add = [dict(ins, 书=book) for ins in (loser.get("实例") or [])] # 跨书归属:每条实例标来源书
mother["实例"] = (mother.get("实例") or []) + add
mother.setdefault("回溯归并审计", []).append(
{"相似度": round(float(sim), 4), "判定依据": why, "被归并卡": loser})
def _sim_band(s):
"""附着边相似度分档(供 dry-run 报告:不同档 LLM 剪枝率差异大,同型词汇共享令 0.75-0.80 多为假近)。"""
return "≥0.85" if s >= 0.85 else ("0.80–0.85" if s >= 0.80 else "0.75–0.80")
@cli.command()
@click.option("--type", "ptype", required=True, type=click.Choice(sorted(PATTERN_TYPES)),
help="回溯聚类的范式型(一次只跑一型)")
@click.option("--dry-run", is_flag=True, help="只算不写:报簇结构/计划归并/N→M + 抽样 merge_judge 判定")
@click.option("--limit", type=int, default=0, help="只取前 N 张卡(调试用,0=全型)")
@click.option("--sample", type=int, default=10, show_default=True,
help="dry-run 下抽样真跑 merge_judge 的候选对数(按相似度档分层取,露出各档合并率)")
@click.option("--model", default="MiniMax-M3", show_default=True, help="归并判定用模型")
def recluster(ptype, dry_run, limit, sample, model):
"""回溯聚类去重:把同型近义卡(嵌入初筛 + merge_judge 终判)归并到母卡。
病根:现行 cards() 入库只比最近 1 张、阈值偏高、按书顺序入库→跨书 0.75-0.85 的近义卡从不送判定。
本命令一次回溯:同型全量按余弦相似度成簇,逐对经 LLM 确认后把输家并入母卡(软删、可逆)。
连接三段式(血泪教训:DB 事务里夹 LLM,长空转会被 Tailscale 掐断整批崩):
读段 短连接取全型卡 + 候选对(pgvector self-join,纯 SQL 无 LLM),读完即释放;
算段 本地贪心成簇 + merge_judge 逐对终判(此阶段 recluster 不持任何 DB 连接,
merge_judge 走 chat_governed 自持额度账本短连接,互不干扰);
写段 全新短连接批量落库(母卡累积 payload / 输家卡 deleted / 输家嵌入 deleted),一次提交。
双保险:embedding 只初筛出候选,是否同一手法一律由 merge_judge 定夺,拿不准 keep(宁重复不误并)。"""
# ── 读段:短连接读完即释放 ──
with psycopg.connect(DSN) as rconn:
rows = rconn.execute(
"""SELECT d.id, d.draft_payload FROM muse_knowledge_draft d
WHERE d.tenant_id=%s AND d.source_type='parse_book' AND d.deleted=FALSE
AND d.draft_payload->>'型'=%s ORDER BY d.id""",
(TENANT, ptype)).fetchall()
if limit:
rows = rows[:limit]
cards = {cid: p for cid, p in rows}
ids = set(cards)
# 候选对:同型嵌入 self-join,余弦相似度 ≥SIM_MARK。纯 SQL(无 LLM);pgvector 一次算完 O(N²),
# combat 886 张实测 ~3s。取回后本地成簇,判定/写库阶段不再回这条连接。
pair_rows = rconn.execute(
"""WITH emb AS (
SELECT e.draft_id, e.embedding FROM example_knowledge_embedding e
JOIN muse_knowledge_draft d ON d.id=e.draft_id
WHERE e.tenant_id=%s AND e.deleted=FALSE AND d.deleted=FALSE
AND d.source_type='parse_book' AND d.draft_payload->>'型'=%s)
SELECT a.draft_id, b.draft_id, 1-(a.embedding<=>b.embedding) AS sim
FROM emb a JOIN emb b ON a.draft_id<b.draft_id
WHERE 1-(a.embedding<=>b.embedding) >= %s""",
(TENANT, ptype, SIM_MARK)).fetchall()
# 邻接表(id→[(邻卡,相似度)]);--limit 调试时过滤掉越界 id
adj = {}
for a, b, s in pair_rows:
if a in ids and b in ids:
s = float(s)
adj.setdefault(a, []).append((b, s))
adj.setdefault(b, []).append((a, s))
n = len(ids)
if not n:
click.echo(f"[recluster·{ptype}] 无活卡")
return
# ── 算段:贪心「最近母卡」聚类(best-first)──
# 为何不用 union-find:0.75 阈下同型卡词汇高度共享,单链传递会把几乎整型并成一个巨簇
#(combat 实测 38665 对 / 817 张连通)。贪心「只附着到最近的一个母卡」天然避免链式膨胀,
# 且判定次数最省:每张非母卡至多 1 次 merge_judge(判它 vs 最近母卡)。
order = sorted(ids, key=lambda cid: _mother_key(cards[cid], cid))
leaders, leader_set = [], set()
plan = [] # 计划/已确认归并:(母卡id, 输家id, 相似度)
def nearest_leader(cid):
"""该卡邻居中"已是母卡"的最近一个(sim 最大);无则 None。"""
best = None
for nbr, s in adj.get(cid, []):
if nbr in leader_set and (best is None or s > best[1]):
best = (nbr, s)
return best
if dry_run:
# 纯向量投影(几乎不烧 LLM):每卡附着到最近母卡=一条计划归并,无最近母卡=自成母卡。
# 这是归并的「上限」——实跑每对还要过 merge_judge,keep 者不并→真实存活更多。
for cid in order:
nb = nearest_leader(cid)
if nb:
plan.append((nb[0], cid, nb[1]))
else:
leaders.append(cid)
leader_set.add(cid)
else:
# 实跑:每次附着都由 merge_judge 终判(受控点②)。此阶段 recluster 不持 DB 连接。
for cid in order:
nb = nearest_leader(cid)
if nb:
lid, sim = nb
verdict, why = merge_judge(_judge_view(cards[cid]), _judge_view(cards[lid]), model)
if verdict == "merge":
_absorb(cards[lid], cards[cid], sim, why) # 内存改母卡,写段统一落库
plan.append((lid, cid, sim))
continue
leaders.append(cid) # 无最近母卡 或 判 keep → 自成母卡
leader_set.add(cid)
# ── 写段:全新短连接批量落库(红线:软删,绝不物理删)──
if not dry_run and plan:
movers = {lid for lid, _lo, _s in plan} # 被吸并过的母卡(去重,每张只写一次最终态)
with psycopg.connect(DSN) as wconn:
for lid in movers:
wconn.execute("UPDATE muse_knowledge_draft SET draft_payload=%s, updater=%s WHERE id=%s",
(Jsonb(cards[lid]), ACTOR, lid))
for _lid, loser_id, _s in plan:
wconn.execute("UPDATE muse_knowledge_draft SET deleted=TRUE, updater=%s WHERE id=%s",
(ACTOR, loser_id)) # 输家卡软删
wconn.execute(
"""UPDATE example_knowledge_embedding SET deleted=TRUE, updater=%s
WHERE tenant_id=%s AND draft_id=%s AND deleted=FALSE""",
(ACTOR, TENANT, loser_id)) # 输家嵌入软删:未来判重/检索不再命中已并走的卡
wconn.commit()
# ── 报告 ──
from collections import Counter
per_mother = Counter(lid for lid, _lo, _s in plan) # 母卡 → 吸并数
band = Counter(_sim_band(s) for _l, _lo, s in plan) # 附着边相似度分档
surv = n - len(plan) # 存活母卡数
tag = "dry-run·向量投影上限" if dry_run else "实跑·LLM确认"
click.echo(f"[recluster·{ptype}·{tag}] 活卡 {n} → 存活 {surv}"
f"(归并 {len(plan)},缩减 {len(plan) / n:.0%})")
if dry_run:
click.echo(" 注:向量投影是归并上限;实跑每对经 merge_judge 终判,keep 者不并→实际存活更多。")
click.echo(" 附着边相似度:" + " ".join(f"{k}={band.get(k, 0)}" for k in ("≥0.85", "0.80–0.85", "0.75–0.80")))
click.echo(f" 含归并的簇 {len(per_mother)}(另有 {surv - len(per_mother)} 张孤卡母卡);最大簇 top5:")
for mid, cnt in per_mother.most_common(5):
click.echo(f" #{mid}《{cards[mid].get('名称')}》聚 {cnt + 1} 张(母1+并{cnt})")
# dry-run 抽样:按相似度档「分层」各取若干真跑 merge_judge——展示判定质量 + 分档外推真实归并数。
# WHY 分层而非等距:附着边多数落在 0.75-0.80(同型词汇高度共享的"假近"区,LLM 几乎全 keep),
# 等距抽样会几乎全落该档、把高档才有的真归并淹没成 0;分层能露出每档合并率,据此把向量上限收敛到现实预估。
if dry_run and plan and sample > 0:
by_band = {"≥0.85": [], "0.80–0.85": [], "0.75–0.80": []}
for e in sorted(plan, key=lambda x: x[2], reverse=True):
by_band[_sim_band(e[2])].append(e)
live_bands = [b for b in by_band if by_band[b]]
per = max(1, sample // max(1, len(live_bands))) # 每档配额(档内等距取,覆盖该档相似度跨度)
picks = []
for b in live_bands:
es = by_band[b]
k = min(per, len(es))
idxs = [round(i * (len(es) - 1) / (k - 1)) for i in range(k)] if k > 1 else [0]
picks += [es[i] for i in dict.fromkeys(idxs)]
click.echo(f" —— 抽样 merge_judge 判定({len(picks)} 对,按相似度档分层)——")
band_hit = {b: [0, 0] for b in by_band} # 档 → [merge数, 抽样数]
for lid, loser_id, sim in picks:
verdict, why = merge_judge(_judge_view(cards[loser_id]), _judge_view(cards[lid]), model)
bd = _sim_band(sim)
band_hit[bd][0] += verdict == "merge"
band_hit[bd][1] += 1
click.echo(f" [{verdict}] {sim:.3f}({bd}) 输《{cards[loser_id].get('名称')}》→ "
f"母《{cards[lid].get('名称')}》:{why}")
# 分档外推:估真实归并 = Σ(该档附着边数 × 该档抽样 merge 率)。样本小,仅供数量级判断。
est = sum(band.get(b, 0) * (band_hit[b][0] / band_hit[b][1] if band_hit[b][1] else 0) for b in by_band)
for b in by_band:
hit, tot = band_hit[b]
click.echo(f" 档 {b}: 抽样 merge {hit}/{tot},本档附着边 {band.get(b, 0)}"
f" → 估归并 ~{band.get(b, 0) * (hit / tot if tot else 0):.0f}")
click.echo(f" 分档外推真实归并 ~{est:.0f}({n}→~{n - est:.0f});样本小仅供数量级,放量以实跑逐对确认为准")
elif not dry_run:
for lid, loser_id, _s in plan[:20]:
click.echo(f" [归并] 《{cards[loser_id].get('名称')}》并入 #{lid}《{cards[lid].get('名称')}》")
if len(plan) > 20:
click.echo(f" …另 {len(plan) - 20} 条归并")
@cli.command() @cli.command()
@click.option("--work-id", type=int) @click.option("--work-id", type=int)
def progress(work_id): def progress(work_id):

View File

@ -3,6 +3,13 @@
> 本文档=compact 后接续工作的详细锚点。概览见 README §九「批9c」条目;本文档给可执行细节。 > 本文档=compact 后接续工作的详细锚点。概览见 README §九「批9c」条目;本文档给可执行细节。
> 任务完成后按仓库约定蒸馏进 README §九 并删本文档。 > 任务完成后按仓库约定蒸馏进 README §九 并删本文档。
## 进展快照(2026-07-17 凌晨,最新——compact 恢复优先读这节)
- **item4 额度系统**:✅ 已实现+主代理亲验+**已提交 `9a513e7`**(chat_governed 统一治理,五书降级链上收,账本 example_llm_quota)。
- **范式卡去重(回溯聚类)**:recluster 子命令已实现+亲验(连接三段/贪心最近母卡/软删可逆/dry-run 门控)。**combat 已放量:886→770(并 116,缩减 13%)**,完整性/可逆性/质量四项亲验通过(116 输家软删+116 嵌入软删+83 母卡带回溯审计,被并卡全文留档可回滚)。**其余四型(emotion/craft/scene_pattern/trope)后台放量中**(/tmp/muse-recluster-rest.log,任务 bl8fwgyjc,受额度系统自动跨窗,约 5550 次判定、跑几小时)。真实归并率 ~13%(非预想 50%,创始人已知情拍板"全量 0.75")。cards() 前向修复(SIM_MERGE 0.85→0.80、0.75+ 均送判定)已随附。
- **升格卡改造**:4 决定已拍(①里程碑=结构化对象 章/台阶/周期 ②补齐=五型全补 ③character 一起改 ④存量迁移=人工精确化;+主代理定 演变概括独立字段),记入 `docs/2026-07-16-升格卡改造设计.md` §十。fable5 已实现**前向代码**+主代理亲验:6 个 schema yaml(演变历程 `[detection,extraction]`/演变概括·前身·后继 `true`/character 成长弧线回归"未来计划")、parse_upgrade.py(里程碑对象合并去重按真实章号+撤销靠 `_win` 键+当前态干净纪律+语义判重默认关)、迁移脚本 migrate_upgrade_windows.py(**确认只读零写库**)、read-context/detect SKILL.md 补说明。
- **待办(gated,需创始人 go)**:①范式卡四型放量完成后核总量+提交框架代码;②升格卡三步动真格——重跑 seed_schemas 灌新字段合同 / 跑真迁移(人工精确化:无内嵌章号处再抽原文定章)/ 开语义判重(需先 embed 落库);③升格卡语义判重读 `type` 键的兼容性主代理启用前单独核。
## 一、批9c 已真收官(五书全量重审) ## 一、批9c 已真收官(五书全量重审)
- **五书 b9-full-v2 审核库内校验 ALLOK**:星环 1525 / 机动 780 / 机战 2336 / 超神 1441 / 深空 963,全书全覆盖。 - **五书 b9-full-v2 审核库内校验 ALLOK**:星环 1525 / 机动 780 / 机战 2336 / 超神 1441 / 深空 963,全书全覆盖。