From f5965577d6243ca33aeb86fbc3d54f3430ea00e8 Mon Sep 17 00:00:00 2001 From: zizi Date: Tue, 21 Jul 2026 13:38:18 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D:=20=E8=8C=83=E5=BC=8F?= =?UTF-8?q?=E6=8A=BD=E5=8F=96=E5=A4=B1=E8=B4=A5=E5=85=B3=E9=97=AD=E5=B9=B6?= =?UTF-8?q?=E9=98=BB=E6=96=AD=E7=BC=BA=E7=AA=97=E5=81=87=E7=BB=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .claude/skills/parse-book/SKILL.md | 2 + .../skills/parse-book/scripts/parse_llm.py | 23 +++- .../parse-book/scripts/parse_outline.py | 129 ++++++++++++++---- .../scripts/test_parse_outline_offline.py | 97 +++++++++++++ 4 files changed, 224 insertions(+), 27 deletions(-) create mode 100644 .claude/skills/parse-book/scripts/test_parse_outline_offline.py diff --git a/.claude/skills/parse-book/SKILL.md b/.claude/skills/parse-book/SKILL.md index 4145aa7..27aa9df 100644 --- a/.claude/skills/parse-book/SKILL.md +++ b/.claude/skills/parse-book/SKILL.md @@ -60,6 +60,8 @@ disable-model-invocation: true **作品面升格执行器 `scripts/parse_upgrade.py`（与上面范式拆书管线并行的另一条线,命令 `windows`/`run`/`status`）**：把参考书正文按窗抽成「会随剧情长大的实体卡」(升格卡,`source_type=upgrade_book`),设计见 `docs/2026-07-16-升格卡改造设计.md`。里程碑除真实章号外，模型输出必须临时携带所标章节正文短引；系统机械核验后删除证据，错章、缺证据或改写证据均拒收入库并留审计。`run` 默认**不发嵌入**;加 `--semantic-dedup` 开语义判重(治改名/跨型漏并)时,每窗按**读→算→写三段式短连接**跑——观察/近邻召回/M3 终判都在**无长连接**段发 LLM 与嵌入 HTTP(不再持窗级连接跨调用存活),写段只查预判结果落库;并**边抽边嵌**：本窗新建/更新卡在窗事务提交后增量嵌入落库(软删旧向量+upsert 新行),**后窗即可语义召回前窗刚长成的卡**,不再依赖"同书须预先全量 embed"。嵌段失败只告警、不牵连已 done 的窗。 +**范式线失败关闭（2026-07-21 假绿复盘后现行）**：`parse_outline.py window`、`parse_llm.py cards`、`parse_outline.py check` 是同一本作品的顺序流水线。普通网络/429/5xx 由 `chat_governed` 在单模型内短重试后沿 `MiniMax-M3 → MiniMax-M2.7 → glm-5.2 → deepseek-v4-flash` 降级；整条链耗尽，或功能层结构输出有限重试仍失败，必须**立即停止当前作品并保留断点**，禁止跳窗/跳型继续。`cards` 和 `check` 开始前机械验证大纲窗对全书章域无头部、中部、尾部缝隙；任一缝隙非零退出。补插窗后所有终检按 `from_order/to_order` 绝对章号排序并重新编号阶段，`window_no` 只作运行展示号，不得参与时序判断。 + ## 步骤(自底向上,与创作期规划的自顶向下互为镜像) 1. 静态分章:import skill(规则,LLM 不参与); diff --git a/.claude/skills/parse-book/scripts/parse_llm.py b/.claude/skills/parse-book/scripts/parse_llm.py index ef8ae1c..0be780c 100644 --- a/.claude/skills/parse-book/scripts/parse_llm.py +++ b/.claude/skills/parse-book/scripts/parse_llm.py @@ -27,6 +27,7 @@ import psycopg # 敏感/额度降级链已上收 llm.chat_governed（全局统一治理），本模块不再自持降级链 sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "llm" / "scripts")) from llm import chat_governed, extract_json # noqa: E402 +from parse_outline import ensure_outline_coverage # noqa: E402 DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example" "?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3") @@ -481,8 +482,21 @@ def cards(work_id, from_order, model, redo): args.append(from_order) wins = conn.execute(sql + " ORDER BY from_order", args).fetchall() if not wins: - click.echo("无大纲窗行——先跑 parse_outline.py window（窗是出卡的切分依据）") - return + raise click.ClickException("无大纲窗行——先跑 parse_outline.py window（窗是出卡的切分依据）") + if from_order is None: + with psycopg.connect(DSN) as conn: + bounds = conn.execute( + """SELECT min(order_no), max(order_no) FROM muse_content_chapter + WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE""", + (TENANT, work_id), + ).fetchone() + if bounds and bounds[0] is not None: + ensure_outline_coverage( + [(row[0], row[1]) for row in wins], + bounds[0], + bounds[1], + context="cards 前", + ) total_in = total_out = 0 for a, b, stage_ol, wno in wins: with psycopg.connect(DSN) as conn: @@ -542,7 +556,10 @@ def cards(work_id, from_order, model, redo): all_cards += cs click.echo(f" 批[{t}] 线索{len(hs)} → 卡{len(cs)}") except RuntimeError as e: - click.echo(f" 批[{t}] M3 失败: {e}（本批线索丢失，重跑 --redo 可补）") + # m3_json 已完成调用/结构重试；继续会让该型线索永久丢失并把整窗误标 done。 + raise click.ClickException( + f"win#{a}–{b} 批[{t}] 连续重试仍失败，停止当前作品并保留断点：{str(e)[:180]}" + ) from e total_in += win_in total_out += win_out _, out = ingest("cards", work_id, a, {"cards": all_cards}, keyflag="--from-order") diff --git a/.claude/skills/parse-book/scripts/parse_outline.py b/.claude/skills/parse-book/scripts/parse_outline.py index 7db7c0b..a47c33e 100644 --- a/.claude/skills/parse-book/scripts/parse_outline.py +++ b/.claude/skills/parse-book/scripts/parse_outline.py @@ -30,6 +30,7 @@ def chat_degrade(prompt, model): DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example" "?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3") TENANT, ACTOR = 1, "1" +MAX_WINDOW_GENERATION_ATTEMPTS = 2 WINDOW_PROMPT = """你是长篇小说结构分析员。下面是《{title}》第 {a}–{b} 章的**逐章细纲**与**正文**。 把这一段（约 {wc} 字）聚合成一份「阶段大纲」，回答结构层的问题（细纲已有的章内细节不要复述）： @@ -78,6 +79,79 @@ def load_chapters(conn, work_id, need_scaffold=True): (TENANT, work_id)).fetchall() +def outline_coverage_gaps(windows, first_order, last_order): + """计算指定章域内未被任何大纲窗覆盖的连续章段。""" + + if first_order > last_order: + return [] + cursor = first_order + gaps = [] + for raw_start, raw_end in sorted(windows): + if raw_end < first_order or raw_start > last_order: + continue + start = max(first_order, raw_start) + end = min(last_order, raw_end) + if start > cursor: + gaps.append((cursor, start - 1)) + cursor = max(cursor, end + 1) + if cursor > last_order: + break + if cursor <= last_order: + gaps.append((cursor, last_order)) + return gaps + + +def ensure_outline_coverage(windows, first_order, last_order, *, context): + """大纲窗覆盖不完整时失败关闭，禁止 cards/check 在缺窗上制造假绿。""" + + gaps = outline_coverage_gaps(windows, first_order, last_order) + if not gaps: + return + rendered = [f"{start}-{end}" for start, end in gaps] + raise click.ClickException( + f"{context}大纲窗覆盖存在缝隙章段 {rendered}；先用 window --from/--to 补窗，禁止继续出卡或终检" + ) + + +def generate_outline_with_retry(prompt, model, *, max_attempts=MAX_WINDOW_GENERATION_ATTEMPTS): + """有限重试大纲结构输出；治理链或结构重试耗尽后立即停止当前作品。""" + + last_error = "输出为空" + for _attempt in range(1, max_attempts + 1): + try: + content, usage = chat_degrade(prompt, model) + except SensitiveError as error: + # chat_governed 内部已经完成网络重试和模型降级；到这里说明整条治理链耗尽。 + raise click.ClickException(f"大纲生成治理链耗尽，停止当前作品：{str(error)[:180]}") from error + try: + data = extract_json(content) + except (RuntimeError, ValueError, json.JSONDecodeError) as error: + last_error = f"结构解析失败：{str(error)[:120]}" + continue + outline = (data.get("outline") or "").strip() + if outline: + return outline, usage + last_error = "outline 为空" + raise click.ClickException( + f"大纲结构输出连续 {max_attempts} 次失败（{last_error}），停止当前作品并保留断点" + ) + + +def order_outline_windows(windows): + """按绝对章号排列大纲窗；window_no 仅是运行展示号，补窗后可能漂移。""" + + return sorted(windows, key=lambda row: (row[2], row[3])) + + +def render_coherence_outlines(windows): + """按章序重新编号阶段，避免补插窗的旧 window_no 误导跨段审校。""" + + return "\n\n".join( + f"【阶段{stage_no}（第{row[2]}–{row[3]}章）】\n{row[4]}" + for stage_no, row in enumerate(order_outline_windows(windows), start=1) + ) + + @click.group() def cli(): """窗级大纲聚合与终检""" @@ -124,10 +198,12 @@ def window(work_id, window, model, from_, to_): """SELECT from_order, to_order FROM example_parse_outline WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE ORDER BY from_order""", (TENANT, work_id)).fetchall() - gaps = [f"{b1 + 1}-{a2 - 1}" for (_, b1), (a2, _) in zip(allw, allw[1:]) if a2 > b1 + 1] - if gaps: - click.echo(f"⚠ 本书窗覆盖存在缝隙章段 {gaps}：用 --from/--to 锁定缝隙补切" - f"（段太小不足半窗时把 --window 调小到段字数以下）") + ensure_outline_coverage( + allw, + from_, + to_ if to_ is not None else last_order, + context="window 后", + ) def _do_window(conn, work_id, title, win_no, chs, model, done): @@ -142,18 +218,12 @@ def _do_window(conn, work_id, title, win_no, chs, model, done): cap = max(200, int(wc * 0.004)) outlines = "\n".join(f"第{no}章《{ct}》：{o}" for no, ct, _, o in chs) text = "\n\n".join(f"【第{no}章 | {ct}】\n{t}" for no, ct, t, _ in chs) - try: - content, usage = chat_degrade(WINDOW_PROMPT.format( - title=title, a=chs[0][0], b=chs[-1][0], wc=wc, cap=cap, outlines=outlines, text=text), model) - except SensitiveError as e: - # 全链被拦：跳过本窗继续后续窗（窗大纲窗间独立，不必停书），重跑自动补 - click.echo(f"win-{win_no:02d} 敏感全链被拦，跳过（重跑补）: {str(e)[:80]}") - return - data = extract_json(content) - ol = (data.get("outline") or "").strip() - if not ol: - click.echo(f"win-{win_no:02d} 大纲为空，跳过（重跑补）") - return + ol, usage = generate_outline_with_retry( + WINDOW_PROMPT.format( + title=title, a=chs[0][0], b=chs[-1][0], wc=wc, cap=cap, outlines=outlines, text=text + ), + model, + ) conn.execute( """INSERT INTO example_parse_outline (work_id, window_no, from_order, to_order, outline_text, creator, updater, tenant_id) @@ -177,10 +247,18 @@ def check(work_id, model): wins = conn.execute( """SELECT id, window_no, from_order, to_order, outline_text, check_status FROM example_parse_outline - WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE ORDER BY window_no""", + WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE ORDER BY from_order""", (TENANT, work_id)).fetchall() + wins = order_outline_windows(wins) chs = load_chapters(conn, work_id) - n_issue = 0 + if chs: + ensure_outline_coverage( + [(row[2], row[3]) for row in wins], + chs[0][0], + chs[-1][0], + context="check 前", + ) + n_issue = checked_n = 0 for oid, wno, a, b, ol, st in wins: # 断点续跑：已检窗（checked/revised）不重刷，敏感崩溃后重启只补 pending if st and st != "pending": @@ -190,15 +268,15 @@ def check(work_id, model): content, _ = chat_degrade(CHECK_PROMPT.format( title=title, a=a, b=b, outline=ol, outlines=outlines), model) except SensitiveError as e: - # 全降级链被拦：标记跳窗继续，不让单窗炸掉整书终检（批9实测超神窗20崩全程） + # 全降级链耗尽：保留失败定位并停止本书，禁止跳窗后伪造终检完成。 conn.execute( "UPDATE example_parse_outline SET check_status='blocked', check_note=%s, updater=%s WHERE id=%s", (f"敏感拦截: {str(e)[:180]}", ACTOR, oid)) conn.commit() - click.echo(f"win-{wno:02d} 对账✗ 全链敏感拦截，跳过") - continue + raise click.ClickException(f"win-{wno:02d} 对账治理链耗尽，停止当前作品") from e data = extract_json(content) issues = data.get("issues") or [] + checked_n += 1 n_issue += len(issues) conn.execute( "UPDATE example_parse_outline SET check_status=%s, check_note=%s, updater=%s WHERE id=%s", @@ -209,15 +287,18 @@ def check(work_id, model): + "".join(f"\n · {i}" for i in issues[:5])) # 全书连贯性纵览（一次调用，只给大纲串） if len(wins) > 1: - allo = "\n\n".join(f"【阶段{w[1]}（第{w[2]}–{w[3]}章）】\n{w[4]}" for w in wins) + allo = render_coherence_outlines(wins) try: content, _ = chat_degrade(COHERENCE_PROMPT.format(title=title, n=len(wins), outlines=allo), model) data = extract_json(content) for i in (data.get("issues") or []): click.echo(f" [跨段] {i}") except SensitiveError: - click.echo(" [跨段] 全链敏感拦截，纵览跳过") - click.echo(f"终检完成：{len(wins)} 窗，细纲对账发现 {n_issue} 处") + raise click.ClickException("跨段纵览治理链耗尽，停止当前作品") + click.echo( + f"终检运行完成：总计 {len(wins)} 窗，本次实际检查 {checked_n} 窗、发现 {n_issue} 处；" + "其余窗沿用既有 check_status/check_note" + ) if __name__ == "__main__": diff --git a/.claude/skills/parse-book/scripts/test_parse_outline_offline.py b/.claude/skills/parse-book/scripts/test_parse_outline_offline.py new file mode 100644 index 0000000..cb8fada --- /dev/null +++ b/.claude/skills/parse-book/scripts/test_parse_outline_offline.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""parse_outline 窗覆盖门离线测试：不连库、不调用模型。""" + +import pathlib +import sys + +import click + +HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import parse_outline as po # noqa: E402 + + +def check(name, condition): + """输出与现有 parse-book 离线测试一致的简明结果。""" + + if not condition: + raise AssertionError(name) + print(f"[PASS] {name}") + + +def main(): + """覆盖连续、头部、中部、尾部和重叠窗五类边界。""" + + check("coverage-连续无缝", po.outline_coverage_gaps([(1, 33), (34, 65)], 1, 65) == []) + check( + "coverage-识别中部缝隙", + po.outline_coverage_gaps([(951, 983), (1015, 1047)], 951, 1047) == [(984, 1014)], + ) + check("coverage-识别头部缝隙", po.outline_coverage_gaps([(5, 10)], 1, 10) == [(1, 4)]) + check("coverage-识别尾部缝隙", po.outline_coverage_gaps([(1, 6)], 1, 10) == [(7, 10)]) + check( + "coverage-重叠窗不误报", + po.outline_coverage_gaps([(1, 20), (15, 30), (31, 40)], 1, 40) == [], + ) + po.ensure_outline_coverage([(1, 33), (34, 65)], 1, 65, context="测试") + try: + po.ensure_outline_coverage([(951, 983), (1015, 1047)], 951, 1047, context="测试") + except click.ClickException as error: + check("coverage-缝隙非零阻断", "984-1014" in str(error)) + else: + raise AssertionError("coverage-缝隙非零阻断") + + original_chat = po.chat_degrade + try: + replies = iter([("{}", {}), ('{"outline":"补窗成功"}', {"prompt_tokens": 1})]) + po.chat_degrade = lambda _prompt, _model: next(replies) + outline, usage = po.generate_outline_with_retry("prompt", "model", max_attempts=2) + check("window-空输出重试后成功", outline == "补窗成功" and usage["prompt_tokens"] == 1) + + calls = {"count": 0} + + def always_empty(_prompt, _model): + calls["count"] += 1 + return "{}", {} + + po.chat_degrade = always_empty + try: + po.generate_outline_with_retry("prompt", "model", max_attempts=2) + except click.ClickException as error: + check("window-有限重试耗尽即停书", calls["count"] == 2 and "2 次" in str(error)) + else: + raise AssertionError("window-有限重试耗尽即停书") + + def exhausted_chain(_prompt, _model): + raise po.SensitiveError("全部渠道耗尽") + + po.chat_degrade = exhausted_chain + try: + po.generate_outline_with_retry("prompt", "model", max_attempts=2) + except click.ClickException as error: + check("window-治理链耗尽即停书", "治理链耗尽" in str(error)) + else: + raise AssertionError("window-治理链耗尽即停书") + finally: + po.chat_degrade = original_chat + + unordered = [ + (3, 83, 984, 1014, "补插窗", "revised"), + (1, 1, 1, 33, "首窗", "revised"), + (4, 52, 1015, 1047, "后续窗", "revised"), + (2, 82, 2226, 2250, "末窗", "revised"), + ] + ordered = po.order_outline_windows(unordered) + check("check-按绝对章号排序", [row[2] for row in ordered] == [1, 984, 1015, 2226]) + rendered = po.render_coherence_outlines(ordered) + check( + "check-阶段号按章序重编", + "【阶段2（第984–1014章）】" in rendered and "【阶段83" not in rendered, + ) + + print("\n全部离线自测通过：11 项（未连库、未发任何 LLM 调用）") + + +if __name__ == "__main__": + main()