310 lines
15 KiB
Python
310 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
"""parse-book skill：窗级大纲聚合（创始人 2026-07-13 拍板）。
|
||
|
||
大纲不从单章按比例抽（单章对大纲层可能零贡献），而是：
|
||
- window：5–10 万字窗（多章细纲+正文）聚合抽取一次窗级大纲 → example_parse_outline；
|
||
- check：整本抽完后，逐窗拿细纲对账大纲（漏事件/矛盾），再做全书大纲连贯性纵览。
|
||
前置：窗内章须已有脚手架细纲（example_parse_scaffold），缺则报缺不硬抽。
|
||
"""
|
||
import json
|
||
import pathlib
|
||
import sys
|
||
|
||
import click
|
||
import psycopg
|
||
|
||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "llm" / "scripts"))
|
||
from llm import chat_governed, extract_json, SensitiveError # noqa: E402
|
||
|
||
# 敏感/额度降级链已上收 llm.chat_governed（BUDGET_CHAIN 全局统一 + 额度治理）；本模块不再自持降级链。
|
||
|
||
|
||
def chat_degrade(prompt, model):
|
||
"""薄封装 chat_governed（全局统一降级链 + 额度治理），保留 (content, usage) 返回契约。
|
||
全链耗尽（used 为 None）时抛 SensitiveError，保留原「交上层跳窗」语义（_do_window/check 靠它跳窗）。"""
|
||
content, usage, used = chat_governed(prompt, model=model)
|
||
if used is None:
|
||
raise SensitiveError("chat_governed 全局降级链全部耗尽（内容安全或模型不可用）")
|
||
return content, usage
|
||
|
||
DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example"
|
||
"?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3")
|
||
TENANT, ACTOR = 1, "1"
|
||
MAX_WINDOW_GENERATION_ATTEMPTS = 2
|
||
|
||
WINDOW_PROMPT = """你是长篇小说结构分析员。下面是《{title}》第 {a}–{b} 章的**逐章细纲**与**正文**。
|
||
把这一段（约 {wc} 字）聚合成一份「阶段大纲」，回答结构层的问题（细纲已有的章内细节不要复述）：
|
||
- 阶段主线：这一段整体在推进什么，起点状态→终点状态；
|
||
- 关键转折：改变走向的少数几个节点（章号+一句话）；
|
||
- 人物与关系变化：谁登场定位、谁的关系变了；
|
||
- 伏笔总账：这一段埋了什么（尚未回收）、收了什么（对应哪章埋的）；
|
||
- 阶段末钩子：留给下一阶段的悬念。
|
||
|
||
**全文不得超过 {cap} 字**（约为正文的 0.4%；超标会被退回）。
|
||
输出规则（只输出一个 JSON 对象）：{{"outline": "阶段大纲全文"}}
|
||
|
||
【逐章细纲】
|
||
{outlines}
|
||
|
||
【正文】
|
||
{text}"""
|
||
|
||
CHECK_PROMPT = """你是长篇小说结构审校员。下面是《{title}》第 {a}–{b} 章的「阶段大纲」与该段**全部逐章细纲**。
|
||
对账：大纲是否遗漏了细纲中的重大事件/转折/伏笔？是否与细纲存在矛盾（人物/因果/顺序）？
|
||
输出规则（只输出一个 JSON 对象）：
|
||
{{"ok": true/false, "issues": ["每条一句话：遗漏/矛盾点（含章号）"]}}
|
||
没有问题时 issues 给空数组。轻微详略取舍不算问题，只报结构性遗漏与事实矛盾。
|
||
|
||
【阶段大纲】
|
||
{outline}
|
||
|
||
【逐章细纲】
|
||
{outlines}"""
|
||
|
||
COHERENCE_PROMPT = """你是长篇小说结构审校员。下面是《{title}》全书 {n} 个阶段大纲（按顺序）。
|
||
纵览检查：阶段之间是否连贯（前一段钩子是否被承接、伏笔跨段是否有账、主线是否断裂）？
|
||
输出规则（只输出一个 JSON 对象）：{{"ok": true/false, "issues": ["跨段问题，一句话一条"]}}
|
||
|
||
{outlines}"""
|
||
|
||
|
||
def load_chapters(conn, work_id, need_scaffold=True):
|
||
"""拉全书章（含细纲）。返回 [(order_no, 章题, 正文, 细纲|None)]。"""
|
||
return conn.execute(
|
||
"""SELECT c.order_no, c.title, b.content_text, s.outline_text
|
||
FROM muse_content_chapter c
|
||
JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE
|
||
LEFT JOIN example_parse_scaffold s ON s.chapter_id=c.id AND s.deleted=FALSE
|
||
WHERE c.tenant_id=%s AND c.work_id=%s AND c.deleted=FALSE ORDER BY c.order_no""",
|
||
(TENANT, work_id)).fetchall()
|
||
|
||
|
||
def outline_coverage_gaps(windows, first_order, last_order):
|
||
"""计算指定章域内未被任何大纲窗覆盖的连续章段。"""
|
||
|
||
if first_order > last_order:
|
||
return []
|
||
cursor = first_order
|
||
gaps = []
|
||
for raw_start, raw_end in sorted(windows):
|
||
if raw_end < first_order or raw_start > last_order:
|
||
continue
|
||
start = max(first_order, raw_start)
|
||
end = min(last_order, raw_end)
|
||
if start > cursor:
|
||
gaps.append((cursor, start - 1))
|
||
cursor = max(cursor, end + 1)
|
||
if cursor > last_order:
|
||
break
|
||
if cursor <= last_order:
|
||
gaps.append((cursor, last_order))
|
||
return gaps
|
||
|
||
|
||
def ensure_outline_coverage(windows, first_order, last_order, *, context):
|
||
"""大纲窗覆盖不完整时失败关闭，禁止 cards/check 在缺窗上制造假绿。"""
|
||
|
||
gaps = outline_coverage_gaps(windows, first_order, last_order)
|
||
if not gaps:
|
||
return
|
||
rendered = [f"{start}-{end}" for start, end in gaps]
|
||
raise click.ClickException(
|
||
f"{context}大纲窗覆盖存在缝隙章段 {rendered}；先用 window --from/--to 补窗，禁止继续出卡或终检"
|
||
)
|
||
|
||
|
||
def generate_outline_with_retry(prompt, model, *, max_attempts=MAX_WINDOW_GENERATION_ATTEMPTS):
|
||
"""有限重试大纲结构输出；治理链或结构重试耗尽后立即停止当前作品。"""
|
||
|
||
last_error = "输出为空"
|
||
for _attempt in range(1, max_attempts + 1):
|
||
try:
|
||
content, usage = chat_degrade(prompt, model)
|
||
except SensitiveError as error:
|
||
# chat_governed 内部已经完成网络重试和模型降级；到这里说明整条治理链耗尽。
|
||
raise click.ClickException(f"大纲生成治理链耗尽，停止当前作品：{str(error)[:180]}") from error
|
||
try:
|
||
data = extract_json(content)
|
||
except (RuntimeError, ValueError, json.JSONDecodeError) as error:
|
||
last_error = f"结构解析失败：{str(error)[:120]}"
|
||
continue
|
||
outline = (data.get("outline") or "").strip()
|
||
if outline:
|
||
return outline, usage
|
||
last_error = "outline 为空"
|
||
raise click.ClickException(
|
||
f"大纲结构输出连续 {max_attempts} 次失败（{last_error}），停止当前作品并保留断点"
|
||
)
|
||
|
||
|
||
def order_outline_windows(windows):
|
||
"""按绝对章号排列大纲窗；window_no 仅是运行展示号，补窗后可能漂移。"""
|
||
|
||
return sorted(windows, key=lambda row: (row[2], row[3]))
|
||
|
||
|
||
def render_coherence_outlines(windows):
|
||
"""按章序重新编号阶段，避免补插窗的旧 window_no 误导跨段审校。"""
|
||
|
||
return "\n\n".join(
|
||
f"【阶段{stage_no}（第{row[2]}–{row[3]}章）】\n{row[4]}"
|
||
for stage_no, row in enumerate(order_outline_windows(windows), start=1)
|
||
)
|
||
|
||
|
||
@click.group()
|
||
def cli():
|
||
"""窗级大纲聚合与终检"""
|
||
|
||
|
||
@cli.command()
|
||
@click.option("--work-id", type=int, required=True)
|
||
@click.option("--window", type=int, default=80000, show_default=True, help="每窗目标字数（5–10万区间）")
|
||
@click.option("--model", default="MiniMax-M3", show_default=True)
|
||
@click.option("--from", "from_", type=int, default=1, help="起始章（断点续跑给上次结束章+1）")
|
||
@click.option("--to", "to_", type=int, help="结束章（限定已拆域，防止切窗吞进没有细纲的章；默认全书）")
|
||
def window(work_id, window, model, from_, to_):
|
||
"""按字数切窗聚合大纲（窗内章须已有细纲；已有窗大纲的窗跳过=断点续跑）。"""
|
||
with psycopg.connect(DSN) as conn:
|
||
title = conn.execute("SELECT title FROM muse_content_work WHERE id=%s", (work_id,)).fetchone()[0]
|
||
all_rows = load_chapters(conn, work_id)
|
||
last_order = all_rows[-1][0] if all_rows else 0 # 全书末章（书末残窗判断用）
|
||
rows = [r for r in all_rows if r[0] >= from_ and (to_ is None or r[0] <= to_)]
|
||
# 幂等键=from_order（内容锚定，B4-S3/fable 审查 E2）：window_no 是流水号，
|
||
# 断点续跑或参数变化时会漂移导致覆盖错窗——只作展示序号，不作跳过依据
|
||
prior = conn.execute(
|
||
"SELECT from_order, window_no FROM example_parse_outline WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE",
|
||
(TENANT, work_id)).fetchall()
|
||
done = {r[0] for r in prior}
|
||
# 章序切窗（与清洗窗同思路：章对齐）；全量重跑序号从头计，续跑接上次
|
||
win_no, cur, cur_len = (0 if from_ <= 1 else max((r[1] for r in prior), default=0)), [], 0
|
||
for no, ct, text, outline in rows:
|
||
cur.append((no, ct, text, outline))
|
||
cur_len += len(text)
|
||
if cur_len >= window:
|
||
win_no += 1
|
||
_do_window(conn, work_id, title, win_no, cur, model, done)
|
||
cur, cur_len = [], 0
|
||
# 尾段：已到全书末章 → 无论大小独立成窗（全书拆完时尾段必须落窗，否则永远留白）；
|
||
# 未到书末 → 足够大才成窗，不足留待与后续章合窗
|
||
if cur and (cur[-1][0] == last_order or cur_len >= window * 0.4):
|
||
win_no += 1
|
||
_do_window(conn, work_id, title, win_no, cur, model, done)
|
||
elif cur:
|
||
click.echo(f"尾段 {len(cur)} 章（{cur_len:,} 字）不足半窗且未到书末，留待与后续章合窗")
|
||
# 收尾缝隙自检（批7 实测：三本书都在 50 章分批断点处留了永久悬空段——
|
||
# "留待合窗"的尾段被下次续跑的 from 跳过。切完就报，别等体检才发现）
|
||
allw = conn.execute(
|
||
"""SELECT from_order, to_order FROM example_parse_outline
|
||
WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE ORDER BY from_order""",
|
||
(TENANT, work_id)).fetchall()
|
||
ensure_outline_coverage(
|
||
allw,
|
||
from_,
|
||
to_ if to_ is not None else last_order,
|
||
context="window 后",
|
||
)
|
||
|
||
|
||
def _do_window(conn, work_id, title, win_no, chs, model, done):
|
||
if chs[0][0] in done: # 幂等按窗起始章判断（不按流水号）
|
||
click.echo(f"win-{win_no:02d}（第{chs[0][0]}章起）已有大纲，跳过")
|
||
return
|
||
missing = [no for no, _, _, o in chs if not o]
|
||
if missing:
|
||
click.echo(f"win-{win_no:02d}（第{chs[0][0]}–{chs[-1][0]}章）缺细纲章 {missing[:10]}…，先补 scaffold")
|
||
return
|
||
wc = sum(len(t) for _, _, t, _ in chs)
|
||
cap = max(200, int(wc * 0.004))
|
||
outlines = "\n".join(f"第{no}章《{ct}》：{o}" for no, ct, _, o in chs)
|
||
text = "\n\n".join(f"【第{no}章 | {ct}】\n{t}" for no, ct, t, _ in chs)
|
||
ol, usage = generate_outline_with_retry(
|
||
WINDOW_PROMPT.format(
|
||
title=title, a=chs[0][0], b=chs[-1][0], wc=wc, cap=cap, outlines=outlines, text=text
|
||
),
|
||
model,
|
||
)
|
||
conn.execute(
|
||
"""INSERT INTO example_parse_outline (work_id, window_no, from_order, to_order, outline_text,
|
||
creator, updater, tenant_id)
|
||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)
|
||
ON CONFLICT (tenant_id, work_id, from_order)
|
||
DO UPDATE SET outline_text=EXCLUDED.outline_text, window_no=EXCLUDED.window_no,
|
||
to_order=EXCLUDED.to_order, deleted=FALSE, check_status='pending', updater=EXCLUDED.updater""",
|
||
(work_id, win_no, chs[0][0], chs[-1][0], ol, ACTOR, ACTOR, TENANT))
|
||
conn.commit()
|
||
click.echo(f"win-{win_no:02d}（第{chs[0][0]}–{chs[-1][0]}章 {wc:,}字）大纲 {len(ol)} 字 ✓ "
|
||
f"(in={usage.get('prompt_tokens')})")
|
||
|
||
|
||
@cli.command()
|
||
@click.option("--work-id", type=int, required=True)
|
||
@click.option("--model", default="MiniMax-M3", show_default=True)
|
||
def check(work_id, model):
|
||
"""终检：逐窗细纲对账 + 全书大纲连贯性纵览（结果写 check_status/check_note）。"""
|
||
with psycopg.connect(DSN) as conn:
|
||
title = conn.execute("SELECT title FROM muse_content_work WHERE id=%s", (work_id,)).fetchone()[0]
|
||
wins = conn.execute(
|
||
"""SELECT id, window_no, from_order, to_order, outline_text, check_status
|
||
FROM example_parse_outline
|
||
WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE ORDER BY from_order""",
|
||
(TENANT, work_id)).fetchall()
|
||
wins = order_outline_windows(wins)
|
||
chs = load_chapters(conn, work_id)
|
||
if chs:
|
||
ensure_outline_coverage(
|
||
[(row[2], row[3]) for row in wins],
|
||
chs[0][0],
|
||
chs[-1][0],
|
||
context="check 前",
|
||
)
|
||
n_issue = checked_n = 0
|
||
for oid, wno, a, b, ol, st in wins:
|
||
# 断点续跑：已检窗（checked/revised）不重刷，敏感崩溃后重启只补 pending
|
||
if st and st != "pending":
|
||
continue
|
||
outlines = "\n".join(f"第{no}章《{ct}》：{o}" for no, ct, _, o in chs if a <= no <= b and o)
|
||
try:
|
||
content, _ = chat_degrade(CHECK_PROMPT.format(
|
||
title=title, a=a, b=b, outline=ol, outlines=outlines), model)
|
||
except SensitiveError as e:
|
||
# 全降级链耗尽：保留失败定位并停止本书，禁止跳窗后伪造终检完成。
|
||
conn.execute(
|
||
"UPDATE example_parse_outline SET check_status='blocked', check_note=%s, updater=%s WHERE id=%s",
|
||
(f"敏感拦截: {str(e)[:180]}", ACTOR, oid))
|
||
conn.commit()
|
||
raise click.ClickException(f"win-{wno:02d} 对账治理链耗尽，停止当前作品") from e
|
||
data = extract_json(content)
|
||
issues = data.get("issues") or []
|
||
checked_n += 1
|
||
n_issue += len(issues)
|
||
conn.execute(
|
||
"UPDATE example_parse_outline SET check_status=%s, check_note=%s, updater=%s WHERE id=%s",
|
||
("checked" if not issues else "revised",
|
||
None if not issues else json.dumps(issues, ensure_ascii=False)[:1900], ACTOR, oid))
|
||
conn.commit()
|
||
click.echo(f"win-{wno:02d} 对账{'✓' if not issues else f' 发现 {len(issues)} 处'}"
|
||
+ "".join(f"\n · {i}" for i in issues[:5]))
|
||
# 全书连贯性纵览（一次调用，只给大纲串）
|
||
if len(wins) > 1:
|
||
allo = render_coherence_outlines(wins)
|
||
try:
|
||
content, _ = chat_degrade(COHERENCE_PROMPT.format(title=title, n=len(wins), outlines=allo), model)
|
||
data = extract_json(content)
|
||
for i in (data.get("issues") or []):
|
||
click.echo(f" [跨段] {i}")
|
||
except SensitiveError:
|
||
raise click.ClickException("跨段纵览治理链耗尽，停止当前作品")
|
||
click.echo(
|
||
f"终检运行完成：总计 {len(wins)} 窗，本次实际检查 {checked_n} 窗、发现 {n_issue} 处；"
|
||
"其余窗沿用既有 check_status/check_note"
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
cli()
|
||
except (psycopg.Error, RuntimeError) as e:
|
||
click.echo(f"[错误] {type(e).__name__}: {e}", err=True)
|
||
sys.exit(1)
|