9a513e79b8
额度系统(item4,创始人2026-07-16拍板):llm.py 新增 chat_governed 统一入口——按5小时窗(00/05/10/15/20,末窗20-24)计钱计次;MiniMax(M3/M2.7)超$10/窗自动切 glm-5.2→deepseek-v4-flash;满4000次/窗自动睡到下一窗续跑;共享账本 example_llm_quota 原子累加,多分片共用一本;连接短开短关(LLM/睡眠期不持DB连接,治Tailscale空转)。parse_llm/parse_outline/parse_ingest/review_cards 各自的降级链上收复用统一路由,保各自异常契约(SensitiveHardStop/SensitiveError/判重保守keep)。新增离线单测 test_quota.py(9例:窗口归窗/计价缓存折扣/六条路由)+DDL db/ddl/95。 主代理亲验上线:9条离线单测亲跑全过;真调用对账端到端闭合(算11.04配额单位≈服务端实扣11);账本两笔原子累加正确;校准噪音已清空、额度从零起。 review_cards.py 一并携带批9c崩溃修复(连接读/算/写三段拆分+分片--shard+敏感降级),五书全量重审赖此收官。批9c收官锚点入 README§九+docs方案doc;深空升格样张入档。
229 lines
12 KiB
Python
229 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""parse-book skill：窗级大纲聚合（创始人 2026-07-13 拍板）。
|
||
|
||
大纲不从单章按比例抽（单章对大纲层可能零贡献），而是：
|
||
- window：5–10 万字窗（多章细纲+正文）聚合抽取一次窗级大纲 → example_parse_outline；
|
||
- check：整本抽完后，逐窗拿细纲对账大纲（漏事件/矛盾），再做全书大纲连贯性纵览。
|
||
前置：窗内章须已有脚手架细纲（example_parse_scaffold），缺则报缺不硬抽。
|
||
"""
|
||
import json
|
||
import pathlib
|
||
import sys
|
||
|
||
import click
|
||
import psycopg
|
||
|
||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "llm" / "scripts"))
|
||
from llm import chat_governed, extract_json, SensitiveError # noqa: E402
|
||
|
||
# 敏感/额度降级链已上收 llm.chat_governed（BUDGET_CHAIN 全局统一 + 额度治理）；本模块不再自持降级链。
|
||
|
||
|
||
def chat_degrade(prompt, model):
|
||
"""薄封装 chat_governed（全局统一降级链 + 额度治理），保留 (content, usage) 返回契约。
|
||
全链耗尽（used 为 None）时抛 SensitiveError，保留原「交上层跳窗」语义（_do_window/check 靠它跳窗）。"""
|
||
content, usage, used = chat_governed(prompt, model=model)
|
||
if used is None:
|
||
raise SensitiveError("chat_governed 全局降级链全部耗尽（内容安全或模型不可用）")
|
||
return content, usage
|
||
|
||
DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example"
|
||
"?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3")
|
||
TENANT, ACTOR = 1, "1"
|
||
|
||
WINDOW_PROMPT = """你是长篇小说结构分析员。下面是《{title}》第 {a}–{b} 章的**逐章细纲**与**正文**。
|
||
把这一段（约 {wc} 字）聚合成一份「阶段大纲」，回答结构层的问题（细纲已有的章内细节不要复述）：
|
||
- 阶段主线：这一段整体在推进什么，起点状态→终点状态；
|
||
- 关键转折：改变走向的少数几个节点（章号+一句话）；
|
||
- 人物与关系变化：谁登场定位、谁的关系变了；
|
||
- 伏笔总账：这一段埋了什么（尚未回收）、收了什么（对应哪章埋的）；
|
||
- 阶段末钩子：留给下一阶段的悬念。
|
||
|
||
**全文不得超过 {cap} 字**（约为正文的 0.4%；超标会被退回）。
|
||
输出规则（只输出一个 JSON 对象）：{{"outline": "阶段大纲全文"}}
|
||
|
||
【逐章细纲】
|
||
{outlines}
|
||
|
||
【正文】
|
||
{text}"""
|
||
|
||
CHECK_PROMPT = """你是长篇小说结构审校员。下面是《{title}》第 {a}–{b} 章的「阶段大纲」与该段**全部逐章细纲**。
|
||
对账：大纲是否遗漏了细纲中的重大事件/转折/伏笔？是否与细纲存在矛盾（人物/因果/顺序）？
|
||
输出规则（只输出一个 JSON 对象）：
|
||
{{"ok": true/false, "issues": ["每条一句话：遗漏/矛盾点（含章号）"]}}
|
||
没有问题时 issues 给空数组。轻微详略取舍不算问题，只报结构性遗漏与事实矛盾。
|
||
|
||
【阶段大纲】
|
||
{outline}
|
||
|
||
【逐章细纲】
|
||
{outlines}"""
|
||
|
||
COHERENCE_PROMPT = """你是长篇小说结构审校员。下面是《{title}》全书 {n} 个阶段大纲（按顺序）。
|
||
纵览检查：阶段之间是否连贯（前一段钩子是否被承接、伏笔跨段是否有账、主线是否断裂）？
|
||
输出规则（只输出一个 JSON 对象）：{{"ok": true/false, "issues": ["跨段问题，一句话一条"]}}
|
||
|
||
{outlines}"""
|
||
|
||
|
||
def load_chapters(conn, work_id, need_scaffold=True):
|
||
"""拉全书章（含细纲）。返回 [(order_no, 章题, 正文, 细纲|None)]。"""
|
||
return conn.execute(
|
||
"""SELECT c.order_no, c.title, b.content_text, s.outline_text
|
||
FROM muse_content_chapter c
|
||
JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE
|
||
LEFT JOIN example_parse_scaffold s ON s.chapter_id=c.id AND s.deleted=FALSE
|
||
WHERE c.tenant_id=%s AND c.work_id=%s AND c.deleted=FALSE ORDER BY c.order_no""",
|
||
(TENANT, work_id)).fetchall()
|
||
|
||
|
||
@click.group()
|
||
def cli():
|
||
"""窗级大纲聚合与终检"""
|
||
|
||
|
||
@cli.command()
|
||
@click.option("--work-id", type=int, required=True)
|
||
@click.option("--window", type=int, default=80000, show_default=True, help="每窗目标字数（5–10万区间）")
|
||
@click.option("--model", default="MiniMax-M3", show_default=True)
|
||
@click.option("--from", "from_", type=int, default=1, help="起始章（断点续跑给上次结束章+1）")
|
||
@click.option("--to", "to_", type=int, help="结束章（限定已拆域，防止切窗吞进没有细纲的章；默认全书）")
|
||
def window(work_id, window, model, from_, to_):
|
||
"""按字数切窗聚合大纲（窗内章须已有细纲；已有窗大纲的窗跳过=断点续跑）。"""
|
||
with psycopg.connect(DSN) as conn:
|
||
title = conn.execute("SELECT title FROM muse_content_work WHERE id=%s", (work_id,)).fetchone()[0]
|
||
all_rows = load_chapters(conn, work_id)
|
||
last_order = all_rows[-1][0] if all_rows else 0 # 全书末章（书末残窗判断用）
|
||
rows = [r for r in all_rows if r[0] >= from_ and (to_ is None or r[0] <= to_)]
|
||
# 幂等键=from_order（内容锚定，B4-S3/fable 审查 E2）：window_no 是流水号，
|
||
# 断点续跑或参数变化时会漂移导致覆盖错窗——只作展示序号，不作跳过依据
|
||
prior = conn.execute(
|
||
"SELECT from_order, window_no FROM example_parse_outline WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE",
|
||
(TENANT, work_id)).fetchall()
|
||
done = {r[0] for r in prior}
|
||
# 章序切窗（与清洗窗同思路：章对齐）；全量重跑序号从头计，续跑接上次
|
||
win_no, cur, cur_len = (0 if from_ <= 1 else max((r[1] for r in prior), default=0)), [], 0
|
||
for no, ct, text, outline in rows:
|
||
cur.append((no, ct, text, outline))
|
||
cur_len += len(text)
|
||
if cur_len >= window:
|
||
win_no += 1
|
||
_do_window(conn, work_id, title, win_no, cur, model, done)
|
||
cur, cur_len = [], 0
|
||
# 尾段：已到全书末章 → 无论大小独立成窗（全书拆完时尾段必须落窗，否则永远留白）；
|
||
# 未到书末 → 足够大才成窗，不足留待与后续章合窗
|
||
if cur and (cur[-1][0] == last_order or cur_len >= window * 0.4):
|
||
win_no += 1
|
||
_do_window(conn, work_id, title, win_no, cur, model, done)
|
||
elif cur:
|
||
click.echo(f"尾段 {len(cur)} 章（{cur_len:,} 字）不足半窗且未到书末，留待与后续章合窗")
|
||
# 收尾缝隙自检（批7 实测：三本书都在 50 章分批断点处留了永久悬空段——
|
||
# "留待合窗"的尾段被下次续跑的 from 跳过。切完就报，别等体检才发现）
|
||
allw = conn.execute(
|
||
"""SELECT from_order, to_order FROM example_parse_outline
|
||
WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE ORDER BY from_order""",
|
||
(TENANT, work_id)).fetchall()
|
||
gaps = [f"{b1 + 1}-{a2 - 1}" for (_, b1), (a2, _) in zip(allw, allw[1:]) if a2 > b1 + 1]
|
||
if gaps:
|
||
click.echo(f"⚠ 本书窗覆盖存在缝隙章段 {gaps}：用 --from/--to 锁定缝隙补切"
|
||
f"（段太小不足半窗时把 --window 调小到段字数以下）")
|
||
|
||
|
||
def _do_window(conn, work_id, title, win_no, chs, model, done):
|
||
if chs[0][0] in done: # 幂等按窗起始章判断（不按流水号）
|
||
click.echo(f"win-{win_no:02d}（第{chs[0][0]}章起）已有大纲，跳过")
|
||
return
|
||
missing = [no for no, _, _, o in chs if not o]
|
||
if missing:
|
||
click.echo(f"win-{win_no:02d}（第{chs[0][0]}–{chs[-1][0]}章）缺细纲章 {missing[:10]}…，先补 scaffold")
|
||
return
|
||
wc = sum(len(t) for _, _, t, _ in chs)
|
||
cap = max(200, int(wc * 0.004))
|
||
outlines = "\n".join(f"第{no}章《{ct}》：{o}" for no, ct, _, o in chs)
|
||
text = "\n\n".join(f"【第{no}章 | {ct}】\n{t}" for no, ct, t, _ in chs)
|
||
try:
|
||
content, usage = chat_degrade(WINDOW_PROMPT.format(
|
||
title=title, a=chs[0][0], b=chs[-1][0], wc=wc, cap=cap, outlines=outlines, text=text), model)
|
||
except SensitiveError as e:
|
||
# 全链被拦：跳过本窗继续后续窗（窗大纲窗间独立，不必停书），重跑自动补
|
||
click.echo(f"win-{win_no:02d} 敏感全链被拦，跳过（重跑补）: {str(e)[:80]}")
|
||
return
|
||
data = extract_json(content)
|
||
ol = (data.get("outline") or "").strip()
|
||
if not ol:
|
||
click.echo(f"win-{win_no:02d} 大纲为空，跳过（重跑补）")
|
||
return
|
||
conn.execute(
|
||
"""INSERT INTO example_parse_outline (work_id, window_no, from_order, to_order, outline_text,
|
||
creator, updater, tenant_id)
|
||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)
|
||
ON CONFLICT (tenant_id, work_id, from_order)
|
||
DO UPDATE SET outline_text=EXCLUDED.outline_text, window_no=EXCLUDED.window_no,
|
||
to_order=EXCLUDED.to_order, deleted=FALSE, check_status='pending', updater=EXCLUDED.updater""",
|
||
(work_id, win_no, chs[0][0], chs[-1][0], ol, ACTOR, ACTOR, TENANT))
|
||
conn.commit()
|
||
click.echo(f"win-{win_no:02d}（第{chs[0][0]}–{chs[-1][0]}章 {wc:,}字）大纲 {len(ol)} 字 ✓ "
|
||
f"(in={usage.get('prompt_tokens')})")
|
||
|
||
|
||
@cli.command()
|
||
@click.option("--work-id", type=int, required=True)
|
||
@click.option("--model", default="MiniMax-M3", show_default=True)
|
||
def check(work_id, model):
|
||
"""终检：逐窗细纲对账 + 全书大纲连贯性纵览（结果写 check_status/check_note）。"""
|
||
with psycopg.connect(DSN) as conn:
|
||
title = conn.execute("SELECT title FROM muse_content_work WHERE id=%s", (work_id,)).fetchone()[0]
|
||
wins = conn.execute(
|
||
"""SELECT id, window_no, from_order, to_order, outline_text, check_status
|
||
FROM example_parse_outline
|
||
WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE ORDER BY window_no""",
|
||
(TENANT, work_id)).fetchall()
|
||
chs = load_chapters(conn, work_id)
|
||
n_issue = 0
|
||
for oid, wno, a, b, ol, st in wins:
|
||
# 断点续跑：已检窗（checked/revised）不重刷，敏感崩溃后重启只补 pending
|
||
if st and st != "pending":
|
||
continue
|
||
outlines = "\n".join(f"第{no}章《{ct}》：{o}" for no, ct, _, o in chs if a <= no <= b and o)
|
||
try:
|
||
content, _ = chat_degrade(CHECK_PROMPT.format(
|
||
title=title, a=a, b=b, outline=ol, outlines=outlines), model)
|
||
except SensitiveError as e:
|
||
# 全降级链被拦：标记跳窗继续，不让单窗炸掉整书终检（批9实测超神窗20崩全程）
|
||
conn.execute(
|
||
"UPDATE example_parse_outline SET check_status='blocked', check_note=%s, updater=%s WHERE id=%s",
|
||
(f"敏感拦截: {str(e)[:180]}", ACTOR, oid))
|
||
conn.commit()
|
||
click.echo(f"win-{wno:02d} 对账✗ 全链敏感拦截，跳过")
|
||
continue
|
||
data = extract_json(content)
|
||
issues = data.get("issues") or []
|
||
n_issue += len(issues)
|
||
conn.execute(
|
||
"UPDATE example_parse_outline SET check_status=%s, check_note=%s, updater=%s WHERE id=%s",
|
||
("checked" if not issues else "revised",
|
||
None if not issues else json.dumps(issues, ensure_ascii=False)[:1900], ACTOR, oid))
|
||
conn.commit()
|
||
click.echo(f"win-{wno:02d} 对账{'✓' if not issues else f' 发现 {len(issues)} 处'}"
|
||
+ "".join(f"\n · {i}" for i in issues[:5]))
|
||
# 全书连贯性纵览（一次调用，只给大纲串）
|
||
if len(wins) > 1:
|
||
allo = "\n\n".join(f"【阶段{w[1]}（第{w[2]}–{w[3]}章）】\n{w[4]}" for w in wins)
|
||
try:
|
||
content, _ = chat_degrade(COHERENCE_PROMPT.format(title=title, n=len(wins), outlines=allo), model)
|
||
data = extract_json(content)
|
||
for i in (data.get("issues") or []):
|
||
click.echo(f" [跨段] {i}")
|
||
except SensitiveError:
|
||
click.echo(" [跨段] 全链敏感拦截，纵览跳过")
|
||
click.echo(f"终检完成：{len(wins)} 窗，细纲对账发现 {n_issue} 处")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
cli()
|
||
except (psycopg.Error, RuntimeError) as e:
|
||
click.echo(f"[错误] {type(e).__name__}: {e}", err=True)
|
||
sys.exit(1)
|