框架: 窗级大纲聚合落地(创始人拍板)——93表+parse_outline(5-10万字窗细纲+正文→阶段大纲;终检=逐窗细纲对账+跨段纵览);表映射补登92/93
This commit is contained in:
zizi
parent
02b983d589
commit
1e15b05ff7
178
.claude/skills/parse-book/scripts/parse_outline.py
Normal file
178
.claude/skills/parse-book/scripts/parse_outline.py
Normal file
@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env python3
|
||||
"""parse-book skill：窗级大纲聚合（创始人 2026-07-13 拍板）。
|
||||
|
||||
大纲不从单章按比例抽（单章对大纲层可能零贡献），而是：
|
||||
- window：5–10 万字窗（多章细纲+正文）聚合抽取一次窗级大纲 → example_parse_outline；
|
||||
- check：整本抽完后，逐窗拿细纲对账大纲（漏事件/矛盾），再做全书大纲连贯性纵览。
|
||||
前置：窗内章须已有脚手架细纲（example_parse_scaffold），缺则报缺不硬抽。
|
||||
"""
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import click
|
||||
import psycopg
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "llm" / "scripts"))
|
||||
from llm import chat, extract_json # noqa: E402
|
||||
|
||||
DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example"
|
||||
"?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3")
|
||||
TENANT, ACTOR = 1, "1"
|
||||
|
||||
WINDOW_PROMPT = """你是长篇小说结构分析员。下面是《{title}》第 {a}–{b} 章的**逐章细纲**与**正文**。
|
||||
把这一段（约 {wc} 字）聚合成一份「阶段大纲」，回答结构层的问题（细纲已有的章内细节不要复述）：
|
||||
- 阶段主线：这一段整体在推进什么，起点状态→终点状态；
|
||||
- 关键转折：改变走向的少数几个节点（章号+一句话）；
|
||||
- 人物与关系变化：谁登场定位、谁的关系变了；
|
||||
- 伏笔总账：这一段埋了什么（尚未回收）、收了什么（对应哪章埋的）；
|
||||
- 阶段末钩子：留给下一阶段的悬念。
|
||||
|
||||
**全文不得超过 {cap} 字**（约为正文的 0.4%；超标会被退回）。
|
||||
输出规则（只输出一个 JSON 对象）：{{"outline": "阶段大纲全文"}}
|
||||
|
||||
【逐章细纲】
|
||||
{outlines}
|
||||
|
||||
【正文】
|
||||
{text}"""
|
||||
|
||||
CHECK_PROMPT = """你是长篇小说结构审校员。下面是《{title}》第 {a}–{b} 章的「阶段大纲」与该段**全部逐章细纲**。
|
||||
对账：大纲是否遗漏了细纲中的重大事件/转折/伏笔？是否与细纲存在矛盾（人物/因果/顺序）？
|
||||
输出规则（只输出一个 JSON 对象）：
|
||||
{{"ok": true/false, "issues": ["每条一句话：遗漏/矛盾点（含章号）"]}}
|
||||
没有问题时 issues 给空数组。轻微详略取舍不算问题，只报结构性遗漏与事实矛盾。
|
||||
|
||||
【阶段大纲】
|
||||
{outline}
|
||||
|
||||
【逐章细纲】
|
||||
{outlines}"""
|
||||
|
||||
COHERENCE_PROMPT = """你是长篇小说结构审校员。下面是《{title}》全书 {n} 个阶段大纲（按顺序）。
|
||||
纵览检查：阶段之间是否连贯（前一段钩子是否被承接、伏笔跨段是否有账、主线是否断裂）？
|
||||
输出规则（只输出一个 JSON 对象）：{{"ok": true/false, "issues": ["跨段问题，一句话一条"]}}
|
||||
|
||||
{outlines}"""
|
||||
|
||||
|
||||
def load_chapters(conn, work_id, need_scaffold=True):
|
||||
"""拉全书章（含细纲）。返回 [(order_no, 章题, 正文, 细纲|None)]。"""
|
||||
return conn.execute(
|
||||
"""SELECT c.order_no, c.title, b.content_text, s.outline_text
|
||||
FROM muse_content_chapter c
|
||||
JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE
|
||||
LEFT JOIN example_parse_scaffold s ON s.chapter_id=c.id AND s.deleted=FALSE
|
||||
WHERE c.tenant_id=%s AND c.work_id=%s AND c.deleted=FALSE ORDER BY c.order_no""",
|
||||
(TENANT, work_id)).fetchall()
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""窗级大纲聚合与终检"""
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--work-id", type=int, required=True)
|
||||
@click.option("--window", type=int, default=80000, show_default=True, help="每窗目标字数（5–10万区间）")
|
||||
@click.option("--model", default="MiniMax-M3", show_default=True)
|
||||
@click.option("--from", "from_", type=int, default=1, help="起始章（断点续跑给上次结束章+1）")
|
||||
def window(work_id, window, model, from_):
|
||||
"""按字数切窗聚合大纲（窗内章须已有细纲；已有窗大纲的窗跳过=断点续跑）。"""
|
||||
with psycopg.connect(DSN) as conn:
|
||||
title = conn.execute("SELECT title FROM muse_content_work WHERE id=%s", (work_id,)).fetchone()[0]
|
||||
rows = [r for r in load_chapters(conn, work_id) if r[0] >= from_]
|
||||
done = {r[0] for r in conn.execute(
|
||||
"SELECT window_no FROM example_parse_outline WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE",
|
||||
(TENANT, work_id)).fetchall()}
|
||||
# 章序切窗（与清洗窗同思路：章对齐）
|
||||
win_no, cur, cur_len = max(done, default=0), [], 0
|
||||
for no, ct, text, outline in rows:
|
||||
cur.append((no, ct, text, outline))
|
||||
cur_len += len(text)
|
||||
if cur_len >= window:
|
||||
win_no += 1
|
||||
_do_window(conn, work_id, title, win_no, cur, model, done)
|
||||
cur, cur_len = [], 0
|
||||
if cur and cur_len >= window * 0.4: # 尾窗过小并入上窗的策略：足够大才单独成窗
|
||||
win_no += 1
|
||||
_do_window(conn, work_id, title, win_no, cur, model, done)
|
||||
elif cur:
|
||||
click.echo(f"尾段 {len(cur)} 章（{cur_len:,} 字）不足半窗，留待与后续章合窗")
|
||||
|
||||
|
||||
def _do_window(conn, work_id, title, win_no, chs, model, done):
|
||||
if win_no in done:
|
||||
click.echo(f"win-{win_no:02d} 已有大纲，跳过")
|
||||
return
|
||||
missing = [no for no, _, _, o in chs if not o]
|
||||
if missing:
|
||||
click.echo(f"win-{win_no:02d}（第{chs[0][0]}–{chs[-1][0]}章）缺细纲章 {missing[:10]}…，先补 scaffold")
|
||||
return
|
||||
wc = sum(len(t) for _, _, t, _ in chs)
|
||||
cap = max(200, int(wc * 0.004))
|
||||
outlines = "\n".join(f"第{no}章《{ct}》：{o}" for no, ct, _, o in chs)
|
||||
text = "\n\n".join(f"【第{no}章 | {ct}】\n{t}" for no, ct, t, _ in chs)
|
||||
content, usage = chat(WINDOW_PROMPT.format(
|
||||
title=title, a=chs[0][0], b=chs[-1][0], wc=wc, cap=cap, outlines=outlines, text=text), model=model)
|
||||
data = extract_json(content)
|
||||
ol = (data.get("outline") or "").strip()
|
||||
if not ol:
|
||||
click.echo(f"win-{win_no:02d} 大纲为空，跳过（重跑补）")
|
||||
return
|
||||
conn.execute(
|
||||
"""INSERT INTO example_parse_outline (work_id, window_no, from_order, to_order, outline_text,
|
||||
creator, updater, tenant_id)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)
|
||||
ON CONFLICT (tenant_id, work_id, window_no)
|
||||
DO UPDATE SET outline_text=EXCLUDED.outline_text, from_order=EXCLUDED.from_order,
|
||||
to_order=EXCLUDED.to_order, deleted=FALSE, check_status='pending', updater=EXCLUDED.updater""",
|
||||
(work_id, win_no, chs[0][0], chs[-1][0], ol, ACTOR, ACTOR, TENANT))
|
||||
conn.commit()
|
||||
click.echo(f"win-{win_no:02d}（第{chs[0][0]}–{chs[-1][0]}章 {wc:,}字）大纲 {len(ol)} 字 ✓ "
|
||||
f"(in={usage.get('prompt_tokens')})")
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--work-id", type=int, required=True)
|
||||
@click.option("--model", default="MiniMax-M3", show_default=True)
|
||||
def check(work_id, model):
|
||||
"""终检：逐窗细纲对账 + 全书大纲连贯性纵览（结果写 check_status/check_note）。"""
|
||||
with psycopg.connect(DSN) as conn:
|
||||
title = conn.execute("SELECT title FROM muse_content_work WHERE id=%s", (work_id,)).fetchone()[0]
|
||||
wins = conn.execute(
|
||||
"""SELECT id, window_no, from_order, to_order, outline_text FROM example_parse_outline
|
||||
WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE ORDER BY window_no""",
|
||||
(TENANT, work_id)).fetchall()
|
||||
chs = load_chapters(conn, work_id)
|
||||
n_issue = 0
|
||||
for oid, wno, a, b, ol in wins:
|
||||
outlines = "\n".join(f"第{no}章《{ct}》：{o}" for no, ct, _, o in chs if a <= no <= b and o)
|
||||
content, _ = chat(CHECK_PROMPT.format(title=title, a=a, b=b, outline=ol, outlines=outlines),
|
||||
model=model)
|
||||
data = extract_json(content)
|
||||
issues = data.get("issues") or []
|
||||
n_issue += len(issues)
|
||||
conn.execute(
|
||||
"UPDATE example_parse_outline SET check_status=%s, check_note=%s, updater=%s WHERE id=%s",
|
||||
("checked" if not issues else "revised",
|
||||
None if not issues else json.dumps(issues, ensure_ascii=False)[:1900], ACTOR, oid))
|
||||
conn.commit()
|
||||
click.echo(f"win-{wno:02d} 对账{'✓' if not issues else f' 发现 {len(issues)} 处'}"
|
||||
+ "".join(f"\n · {i}" for i in issues[:5]))
|
||||
# 全书连贯性纵览（一次调用，只给大纲串）
|
||||
if len(wins) > 1:
|
||||
allo = "\n\n".join(f"【阶段{w[1]}（第{w[2]}–{w[3]}章）】\n{w[4]}" for w in wins)
|
||||
content, _ = chat(COHERENCE_PROMPT.format(title=title, n=len(wins), outlines=allo), model=model)
|
||||
data = extract_json(content)
|
||||
for i in (data.get("issues") or []):
|
||||
click.echo(f" [跨段] {i}")
|
||||
click.echo(f"终检完成：{len(wins)} 窗，细纲对账发现 {n_issue} 处")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
cli()
|
||||
except (psycopg.Error, RuntimeError) as e:
|
||||
click.echo(f"[错误] {type(e).__name__}: {e}", err=True)
|
||||
sys.exit(1)
|
||||
23
db/ddl/93-example大纲聚合.sql
Normal file
23
db/ddl/93-example大纲聚合.sql
Normal file
@ -0,0 +1,23 @@
|
||||
-- example 私货：拆书大纲聚合窗表（创始人 2026-07-13 拍板：大纲不从单章按比例抽，
|
||||
-- 而是 5–10 万字窗（多章细纲+正文）聚合抽取一次；整本完成后拿全体细纲终检大纲）
|
||||
-- 命名与公共列遵循主仓 yudao 惯例（无外键、软删、tenant_id）。
|
||||
|
||||
CREATE TABLE IF NOT EXISTS example_parse_outline (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
work_id BIGINT NOT NULL, -- 参考书 work
|
||||
window_no INT NOT NULL, -- 窗序号（1 起）
|
||||
from_order INT NOT NULL, -- 窗起始章 order_no
|
||||
to_order INT NOT NULL, -- 窗结束章 order_no
|
||||
outline_text TEXT NOT NULL, -- 窗级大纲（阶段主线/关键转折/伏笔总账）
|
||||
check_status VARCHAR(20) NOT NULL DEFAULT 'pending', -- 终检状态: pending/checked/revised
|
||||
check_note TEXT, -- 终检发现（细纲对账不一致处）
|
||||
creator VARCHAR(64) DEFAULT '',
|
||||
create_time TIMESTAMP NOT NULL DEFAULT now(),
|
||||
updater VARCHAR(64) DEFAULT '',
|
||||
update_time TIMESTAMP NOT NULL DEFAULT now(),
|
||||
deleted BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
tenant_id BIGINT NOT NULL DEFAULT 0,
|
||||
CONSTRAINT uk_example_parse_outline UNIQUE (tenant_id, work_id, window_no)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE example_parse_outline IS 'example 私货：拆书窗级大纲聚合（5–10万字/窗，细纲+正文→大纲；终检=全体细纲对账）';
|
||||
@ -29,7 +29,7 @@
|
||||
| muse_knowledge_draft | V5 | V14（两快照列→varchar(128)） | **草稿**知识行（B2 拆书产出落此） |
|
||||
| muse_knowledge_binding | V5 | V14（同上） | 作品↔库绑定（C3 起用） |
|
||||
|
||||
## 实验私货表（4 张，`db/ddl/91-example实验私货.sql`）
|
||||
## 实验私货表（6 张，`db/ddl/91-example实验私货.sql` + 92 + 93）
|
||||
|
||||
| 表 | 用途 |
|
||||
|---|---|
|
||||
@ -37,6 +37,8 @@
|
||||
| example_parse_task | 拆书任务状态机（一章一行，scaffold/pattern 两阶段状态）——workflow 断点续跑与幂等 |
|
||||
| example_parse_scaffold | 逐章脚手架：逆推细纲（3–5%）+ 实体清单 JSONB |
|
||||
| example_knowledge_embedding | 知识行嵌入边表：vector(1024)+HNSW（余弦），content_hash 幂等，挂 draft 或 entity |
|
||||
| example_clean_log | 清洗审计（92）：每次删除一行（原文/理由/模型/批次），源 txt 不动可回放 |
|
||||
| example_parse_outline | 窗级大纲聚合（93）：5–10万字窗（细纲+正文）→阶段大纲；终检=细纲对账+跨段纵览 |
|
||||
|
||||
## 暂缓建表登记（主仓有、实验现阶段未建；需要时按原样加建）
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user