修复: 范式抽取失败关闭并阻断缺窗假绿
This commit is contained in:
zizi
parent
c969fa637d
commit
f5965577d6
@ -60,6 +60,8 @@ disable-model-invocation: true
|
||||
|
||||
**作品面升格执行器 `scripts/parse_upgrade.py`（与上面范式拆书管线并行的另一条线,命令 `windows`/`run`/`status`）**：把参考书正文按窗抽成「会随剧情长大的实体卡」(升格卡,`source_type=upgrade_book`),设计见 `docs/2026-07-16-升格卡改造设计.md`。里程碑除真实章号外，模型输出必须临时携带所标章节正文短引；系统机械核验后删除证据，错章、缺证据或改写证据均拒收入库并留审计。`run` 默认**不发嵌入**;加 `--semantic-dedup` 开语义判重(治改名/跨型漏并)时,每窗按**读→算→写三段式短连接**跑——观察/近邻召回/M3 终判都在**无长连接**段发 LLM 与嵌入 HTTP(不再持窗级连接跨调用存活),写段只查预判结果落库;并**边抽边嵌**：本窗新建/更新卡在窗事务提交后增量嵌入落库(软删旧向量+upsert 新行),**后窗即可语义召回前窗刚长成的卡**,不再依赖"同书须预先全量 embed"。嵌段失败只告警、不牵连已 done 的窗。
|
||||
|
||||
**范式线失败关闭（2026-07-21 假绿复盘后现行）**：`parse_outline.py window`、`parse_llm.py cards`、`parse_outline.py check` 是同一本作品的顺序流水线。普通网络/429/5xx 由 `chat_governed` 在单模型内短重试后沿 `MiniMax-M3 → MiniMax-M2.7 → glm-5.2 → deepseek-v4-flash` 降级；整条链耗尽，或功能层结构输出有限重试仍失败，必须**立即停止当前作品并保留断点**，禁止跳窗/跳型继续。`cards` 和 `check` 开始前机械验证大纲窗对全书章域无头部、中部、尾部缝隙；任一缝隙非零退出。补插窗后所有终检按 `from_order/to_order` 绝对章号排序并重新编号阶段，`window_no` 只作运行展示号，不得参与时序判断。
|
||||
|
||||
## 步骤(自底向上,与创作期规划的自顶向下互为镜像)
|
||||
|
||||
1. 静态分章:import skill(规则,LLM 不参与);
|
||||
|
||||
@ -27,6 +27,7 @@ import psycopg
|
||||
# 敏感/额度降级链已上收 llm.chat_governed（全局统一治理），本模块不再自持降级链
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "llm" / "scripts"))
|
||||
from llm import chat_governed, extract_json # noqa: E402
|
||||
from parse_outline import ensure_outline_coverage # noqa: E402
|
||||
|
||||
DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example"
|
||||
"?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3")
|
||||
@ -481,8 +482,21 @@ def cards(work_id, from_order, model, redo):
|
||||
args.append(from_order)
|
||||
wins = conn.execute(sql + " ORDER BY from_order", args).fetchall()
|
||||
if not wins:
|
||||
click.echo("无大纲窗行——先跑 parse_outline.py window（窗是出卡的切分依据）")
|
||||
return
|
||||
raise click.ClickException("无大纲窗行——先跑 parse_outline.py window（窗是出卡的切分依据）")
|
||||
if from_order is None:
|
||||
with psycopg.connect(DSN) as conn:
|
||||
bounds = conn.execute(
|
||||
"""SELECT min(order_no), max(order_no) FROM muse_content_chapter
|
||||
WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE""",
|
||||
(TENANT, work_id),
|
||||
).fetchone()
|
||||
if bounds and bounds[0] is not None:
|
||||
ensure_outline_coverage(
|
||||
[(row[0], row[1]) for row in wins],
|
||||
bounds[0],
|
||||
bounds[1],
|
||||
context="cards 前",
|
||||
)
|
||||
total_in = total_out = 0
|
||||
for a, b, stage_ol, wno in wins:
|
||||
with psycopg.connect(DSN) as conn:
|
||||
@ -542,7 +556,10 @@ def cards(work_id, from_order, model, redo):
|
||||
all_cards += cs
|
||||
click.echo(f" 批[{t}] 线索{len(hs)} → 卡{len(cs)}")
|
||||
except RuntimeError as e:
|
||||
click.echo(f" 批[{t}] M3 失败: {e}（本批线索丢失，重跑 --redo 可补）")
|
||||
# m3_json 已完成调用/结构重试；继续会让该型线索永久丢失并把整窗误标 done。
|
||||
raise click.ClickException(
|
||||
f"win#{a}–{b} 批[{t}] 连续重试仍失败，停止当前作品并保留断点：{str(e)[:180]}"
|
||||
) from e
|
||||
total_in += win_in
|
||||
total_out += win_out
|
||||
_, out = ingest("cards", work_id, a, {"cards": all_cards}, keyflag="--from-order")
|
||||
|
||||
@ -30,6 +30,7 @@ def chat_degrade(prompt, model):
|
||||
DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example"
|
||||
"?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3")
|
||||
TENANT, ACTOR = 1, "1"
|
||||
MAX_WINDOW_GENERATION_ATTEMPTS = 2
|
||||
|
||||
WINDOW_PROMPT = """你是长篇小说结构分析员。下面是《{title}》第 {a}–{b} 章的**逐章细纲**与**正文**。
|
||||
把这一段（约 {wc} 字）聚合成一份「阶段大纲」，回答结构层的问题（细纲已有的章内细节不要复述）：
|
||||
@ -78,6 +79,79 @@ def load_chapters(conn, work_id, need_scaffold=True):
|
||||
(TENANT, work_id)).fetchall()
|
||||
|
||||
|
||||
def outline_coverage_gaps(windows, first_order, last_order):
|
||||
"""计算指定章域内未被任何大纲窗覆盖的连续章段。"""
|
||||
|
||||
if first_order > last_order:
|
||||
return []
|
||||
cursor = first_order
|
||||
gaps = []
|
||||
for raw_start, raw_end in sorted(windows):
|
||||
if raw_end < first_order or raw_start > last_order:
|
||||
continue
|
||||
start = max(first_order, raw_start)
|
||||
end = min(last_order, raw_end)
|
||||
if start > cursor:
|
||||
gaps.append((cursor, start - 1))
|
||||
cursor = max(cursor, end + 1)
|
||||
if cursor > last_order:
|
||||
break
|
||||
if cursor <= last_order:
|
||||
gaps.append((cursor, last_order))
|
||||
return gaps
|
||||
|
||||
|
||||
def ensure_outline_coverage(windows, first_order, last_order, *, context):
|
||||
"""大纲窗覆盖不完整时失败关闭，禁止 cards/check 在缺窗上制造假绿。"""
|
||||
|
||||
gaps = outline_coverage_gaps(windows, first_order, last_order)
|
||||
if not gaps:
|
||||
return
|
||||
rendered = [f"{start}-{end}" for start, end in gaps]
|
||||
raise click.ClickException(
|
||||
f"{context}大纲窗覆盖存在缝隙章段 {rendered}；先用 window --from/--to 补窗，禁止继续出卡或终检"
|
||||
)
|
||||
|
||||
|
||||
def generate_outline_with_retry(prompt, model, *, max_attempts=MAX_WINDOW_GENERATION_ATTEMPTS):
|
||||
"""有限重试大纲结构输出；治理链或结构重试耗尽后立即停止当前作品。"""
|
||||
|
||||
last_error = "输出为空"
|
||||
for _attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
content, usage = chat_degrade(prompt, model)
|
||||
except SensitiveError as error:
|
||||
# chat_governed 内部已经完成网络重试和模型降级；到这里说明整条治理链耗尽。
|
||||
raise click.ClickException(f"大纲生成治理链耗尽，停止当前作品：{str(error)[:180]}") from error
|
||||
try:
|
||||
data = extract_json(content)
|
||||
except (RuntimeError, ValueError, json.JSONDecodeError) as error:
|
||||
last_error = f"结构解析失败：{str(error)[:120]}"
|
||||
continue
|
||||
outline = (data.get("outline") or "").strip()
|
||||
if outline:
|
||||
return outline, usage
|
||||
last_error = "outline 为空"
|
||||
raise click.ClickException(
|
||||
f"大纲结构输出连续 {max_attempts} 次失败（{last_error}），停止当前作品并保留断点"
|
||||
)
|
||||
|
||||
|
||||
def order_outline_windows(windows):
|
||||
"""按绝对章号排列大纲窗；window_no 仅是运行展示号，补窗后可能漂移。"""
|
||||
|
||||
return sorted(windows, key=lambda row: (row[2], row[3]))
|
||||
|
||||
|
||||
def render_coherence_outlines(windows):
|
||||
"""按章序重新编号阶段，避免补插窗的旧 window_no 误导跨段审校。"""
|
||||
|
||||
return "\n\n".join(
|
||||
f"【阶段{stage_no}（第{row[2]}–{row[3]}章）】\n{row[4]}"
|
||||
for stage_no, row in enumerate(order_outline_windows(windows), start=1)
|
||||
)
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""窗级大纲聚合与终检"""
|
||||
@ -124,10 +198,12 @@ def window(work_id, window, model, from_, to_):
|
||||
"""SELECT from_order, to_order FROM example_parse_outline
|
||||
WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE ORDER BY from_order""",
|
||||
(TENANT, work_id)).fetchall()
|
||||
gaps = [f"{b1 + 1}-{a2 - 1}" for (_, b1), (a2, _) in zip(allw, allw[1:]) if a2 > b1 + 1]
|
||||
if gaps:
|
||||
click.echo(f"⚠ 本书窗覆盖存在缝隙章段 {gaps}：用 --from/--to 锁定缝隙补切"
|
||||
f"（段太小不足半窗时把 --window 调小到段字数以下）")
|
||||
ensure_outline_coverage(
|
||||
allw,
|
||||
from_,
|
||||
to_ if to_ is not None else last_order,
|
||||
context="window 后",
|
||||
)
|
||||
|
||||
|
||||
def _do_window(conn, work_id, title, win_no, chs, model, done):
|
||||
@ -142,18 +218,12 @@ def _do_window(conn, work_id, title, win_no, chs, model, done):
|
||||
cap = max(200, int(wc * 0.004))
|
||||
outlines = "\n".join(f"第{no}章《{ct}》：{o}" for no, ct, _, o in chs)
|
||||
text = "\n\n".join(f"【第{no}章 | {ct}】\n{t}" for no, ct, t, _ in chs)
|
||||
try:
|
||||
content, usage = chat_degrade(WINDOW_PROMPT.format(
|
||||
title=title, a=chs[0][0], b=chs[-1][0], wc=wc, cap=cap, outlines=outlines, text=text), model)
|
||||
except SensitiveError as e:
|
||||
# 全链被拦：跳过本窗继续后续窗（窗大纲窗间独立，不必停书），重跑自动补
|
||||
click.echo(f"win-{win_no:02d} 敏感全链被拦，跳过（重跑补）: {str(e)[:80]}")
|
||||
return
|
||||
data = extract_json(content)
|
||||
ol = (data.get("outline") or "").strip()
|
||||
if not ol:
|
||||
click.echo(f"win-{win_no:02d} 大纲为空，跳过（重跑补）")
|
||||
return
|
||||
ol, usage = generate_outline_with_retry(
|
||||
WINDOW_PROMPT.format(
|
||||
title=title, a=chs[0][0], b=chs[-1][0], wc=wc, cap=cap, outlines=outlines, text=text
|
||||
),
|
||||
model,
|
||||
)
|
||||
conn.execute(
|
||||
"""INSERT INTO example_parse_outline (work_id, window_no, from_order, to_order, outline_text,
|
||||
creator, updater, tenant_id)
|
||||
@ -177,10 +247,18 @@ def check(work_id, model):
|
||||
wins = conn.execute(
|
||||
"""SELECT id, window_no, from_order, to_order, outline_text, check_status
|
||||
FROM example_parse_outline
|
||||
WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE ORDER BY window_no""",
|
||||
WHERE tenant_id=%s AND work_id=%s AND deleted=FALSE ORDER BY from_order""",
|
||||
(TENANT, work_id)).fetchall()
|
||||
wins = order_outline_windows(wins)
|
||||
chs = load_chapters(conn, work_id)
|
||||
n_issue = 0
|
||||
if chs:
|
||||
ensure_outline_coverage(
|
||||
[(row[2], row[3]) for row in wins],
|
||||
chs[0][0],
|
||||
chs[-1][0],
|
||||
context="check 前",
|
||||
)
|
||||
n_issue = checked_n = 0
|
||||
for oid, wno, a, b, ol, st in wins:
|
||||
# 断点续跑：已检窗（checked/revised）不重刷，敏感崩溃后重启只补 pending
|
||||
if st and st != "pending":
|
||||
@ -190,15 +268,15 @@ def check(work_id, model):
|
||||
content, _ = chat_degrade(CHECK_PROMPT.format(
|
||||
title=title, a=a, b=b, outline=ol, outlines=outlines), model)
|
||||
except SensitiveError as e:
|
||||
# 全降级链被拦：标记跳窗继续，不让单窗炸掉整书终检（批9实测超神窗20崩全程）
|
||||
# 全降级链耗尽：保留失败定位并停止本书，禁止跳窗后伪造终检完成。
|
||||
conn.execute(
|
||||
"UPDATE example_parse_outline SET check_status='blocked', check_note=%s, updater=%s WHERE id=%s",
|
||||
(f"敏感拦截: {str(e)[:180]}", ACTOR, oid))
|
||||
conn.commit()
|
||||
click.echo(f"win-{wno:02d} 对账✗ 全链敏感拦截，跳过")
|
||||
continue
|
||||
raise click.ClickException(f"win-{wno:02d} 对账治理链耗尽，停止当前作品") from e
|
||||
data = extract_json(content)
|
||||
issues = data.get("issues") or []
|
||||
checked_n += 1
|
||||
n_issue += len(issues)
|
||||
conn.execute(
|
||||
"UPDATE example_parse_outline SET check_status=%s, check_note=%s, updater=%s WHERE id=%s",
|
||||
@ -209,15 +287,18 @@ def check(work_id, model):
|
||||
+ "".join(f"\n · {i}" for i in issues[:5]))
|
||||
# 全书连贯性纵览（一次调用，只给大纲串）
|
||||
if len(wins) > 1:
|
||||
allo = "\n\n".join(f"【阶段{w[1]}（第{w[2]}–{w[3]}章）】\n{w[4]}" for w in wins)
|
||||
allo = render_coherence_outlines(wins)
|
||||
try:
|
||||
content, _ = chat_degrade(COHERENCE_PROMPT.format(title=title, n=len(wins), outlines=allo), model)
|
||||
data = extract_json(content)
|
||||
for i in (data.get("issues") or []):
|
||||
click.echo(f" [跨段] {i}")
|
||||
except SensitiveError:
|
||||
click.echo(" [跨段] 全链敏感拦截，纵览跳过")
|
||||
click.echo(f"终检完成：{len(wins)} 窗，细纲对账发现 {n_issue} 处")
|
||||
raise click.ClickException("跨段纵览治理链耗尽，停止当前作品")
|
||||
click.echo(
|
||||
f"终检运行完成：总计 {len(wins)} 窗，本次实际检查 {checked_n} 窗、发现 {n_issue} 处；"
|
||||
"其余窗沿用既有 check_status/check_note"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""parse_outline 窗覆盖门离线测试：不连库、不调用模型。"""
|
||||
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import click
|
||||
|
||||
HERE = pathlib.Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
import parse_outline as po # noqa: E402
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
"""输出与现有 parse-book 离线测试一致的简明结果。"""
|
||||
|
||||
if not condition:
|
||||
raise AssertionError(name)
|
||||
print(f"[PASS] {name}")
|
||||
|
||||
|
||||
def main():
|
||||
"""覆盖连续、头部、中部、尾部和重叠窗五类边界。"""
|
||||
|
||||
check("coverage-连续无缝", po.outline_coverage_gaps([(1, 33), (34, 65)], 1, 65) == [])
|
||||
check(
|
||||
"coverage-识别中部缝隙",
|
||||
po.outline_coverage_gaps([(951, 983), (1015, 1047)], 951, 1047) == [(984, 1014)],
|
||||
)
|
||||
check("coverage-识别头部缝隙", po.outline_coverage_gaps([(5, 10)], 1, 10) == [(1, 4)])
|
||||
check("coverage-识别尾部缝隙", po.outline_coverage_gaps([(1, 6)], 1, 10) == [(7, 10)])
|
||||
check(
|
||||
"coverage-重叠窗不误报",
|
||||
po.outline_coverage_gaps([(1, 20), (15, 30), (31, 40)], 1, 40) == [],
|
||||
)
|
||||
po.ensure_outline_coverage([(1, 33), (34, 65)], 1, 65, context="测试")
|
||||
try:
|
||||
po.ensure_outline_coverage([(951, 983), (1015, 1047)], 951, 1047, context="测试")
|
||||
except click.ClickException as error:
|
||||
check("coverage-缝隙非零阻断", "984-1014" in str(error))
|
||||
else:
|
||||
raise AssertionError("coverage-缝隙非零阻断")
|
||||
|
||||
original_chat = po.chat_degrade
|
||||
try:
|
||||
replies = iter([("{}", {}), ('{"outline":"补窗成功"}', {"prompt_tokens": 1})])
|
||||
po.chat_degrade = lambda _prompt, _model: next(replies)
|
||||
outline, usage = po.generate_outline_with_retry("prompt", "model", max_attempts=2)
|
||||
check("window-空输出重试后成功", outline == "补窗成功" and usage["prompt_tokens"] == 1)
|
||||
|
||||
calls = {"count": 0}
|
||||
|
||||
def always_empty(_prompt, _model):
|
||||
calls["count"] += 1
|
||||
return "{}", {}
|
||||
|
||||
po.chat_degrade = always_empty
|
||||
try:
|
||||
po.generate_outline_with_retry("prompt", "model", max_attempts=2)
|
||||
except click.ClickException as error:
|
||||
check("window-有限重试耗尽即停书", calls["count"] == 2 and "2 次" in str(error))
|
||||
else:
|
||||
raise AssertionError("window-有限重试耗尽即停书")
|
||||
|
||||
def exhausted_chain(_prompt, _model):
|
||||
raise po.SensitiveError("全部渠道耗尽")
|
||||
|
||||
po.chat_degrade = exhausted_chain
|
||||
try:
|
||||
po.generate_outline_with_retry("prompt", "model", max_attempts=2)
|
||||
except click.ClickException as error:
|
||||
check("window-治理链耗尽即停书", "治理链耗尽" in str(error))
|
||||
else:
|
||||
raise AssertionError("window-治理链耗尽即停书")
|
||||
finally:
|
||||
po.chat_degrade = original_chat
|
||||
|
||||
unordered = [
|
||||
(3, 83, 984, 1014, "补插窗", "revised"),
|
||||
(1, 1, 1, 33, "首窗", "revised"),
|
||||
(4, 52, 1015, 1047, "后续窗", "revised"),
|
||||
(2, 82, 2226, 2250, "末窗", "revised"),
|
||||
]
|
||||
ordered = po.order_outline_windows(unordered)
|
||||
check("check-按绝对章号排序", [row[2] for row in ordered] == [1, 984, 1015, 2226])
|
||||
rendered = po.render_coherence_outlines(ordered)
|
||||
check(
|
||||
"check-阶段号按章序重编",
|
||||
"【阶段2（第984–1014章）】" in rendered and "【阶段83" not in rendered,
|
||||
)
|
||||
|
||||
print("\n全部离线自测通过：11 项（未连库、未发任何 LLM 调用）")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user