102 lines
5.0 KiB
Python
102 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
||
"""clean skill：LLM 探测执行器——每窗一次 MiniMax-M3 调用，产出 deletions JSON。
|
||
|
||
读 clean_prep 产出的 manifest.json，逐窗调 M3 探测垃圾段（只报逐字原文，不改写），
|
||
写 /tmp/muse-clean/<work>/deletions-NNN.json。断点续跑：已有产物的窗自动跳过。
|
||
删除动作不在本脚本：由 clean_apply 守卫裁决执行。
|
||
"""
|
||
import json
|
||
import pathlib
|
||
import sys
|
||
import time
|
||
|
||
import click
|
||
|
||
# 统一走 llm skill 入口（trust_env/重试/<think>剥离/JSON 容错都在那边）
|
||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "llm" / "scripts"))
|
||
from llm import chat, extract_json # noqa: E402
|
||
|
||
OUT = pathlib.Path("/tmp/muse-clean")
|
||
|
||
# 探测 prompt：与 SKILL.md 模板同源；要求纯 JSON 输出便于机器解析
|
||
PROMPT = """你是网文正文清洗探测器。下面是《{title}》第 {a}–{b} 章的原文（每章以【第N章 | 章题】标记行开头）。
|
||
|
||
找出**所有非正文垃圾**：
|
||
- 书站广告及变体（如「一秒记住♂粒÷小÷说→网」「天才一秒记住本站地址」）
|
||
- 网址/域名残留、"首发""手机版阅读""无弹窗"类导流语
|
||
- 求票拉票/求订阅收藏/打赏鸣谢段
|
||
- 章尾作者话（ps：… / PS：… / 附：…）、上架感言、请假条类整段
|
||
- 乱码水印、明显不属于故事的插入符号串
|
||
|
||
输出规则（只输出一个 JSON 对象，禁止任何其他文字）：
|
||
{{"deletions": [{{"chapter_order": 章序号, "exact": "待删段逐字原文", "reason": "简短理由"}}]}}
|
||
|
||
硬性要求：
|
||
- `exact` 必须与原文**逐字一致**（含标点、空格），单项 ≤300 字；同一段垃圾一项，不合并多段；
|
||
- 只报确定是垃圾的，拿不准的不报；**情节正文一个字都不许报**；
|
||
- 【第N章 | 章题】标记行不报（即使含求票字样）；
|
||
- 若该窗没有垃圾，输出 {{"deletions": []}}。
|
||
|
||
原文开始：
|
||
{text}"""
|
||
|
||
|
||
@click.command()
|
||
@click.option("--work-id", type=int, required=True)
|
||
@click.option("--win", "wins", type=int, multiple=True, help="指定窗号（可多次）；不给则全部窗")
|
||
@click.option("--model", default="MiniMax-M3", show_default=True)
|
||
@click.option("--fallback-model", default="deepseek-v4-flash", show_default=True,
|
||
help="主模型失败（如上游敏感词拦截）时的备选；再失败则记录跳过该窗")
|
||
@click.option("--force", is_flag=True, help="已有产物也重跑")
|
||
def main(work_id, wins, model, fallback_model, force):
|
||
d = OUT / str(work_id)
|
||
manifest = json.loads((d / "manifest.json").read_text())
|
||
title = manifest["title"]
|
||
targets = [m for m in manifest["windows"] if not wins or m["win"] in wins]
|
||
total_in = total_out = n_del = n_skip = 0
|
||
for m in targets:
|
||
f_out = d / f"deletions-{m['win']:03d}.json"
|
||
if f_out.exists() and not force:
|
||
click.echo(f"win-{m['win']:03d} 已有产物，跳过（--force 重跑）", err=True)
|
||
continue
|
||
text = pathlib.Path(m["file"]).read_text()
|
||
t0 = time.time()
|
||
prompt = PROMPT.format(title=title, a=m["from"], b=m["to"], text=text)
|
||
# 单窗失败不中断整书：主模型 → 备选模型 → 记录跳过（网文正文可能触发上游敏感词拦截）
|
||
dels, used_model = None, model
|
||
for cand in (model, fallback_model):
|
||
try:
|
||
content, usage = chat(prompt, model=cand)
|
||
data = extract_json(content)
|
||
dels = data.get("deletions", data if isinstance(data, list) else [])
|
||
used_model = cand
|
||
total_in += usage.get("prompt_tokens", 0)
|
||
total_out += usage.get("completion_tokens", 0)
|
||
break
|
||
except (RuntimeError, ValueError) as e:
|
||
click.echo(f"win-{m['win']:03d} {cand} 失败: {str(e)[:120]}", err=True)
|
||
if dels is None:
|
||
# 双模型全败：写空产物占位（含跳过原因），apply 端窗产物齐备可继续，审计可追
|
||
f_out.write_text(json.dumps(
|
||
{"deletions": [], "skipped": "主/备模型均失败（多为上游敏感词拦截）"},
|
||
ensure_ascii=False, indent=1))
|
||
n_skip += 1
|
||
click.echo(f"win-{m['win']:03d}（第{m['from']}–{m['to']}章）探测跳过，该窗不清洗")
|
||
continue
|
||
f_out.write_text(json.dumps({"deletions": dels, "model": used_model},
|
||
ensure_ascii=False, indent=1))
|
||
n_del += len(dels)
|
||
tag = "" if used_model == model else f"（备选 {used_model}）"
|
||
click.echo(f"win-{m['win']:03d}（第{m['from']}–{m['to']}章）检出 {len(dels)} 段{tag} "
|
||
f"→ {f_out.name}（{time.time() - t0:.0f}s）")
|
||
click.echo(f"探测完成：{len(targets)} 窗共检出 {n_del} 段（跳过 {n_skip} 窗）；"
|
||
f"token in={total_in:,} out={total_out:,}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
main()
|
||
except RuntimeError as e:
|
||
click.echo(f"[llm错误] {e}", err=True)
|
||
sys.exit(1)
|