框架: clean抗性三连——detect单窗失败降级备选模型不断书+apply书名水印豁免+水印裁剪防正文前缀误删(路C实测病对症)
This commit is contained in:
zizi
parent
a18db4bb01
commit
0a3a576704
@ -35,6 +35,10 @@ GARBAGE_LINE_PAT = re.compile(
|
||||
r"|手打中|稍等片刻|重新刷新|获取最新", # 盗版站占位残留（1040-1060 验证窗实测）
|
||||
re.I)
|
||||
|
||||
# 水印裁剪：盗版源把水印粘在正文行尾（如『“我草！”百镀一下“××”最新章节第一时间免费阅读。』），
|
||||
# LLM 会把整行报上来——删除段若含此类明确水印且水印前有前缀（多为正文），只删水印部分。
|
||||
WATERMARK_PAT = re.compile(r"(?:百镀一下|百度一下|一秒记住).{0,45}?(?:免费阅读。?|免费读！?|阅读。)")
|
||||
|
||||
|
||||
def flex_pattern(exact):
|
||||
"""空白弹性正则：非空白文字逐字 escape，连续空白段 → \\s+。
|
||||
@ -92,6 +96,10 @@ def main(work_id, batch, file_, model, dry_run, report_md):
|
||||
targets = sorted({n for no in by_ch for n in (no - 1, no, no + 1) if n >= 1})
|
||||
stats = {"删除段": 0, "删除字数": 0, "拒绝": [], "弹性命中": 0, "邻章命中": 0}
|
||||
with psycopg.connect(DSN) as conn:
|
||||
# 书名水印豁免：盗版源每章插孤立书名行——孤行精确等于本书书名不可能是正文叙述
|
||||
work_title = (conn.execute(
|
||||
"SELECT title FROM muse_content_work WHERE tenant_id=%s AND id=%s",
|
||||
(TENANT, work_id)).fetchone() or ("",))[0]
|
||||
rows = conn.execute(
|
||||
"""SELECT c.order_no, c.id, c.title, b.id, b.content_text
|
||||
FROM muse_content_chapter c JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE
|
||||
@ -110,9 +118,11 @@ def main(work_id, batch, file_, model, dry_run, report_md):
|
||||
if not (MIN_LEN <= len(exact) <= MAX_LEN):
|
||||
stats["拒绝"].append(f"#{no}: 长度越界({len(exact)}) {exact[:24]!r}")
|
||||
continue
|
||||
# 守卫4：行级垃圾特征——无特征的行=疑混正文
|
||||
# 守卫4：行级垃圾特征——无特征的行=疑混正文（孤立书名行水印豁免）
|
||||
lines = [ln for ln in exact.split("\n") if ln.strip()]
|
||||
plain = [ln for ln in lines if not GARBAGE_LINE_PAT.search(ln)]
|
||||
plain = [ln for ln in lines
|
||||
if not GARBAGE_LINE_PAT.search(ln)
|
||||
and not (work_title and ln.strip() == work_title)]
|
||||
if plain and len(lines) == 1:
|
||||
stats["拒绝"].append(f"#{no}: 无垃圾特征疑正文 {exact[:24]!r}")
|
||||
continue
|
||||
@ -121,10 +131,17 @@ def main(work_id, batch, file_, model, dry_run, report_md):
|
||||
for ln in plain:
|
||||
stats["拒绝"].append(f"#{no}: 拆行跳过疑正文行 {ln[:24]!r}")
|
||||
units = [(ln, reason + "(拆行)") for ln in lines
|
||||
if GARBAGE_LINE_PAT.search(ln) and MIN_LEN <= len(ln) <= MAX_LEN]
|
||||
if (GARBAGE_LINE_PAT.search(ln)
|
||||
or (work_title and ln.strip() == work_title))
|
||||
and MIN_LEN <= len(ln) <= MAX_LEN]
|
||||
else:
|
||||
units = [(exact, reason)]
|
||||
for u_exact, u_reason in units:
|
||||
# 水印裁剪：水印前的前缀（多为正文）不删，exact 缩到水印起点
|
||||
wm = WATERMARK_PAT.search(u_exact)
|
||||
if wm and wm.start() > 0:
|
||||
u_exact = u_exact[wm.start():]
|
||||
u_reason += "(水印裁剪)"
|
||||
# 本章两级匹配 → 邻章容错（章号偏移是 LLM 在长窗里的已知病）
|
||||
hit_no = None
|
||||
for cand in (no, no - 1, no + 1):
|
||||
|
||||
@ -45,13 +45,15 @@ PROMPT = """你是网文正文清洗探测器。下面是《{title}》第 {a}–
|
||||
@click.option("--work-id", type=int, required=True)
|
||||
@click.option("--win", "wins", type=int, multiple=True, help="指定窗号（可多次）；不给则全部窗")
|
||||
@click.option("--model", default="MiniMax-M3", show_default=True)
|
||||
@click.option("--fallback-model", default="deepseek-v4-flash", show_default=True,
|
||||
help="主模型失败（如上游敏感词拦截）时的备选；再失败则记录跳过该窗")
|
||||
@click.option("--force", is_flag=True, help="已有产物也重跑")
|
||||
def main(work_id, wins, model, force):
|
||||
def main(work_id, wins, model, fallback_model, force):
|
||||
d = OUT / str(work_id)
|
||||
manifest = json.loads((d / "manifest.json").read_text())
|
||||
title = manifest["title"]
|
||||
targets = [m for m in manifest["windows"] if not wins or m["win"] in wins]
|
||||
total_in = total_out = n_del = 0
|
||||
total_in = total_out = n_del = n_skip = 0
|
||||
for m in targets:
|
||||
f_out = d / f"deletions-{m['win']:03d}.json"
|
||||
if f_out.exists() and not force:
|
||||
@ -59,18 +61,35 @@ def main(work_id, wins, model, force):
|
||||
continue
|
||||
text = pathlib.Path(m["file"]).read_text()
|
||||
t0 = time.time()
|
||||
content, usage = chat(
|
||||
PROMPT.format(title=title, a=m["from"], b=m["to"], text=text),
|
||||
model=model)
|
||||
prompt = PROMPT.format(title=title, a=m["from"], b=m["to"], text=text)
|
||||
# 单窗失败不中断整书：主模型 → 备选模型 → 记录跳过（网文正文可能触发上游敏感词拦截）
|
||||
dels, used_model = None, model
|
||||
for cand in (model, fallback_model):
|
||||
try:
|
||||
content, usage = chat(prompt, model=cand)
|
||||
data = extract_json(content)
|
||||
dels = data.get("deletions", data if isinstance(data, list) else [])
|
||||
f_out.write_text(json.dumps({"deletions": dels}, ensure_ascii=False, indent=1))
|
||||
used_model = cand
|
||||
total_in += usage.get("prompt_tokens", 0)
|
||||
total_out += usage.get("completion_tokens", 0)
|
||||
break
|
||||
except (RuntimeError, ValueError) as e:
|
||||
click.echo(f"win-{m['win']:03d} {cand} 失败: {str(e)[:120]}", err=True)
|
||||
if dels is None:
|
||||
# 双模型全败：写空产物占位（含跳过原因），apply 端窗产物齐备可继续，审计可追
|
||||
f_out.write_text(json.dumps(
|
||||
{"deletions": [], "skipped": "主/备模型均失败（多为上游敏感词拦截）"},
|
||||
ensure_ascii=False, indent=1))
|
||||
n_skip += 1
|
||||
click.echo(f"win-{m['win']:03d}（第{m['from']}–{m['to']}章）探测跳过，该窗不清洗")
|
||||
continue
|
||||
f_out.write_text(json.dumps({"deletions": dels, "model": used_model},
|
||||
ensure_ascii=False, indent=1))
|
||||
n_del += len(dels)
|
||||
click.echo(f"win-{m['win']:03d}（第{m['from']}–{m['to']}章）检出 {len(dels)} 段 "
|
||||
tag = "" if used_model == model else f"（备选 {used_model}）"
|
||||
click.echo(f"win-{m['win']:03d}（第{m['from']}–{m['to']}章）检出 {len(dels)} 段{tag} "
|
||||
f"→ {f_out.name}（{time.time() - t0:.0f}s）")
|
||||
click.echo(f"探测完成：{len(targets)} 窗共检出 {n_del} 段；"
|
||||
click.echo(f"探测完成：{len(targets)} 窗共检出 {n_del} 段（跳过 {n_skip} 窗）；"
|
||||
f"token in={total_in:,} out={total_out:,}")
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user