From 0a3a576704d4c0e73ea017f651ee7bf855201fec Mon Sep 17 00:00:00 2001 From: zizi Date: Mon, 13 Jul 2026 15:29:05 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A1=86=E6=9E=B6:=20clean=E6=8A=97=E6=80=A7?= =?UTF-8?q?=E4=B8=89=E8=BF=9E=E2=80=94=E2=80=94detect=E5=8D=95=E7=AA=97?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E9=99=8D=E7=BA=A7=E5=A4=87=E9=80=89=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E4=B8=8D=E6=96=AD=E4=B9=A6+apply=E4=B9=A6=E5=90=8D?= =?UTF-8?q?=E6=B0=B4=E5=8D=B0=E8=B1=81=E5=85=8D+=E6=B0=B4=E5=8D=B0?= =?UTF-8?q?=E8=A3=81=E5=89=AA=E9=98=B2=E6=AD=A3=E6=96=87=E5=89=8D=E7=BC=80?= =?UTF-8?q?=E8=AF=AF=E5=88=A0(=E8=B7=AFC=E5=AE=9E=E6=B5=8B=E7=97=85?= =?UTF-8?q?=E5=AF=B9=E7=97=87)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .claude/skills/clean/scripts/clean_apply.py | 23 +++++++++-- .claude/skills/clean/scripts/clean_detect.py | 43 ++++++++++++++------ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/.claude/skills/clean/scripts/clean_apply.py b/.claude/skills/clean/scripts/clean_apply.py index ae6a3e9..6869504 100644 --- a/.claude/skills/clean/scripts/clean_apply.py +++ b/.claude/skills/clean/scripts/clean_apply.py @@ -35,6 +35,10 @@ GARBAGE_LINE_PAT = re.compile( r"|手打中|稍等片刻|重新刷新|获取最新", # 盗版站占位残留（1040-1060 验证窗实测） re.I) +# 水印裁剪：盗版源把水印粘在正文行尾（如『“我草！”百镀一下“××”最新章节第一时间免费阅读。』）， +# LLM 会把整行报上来——删除段若含此类明确水印且水印前有前缀（多为正文），只删水印部分。 +WATERMARK_PAT = re.compile(r"(?:百镀一下|百度一下|一秒记住).{0,45}?(?:免费阅读。?|免费读！?|阅读。)") + def flex_pattern(exact): """空白弹性正则：非空白文字逐字 escape，连续空白段 → \\s+。 @@ -92,6 +96,10 @@ def main(work_id, batch, file_, model, dry_run, report_md): targets = sorted({n for no in by_ch for n in (no - 1, no, no + 1) if n >= 1}) stats = {"删除段": 0, "删除字数": 0, "拒绝": [], "弹性命中": 0, "邻章命中": 0} with psycopg.connect(DSN) as conn: + # 书名水印豁免：盗版源每章插孤立书名行——孤行精确等于本书书名不可能是正文叙述 + work_title = (conn.execute( + "SELECT title FROM muse_content_work WHERE tenant_id=%s AND id=%s", + (TENANT, work_id)).fetchone() or ("",))[0] rows = conn.execute( """SELECT c.order_no, c.id, c.title, b.id, b.content_text FROM muse_content_chapter c JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE @@ -110,9 +118,11 @@ def main(work_id, batch, file_, model, dry_run, report_md): if not (MIN_LEN <= len(exact) <= MAX_LEN): stats["拒绝"].append(f"#{no}: 长度越界({len(exact)}) {exact[:24]!r}") continue - # 守卫4：行级垃圾特征——无特征的行=疑混正文 + # 守卫4：行级垃圾特征——无特征的行=疑混正文（孤立书名行水印豁免） lines = [ln for ln in exact.split("\n") if ln.strip()] - plain = [ln for ln in lines if not GARBAGE_LINE_PAT.search(ln)] + plain = [ln for ln in lines + if not GARBAGE_LINE_PAT.search(ln) + and not (work_title and ln.strip() == work_title)] if plain and len(lines) == 1: stats["拒绝"].append(f"#{no}: 无垃圾特征疑正文 {exact[:24]!r}") continue @@ -121,10 +131,17 @@ def main(work_id, batch, file_, model, dry_run, report_md): for ln in plain: stats["拒绝"].append(f"#{no}: 拆行跳过疑正文行 {ln[:24]!r}") units = [(ln, reason + "(拆行)") for ln in lines - if GARBAGE_LINE_PAT.search(ln) and MIN_LEN <= len(ln) <= MAX_LEN] + if (GARBAGE_LINE_PAT.search(ln) + or (work_title and ln.strip() == work_title)) + and MIN_LEN <= len(ln) <= MAX_LEN] else: units = [(exact, reason)] for u_exact, u_reason in units: + # 水印裁剪：水印前的前缀（多为正文）不删，exact 缩到水印起点 + wm = WATERMARK_PAT.search(u_exact) + if wm and wm.start() > 0: + u_exact = u_exact[wm.start():] + u_reason += "(水印裁剪)" # 本章两级匹配 → 邻章容错（章号偏移是 LLM 在长窗里的已知病） hit_no = None for cand in (no, no - 1, no + 1): diff --git a/.claude/skills/clean/scripts/clean_detect.py b/.claude/skills/clean/scripts/clean_detect.py index a03e3e3..f0019b3 100644 --- a/.claude/skills/clean/scripts/clean_detect.py +++ b/.claude/skills/clean/scripts/clean_detect.py @@ -45,13 +45,15 @@ PROMPT = """你是网文正文清洗探测器。下面是《{title}》第 {a}– @click.option("--work-id", type=int, required=True) @click.option("--win", "wins", type=int, multiple=True, help="指定窗号（可多次）；不给则全部窗") @click.option("--model", default="MiniMax-M3", show_default=True) +@click.option("--fallback-model", default="deepseek-v4-flash", show_default=True, + help="主模型失败（如上游敏感词拦截）时的备选；再失败则记录跳过该窗") @click.option("--force", is_flag=True, help="已有产物也重跑") -def main(work_id, wins, model, force): +def main(work_id, wins, model, fallback_model, force): d = OUT / str(work_id) manifest = json.loads((d / "manifest.json").read_text()) title = manifest["title"] targets = [m for m in manifest["windows"] if not wins or m["win"] in wins] - total_in = total_out = n_del = 0 + total_in = total_out = n_del = n_skip = 0 for m in targets: f_out = d / f"deletions-{m['win']:03d}.json" if f_out.exists() and not force: @@ -59,18 +61,35 @@ def main(work_id, wins, model, force): continue text = pathlib.Path(m["file"]).read_text() t0 = time.time() - content, usage = chat( - PROMPT.format(title=title, a=m["from"], b=m["to"], text=text), - model=model) - data = extract_json(content) - dels = data.get("deletions", data if isinstance(data, list) else []) - f_out.write_text(json.dumps({"deletions": dels}, ensure_ascii=False, indent=1)) - total_in += usage.get("prompt_tokens", 0) - total_out += usage.get("completion_tokens", 0) + prompt = PROMPT.format(title=title, a=m["from"], b=m["to"], text=text) + # 单窗失败不中断整书：主模型 → 备选模型 → 记录跳过（网文正文可能触发上游敏感词拦截） + dels, used_model = None, model + for cand in (model, fallback_model): + try: + content, usage = chat(prompt, model=cand) + data = extract_json(content) + dels = data.get("deletions", data if isinstance(data, list) else []) + used_model = cand + total_in += usage.get("prompt_tokens", 0) + total_out += usage.get("completion_tokens", 0) + break + except (RuntimeError, ValueError) as e: + click.echo(f"win-{m['win']:03d} {cand} 失败: {str(e)[:120]}", err=True) + if dels is None: + # 双模型全败：写空产物占位（含跳过原因），apply 端窗产物齐备可继续，审计可追 + f_out.write_text(json.dumps( + {"deletions": [], "skipped": "主/备模型均失败（多为上游敏感词拦截）"}, + ensure_ascii=False, indent=1)) + n_skip += 1 + click.echo(f"win-{m['win']:03d}（第{m['from']}–{m['to']}章）探测跳过，该窗不清洗") + continue + f_out.write_text(json.dumps({"deletions": dels, "model": used_model}, + ensure_ascii=False, indent=1)) n_del += len(dels) - click.echo(f"win-{m['win']:03d}（第{m['from']}–{m['to']}章）检出 {len(dels)} 段 " + tag = "" if used_model == model else f"（备选 {used_model}）" + click.echo(f"win-{m['win']:03d}（第{m['from']}–{m['to']}章）检出 {len(dels)} 段{tag} " f"→ {f_out.name}（{time.time() - t0:.0f}s）") - click.echo(f"探测完成：{len(targets)} 窗共检出 {n_del} 段；" + click.echo(f"探测完成：{len(targets)} 窗共检出 {n_del} 段（跳过 {n_skip} 窗）；" f"token in={total_in:,} out={total_out:,}")