框架: clean_apply加--report-md质检报告(每条删除带上下文对照,门①裁决证据面)

This commit is contained in:
zizi 2026-07-13 14:13:02 +08:00
parent 2b6f83c097
commit bcd502688b

View File

@ -47,16 +47,24 @@ def flex_pattern(exact):
def try_remove(text, exact):
"""在 text 中删除 exact精确→弹性两级。返回 (新text, 命中次数, 实删字数, 方式)。"""
"""在 text 中删除 exact精确→弹性两级
返回 (新text, 命中次数, 实删字数, 方式, 首次命中前文, 首次命中后文)
前后文各取 ~60 质检报告用人工确认删除段是独立垃圾而非正文一部分
"""
n = text.count(exact)
if n:
return text.replace(exact, ""), n, len(exact) * n, "精确"
i = text.find(exact)
before, after = text[max(0, i - 60):i], text[i + len(exact):i + len(exact) + 60]
return text.replace(exact, ""), n, len(exact) * n, "精确", before, after
pat = flex_pattern(exact)
if pat:
new, n = pat.subn("", text)
if n:
return new, n, len(text) - len(new), "空白弹性"
return text, 0, 0, ""
m = pat.search(text)
if m:
before, after = text[max(0, m.start() - 60):m.start()], text[m.end():m.end() + 60]
new, n = pat.subn("", text)
return new, n, len(text) - len(new), "空白弹性", before, after
return text, 0, 0, "", "", ""
@click.command()
@ -65,7 +73,8 @@ def try_remove(text, exact):
@click.option("--file", "file_", type=click.Path(exists=True), required=True)
@click.option("--model", default="MiniMax-M3", show_default=True, help="建议来源模型(审计标注)")
@click.option("--dry-run", is_flag=True, help="只对账不落库")
def main(work_id, batch, file_, model, dry_run):
@click.option("--report-md", type=click.Path(), help="人读质检报告输出路径(每条删除带上下文对照)")
def main(work_id, batch, file_, model, dry_run, report_md):
raw = json.loads(pathlib.Path(file_).read_text())
if isinstance(raw, dict):
raw = raw.get("deletions") or raw.get("删除") or []
@ -126,7 +135,7 @@ def main(work_id, batch, file_, model, dry_run):
stats["拒绝"].append(f"#{no}: 含章题 {u_exact[:24]!r}")
hit_no = -1 # 标记已裁决,不再试邻章
break
new, n, removed, how = try_remove(ch["text"], u_exact)
new, n, removed, how, before, after = try_remove(ch["text"], u_exact)
if n:
ch["text"] = new
ch["removed"] += removed
@ -137,7 +146,7 @@ def main(work_id, batch, file_, model, dry_run):
if cand != no:
stats["邻章命中"] += 1
note += f"(报#{no}实删#{cand})"
ch["audit"].append((u_exact, n, note))
ch["audit"].append((u_exact, n, note, before, after))
stats["删除段"] += 1
stats["删除字数"] += removed
hit_no = cand
@ -163,7 +172,7 @@ def main(work_id, batch, file_, model, dry_run):
wc = len(re.sub(r"\s", "", ch["text"]))
conn.execute("UPDATE muse_content_block SET content_text=%s, word_count=%s, updater=%s WHERE id=%s",
(ch["text"], wc, ACTOR, ch["blk_id"]))
for exact, n, note in ch["audit"]:
for exact, n, note, _, _ in ch["audit"]:
conn.execute(
"""INSERT INTO example_clean_log (work_id, chapter_id, batch, removed_text, occurrences,
reason, model, creator, updater, tenant_id)
@ -184,8 +193,30 @@ def main(work_id, batch, file_, model, dry_run):
# dry-run 打印完整删除明细:门①人工质检的数据源
click.echo("── 删除明细(质检用) ──")
for no, ch in dirty:
for exact, n, note in ch["audit"]:
for exact, n, note, _, _ in ch["audit"]:
click.echo(f" [删] #{no} ×{n} {note or '(无理由)'}\n {exact!r}")
if report_md:
# 人读质检报告:每条删除带前后文对照——放行裁决的证据面
lines_md = [f"# 清洗质检报告work={work_id} batch={batch}{mode}",
"",
f"拟删 **{stats['删除段']} 段 / {stats['删除字数']:,} 字**"
f"(弹性命中 {stats['弹性命中']}、邻章命中 {stats['邻章命中']}"
f"守卫拒绝 {len(stats['拒绝'])} 条(见文末)。",
"",
"> 看法:确认每条【删】段是独立垃圾、上下文(灰引文)是完整正文即可放行。", ""]
seq = 0
for no, ch in dirty:
for exact, n, note, before, after in ch["audit"]:
seq += 1
ctx_b = before.replace("\n", "").strip() or "(章首)"
ctx_a = after.replace("\n", "").strip() or "(章尾)"
body = exact.replace("\n", "")
lines_md += [f"### {seq}. 第{no}章《{ch['title']}×{n} {note or '(无理由)'}",
f"> …{ctx_b}", f"", f"**【删】** `{body}`", f"", f"> {ctx_a}", ""]
lines_md += ["## 守卫拒绝清单(未删,含保护的疑正文行)", ""]
lines_md += [f"- {r}" for r in stats["拒绝"]] or ["- 无"]
pathlib.Path(report_md).write_text("\n".join(lines_md))
click.echo(f"质检报告已写 {report_md}")
if __name__ == "__main__":