框架: clean_apply加--report-md质检报告(每条删除带上下文对照,门①裁决证据面)
This commit is contained in:
parent
2b6f83c097
commit
bcd502688b
@ -47,16 +47,24 @@ def flex_pattern(exact):
|
|||||||
|
|
||||||
|
|
||||||
def try_remove(text, exact):
|
def try_remove(text, exact):
|
||||||
"""在 text 中删除 exact(精确→弹性两级)。返回 (新text, 命中次数, 实删字数, 方式)。"""
|
"""在 text 中删除 exact(精确→弹性两级)。
|
||||||
|
|
||||||
|
返回 (新text, 命中次数, 实删字数, 方式, 首次命中前文, 首次命中后文)。
|
||||||
|
前后文各取 ~60 字:质检报告用——人工确认删除段是独立垃圾而非正文一部分。
|
||||||
|
"""
|
||||||
n = text.count(exact)
|
n = text.count(exact)
|
||||||
if n:
|
if n:
|
||||||
return text.replace(exact, ""), n, len(exact) * n, "精确"
|
i = text.find(exact)
|
||||||
|
before, after = text[max(0, i - 60):i], text[i + len(exact):i + len(exact) + 60]
|
||||||
|
return text.replace(exact, ""), n, len(exact) * n, "精确", before, after
|
||||||
pat = flex_pattern(exact)
|
pat = flex_pattern(exact)
|
||||||
if pat:
|
if pat:
|
||||||
new, n = pat.subn("", text)
|
m = pat.search(text)
|
||||||
if n:
|
if m:
|
||||||
return new, n, len(text) - len(new), "空白弹性"
|
before, after = text[max(0, m.start() - 60):m.start()], text[m.end():m.end() + 60]
|
||||||
return text, 0, 0, ""
|
new, n = pat.subn("", text)
|
||||||
|
return new, n, len(text) - len(new), "空白弹性", before, after
|
||||||
|
return text, 0, 0, "", "", ""
|
||||||
|
|
||||||
|
|
||||||
@click.command()
|
@click.command()
|
||||||
@ -65,7 +73,8 @@ def try_remove(text, exact):
|
|||||||
@click.option("--file", "file_", type=click.Path(exists=True), required=True)
|
@click.option("--file", "file_", type=click.Path(exists=True), required=True)
|
||||||
@click.option("--model", default="MiniMax-M3", show_default=True, help="建议来源模型(审计标注)")
|
@click.option("--model", default="MiniMax-M3", show_default=True, help="建议来源模型(审计标注)")
|
||||||
@click.option("--dry-run", is_flag=True, help="只对账不落库")
|
@click.option("--dry-run", is_flag=True, help="只对账不落库")
|
||||||
def main(work_id, batch, file_, model, dry_run):
|
@click.option("--report-md", type=click.Path(), help="人读质检报告输出路径(每条删除带上下文对照)")
|
||||||
|
def main(work_id, batch, file_, model, dry_run, report_md):
|
||||||
raw = json.loads(pathlib.Path(file_).read_text())
|
raw = json.loads(pathlib.Path(file_).read_text())
|
||||||
if isinstance(raw, dict):
|
if isinstance(raw, dict):
|
||||||
raw = raw.get("deletions") or raw.get("删除") or []
|
raw = raw.get("deletions") or raw.get("删除") or []
|
||||||
@ -126,7 +135,7 @@ def main(work_id, batch, file_, model, dry_run):
|
|||||||
stats["拒绝"].append(f"#{no}: 含章题 {u_exact[:24]!r}")
|
stats["拒绝"].append(f"#{no}: 含章题 {u_exact[:24]!r}")
|
||||||
hit_no = -1 # 标记已裁决,不再试邻章
|
hit_no = -1 # 标记已裁决,不再试邻章
|
||||||
break
|
break
|
||||||
new, n, removed, how = try_remove(ch["text"], u_exact)
|
new, n, removed, how, before, after = try_remove(ch["text"], u_exact)
|
||||||
if n:
|
if n:
|
||||||
ch["text"] = new
|
ch["text"] = new
|
||||||
ch["removed"] += removed
|
ch["removed"] += removed
|
||||||
@ -137,7 +146,7 @@ def main(work_id, batch, file_, model, dry_run):
|
|||||||
if cand != no:
|
if cand != no:
|
||||||
stats["邻章命中"] += 1
|
stats["邻章命中"] += 1
|
||||||
note += f"(报#{no}实删#{cand})"
|
note += f"(报#{no}实删#{cand})"
|
||||||
ch["audit"].append((u_exact, n, note))
|
ch["audit"].append((u_exact, n, note, before, after))
|
||||||
stats["删除段"] += 1
|
stats["删除段"] += 1
|
||||||
stats["删除字数"] += removed
|
stats["删除字数"] += removed
|
||||||
hit_no = cand
|
hit_no = cand
|
||||||
@ -163,7 +172,7 @@ def main(work_id, batch, file_, model, dry_run):
|
|||||||
wc = len(re.sub(r"\s", "", ch["text"]))
|
wc = len(re.sub(r"\s", "", ch["text"]))
|
||||||
conn.execute("UPDATE muse_content_block SET content_text=%s, word_count=%s, updater=%s WHERE id=%s",
|
conn.execute("UPDATE muse_content_block SET content_text=%s, word_count=%s, updater=%s WHERE id=%s",
|
||||||
(ch["text"], wc, ACTOR, ch["blk_id"]))
|
(ch["text"], wc, ACTOR, ch["blk_id"]))
|
||||||
for exact, n, note in ch["audit"]:
|
for exact, n, note, _, _ in ch["audit"]:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""INSERT INTO example_clean_log (work_id, chapter_id, batch, removed_text, occurrences,
|
"""INSERT INTO example_clean_log (work_id, chapter_id, batch, removed_text, occurrences,
|
||||||
reason, model, creator, updater, tenant_id)
|
reason, model, creator, updater, tenant_id)
|
||||||
@ -184,8 +193,30 @@ def main(work_id, batch, file_, model, dry_run):
|
|||||||
# dry-run 打印完整删除明细:门①人工质检的数据源
|
# dry-run 打印完整删除明细:门①人工质检的数据源
|
||||||
click.echo("── 删除明细(质检用) ──")
|
click.echo("── 删除明细(质检用) ──")
|
||||||
for no, ch in dirty:
|
for no, ch in dirty:
|
||||||
for exact, n, note in ch["audit"]:
|
for exact, n, note, _, _ in ch["audit"]:
|
||||||
click.echo(f" [删] #{no} ×{n} {note or '(无理由)'}\n {exact!r}")
|
click.echo(f" [删] #{no} ×{n} {note or '(无理由)'}\n {exact!r}")
|
||||||
|
if report_md:
|
||||||
|
# 人读质检报告:每条删除带前后文对照——放行裁决的证据面
|
||||||
|
lines_md = [f"# 清洗质检报告(work={work_id} batch={batch}{mode})",
|
||||||
|
"",
|
||||||
|
f"拟删 **{stats['删除段']} 段 / {stats['删除字数']:,} 字**"
|
||||||
|
f"(弹性命中 {stats['弹性命中']}、邻章命中 {stats['邻章命中']});"
|
||||||
|
f"守卫拒绝 {len(stats['拒绝'])} 条(见文末)。",
|
||||||
|
"",
|
||||||
|
"> 看法:确认每条【删】段是独立垃圾、上下文(灰引文)是完整正文即可放行。", ""]
|
||||||
|
seq = 0
|
||||||
|
for no, ch in dirty:
|
||||||
|
for exact, n, note, before, after in ch["audit"]:
|
||||||
|
seq += 1
|
||||||
|
ctx_b = before.replace("\n", " ⏎ ").strip() or "(章首)"
|
||||||
|
ctx_a = after.replace("\n", " ⏎ ").strip() or "(章尾)"
|
||||||
|
body = exact.replace("\n", " ⏎ ")
|
||||||
|
lines_md += [f"### {seq}. 第{no}章《{ch['title']}》 ×{n} | {note or '(无理由)'}",
|
||||||
|
f"> …{ctx_b}", f"", f"**【删】** `{body}`", f"", f"> {ctx_a}…", ""]
|
||||||
|
lines_md += ["## 守卫拒绝清单(未删,含保护的疑正文行)", ""]
|
||||||
|
lines_md += [f"- {r}" for r in stats["拒绝"]] or ["- 无"]
|
||||||
|
pathlib.Path(report_md).write_text("\n".join(lines_md))
|
||||||
|
click.echo(f"质检报告已写 {report_md}")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user