框架: clean_apply加--report-md质检报告(每条删除带上下文对照,门①裁决证据面)
This commit is contained in:
parent
2b6f83c097
commit
bcd502688b
@ -47,16 +47,24 @@ def flex_pattern(exact):
|
||||
|
||||
|
||||
def try_remove(text, exact):
|
||||
"""在 text 中删除 exact(精确→弹性两级)。返回 (新text, 命中次数, 实删字数, 方式)。"""
|
||||
"""在 text 中删除 exact(精确→弹性两级)。
|
||||
|
||||
返回 (新text, 命中次数, 实删字数, 方式, 首次命中前文, 首次命中后文)。
|
||||
前后文各取 ~60 字:质检报告用——人工确认删除段是独立垃圾而非正文一部分。
|
||||
"""
|
||||
n = text.count(exact)
|
||||
if n:
|
||||
return text.replace(exact, ""), n, len(exact) * n, "精确"
|
||||
i = text.find(exact)
|
||||
before, after = text[max(0, i - 60):i], text[i + len(exact):i + len(exact) + 60]
|
||||
return text.replace(exact, ""), n, len(exact) * n, "精确", before, after
|
||||
pat = flex_pattern(exact)
|
||||
if pat:
|
||||
new, n = pat.subn("", text)
|
||||
if n:
|
||||
return new, n, len(text) - len(new), "空白弹性"
|
||||
return text, 0, 0, ""
|
||||
m = pat.search(text)
|
||||
if m:
|
||||
before, after = text[max(0, m.start() - 60):m.start()], text[m.end():m.end() + 60]
|
||||
new, n = pat.subn("", text)
|
||||
return new, n, len(text) - len(new), "空白弹性", before, after
|
||||
return text, 0, 0, "", "", ""
|
||||
|
||||
|
||||
@click.command()
|
||||
@ -65,7 +73,8 @@ def try_remove(text, exact):
|
||||
@click.option("--file", "file_", type=click.Path(exists=True), required=True)
|
||||
@click.option("--model", default="MiniMax-M3", show_default=True, help="建议来源模型(审计标注)")
|
||||
@click.option("--dry-run", is_flag=True, help="只对账不落库")
|
||||
def main(work_id, batch, file_, model, dry_run):
|
||||
@click.option("--report-md", type=click.Path(), help="人读质检报告输出路径(每条删除带上下文对照)")
|
||||
def main(work_id, batch, file_, model, dry_run, report_md):
|
||||
raw = json.loads(pathlib.Path(file_).read_text())
|
||||
if isinstance(raw, dict):
|
||||
raw = raw.get("deletions") or raw.get("删除") or []
|
||||
@ -126,7 +135,7 @@ def main(work_id, batch, file_, model, dry_run):
|
||||
stats["拒绝"].append(f"#{no}: 含章题 {u_exact[:24]!r}")
|
||||
hit_no = -1 # 标记已裁决,不再试邻章
|
||||
break
|
||||
new, n, removed, how = try_remove(ch["text"], u_exact)
|
||||
new, n, removed, how, before, after = try_remove(ch["text"], u_exact)
|
||||
if n:
|
||||
ch["text"] = new
|
||||
ch["removed"] += removed
|
||||
@ -137,7 +146,7 @@ def main(work_id, batch, file_, model, dry_run):
|
||||
if cand != no:
|
||||
stats["邻章命中"] += 1
|
||||
note += f"(报#{no}实删#{cand})"
|
||||
ch["audit"].append((u_exact, n, note))
|
||||
ch["audit"].append((u_exact, n, note, before, after))
|
||||
stats["删除段"] += 1
|
||||
stats["删除字数"] += removed
|
||||
hit_no = cand
|
||||
@ -163,7 +172,7 @@ def main(work_id, batch, file_, model, dry_run):
|
||||
wc = len(re.sub(r"\s", "", ch["text"]))
|
||||
conn.execute("UPDATE muse_content_block SET content_text=%s, word_count=%s, updater=%s WHERE id=%s",
|
||||
(ch["text"], wc, ACTOR, ch["blk_id"]))
|
||||
for exact, n, note in ch["audit"]:
|
||||
for exact, n, note, _, _ in ch["audit"]:
|
||||
conn.execute(
|
||||
"""INSERT INTO example_clean_log (work_id, chapter_id, batch, removed_text, occurrences,
|
||||
reason, model, creator, updater, tenant_id)
|
||||
@ -184,8 +193,30 @@ def main(work_id, batch, file_, model, dry_run):
|
||||
# dry-run 打印完整删除明细:门①人工质检的数据源
|
||||
click.echo("── 删除明细(质检用) ──")
|
||||
for no, ch in dirty:
|
||||
for exact, n, note in ch["audit"]:
|
||||
for exact, n, note, _, _ in ch["audit"]:
|
||||
click.echo(f" [删] #{no} ×{n} {note or '(无理由)'}\n {exact!r}")
|
||||
if report_md:
|
||||
# 人读质检报告:每条删除带前后文对照——放行裁决的证据面
|
||||
lines_md = [f"# 清洗质检报告(work={work_id} batch={batch}{mode})",
|
||||
"",
|
||||
f"拟删 **{stats['删除段']} 段 / {stats['删除字数']:,} 字**"
|
||||
f"(弹性命中 {stats['弹性命中']}、邻章命中 {stats['邻章命中']});"
|
||||
f"守卫拒绝 {len(stats['拒绝'])} 条(见文末)。",
|
||||
"",
|
||||
"> 看法:确认每条【删】段是独立垃圾、上下文(灰引文)是完整正文即可放行。", ""]
|
||||
seq = 0
|
||||
for no, ch in dirty:
|
||||
for exact, n, note, before, after in ch["audit"]:
|
||||
seq += 1
|
||||
ctx_b = before.replace("\n", " ⏎ ").strip() or "(章首)"
|
||||
ctx_a = after.replace("\n", " ⏎ ").strip() or "(章尾)"
|
||||
body = exact.replace("\n", " ⏎ ")
|
||||
lines_md += [f"### {seq}. 第{no}章《{ch['title']}》 ×{n} | {note or '(无理由)'}",
|
||||
f"> …{ctx_b}", f"", f"**【删】** `{body}`", f"", f"> {ctx_a}…", ""]
|
||||
lines_md += ["## 守卫拒绝清单(未删,含保护的疑正文行)", ""]
|
||||
lines_md += [f"- {r}" for r in stats["拒绝"]] or ["- 无"]
|
||||
pathlib.Path(report_md).write_text("\n".join(lines_md))
|
||||
click.echo(f"质检报告已写 {report_md}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user