82 lines
3.7 KiB
Python
82 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
||
"""clean skill：放量驱动——每书串行跑 prep→detect→apply 全链，可多进程分书并行。
|
||
|
||
断点续跑设计：
|
||
- prep 仅在该书 manifest 缺失时执行（防覆盖已切窗）；
|
||
- detect 天然跳过已有 deletions-*.json 的窗；
|
||
- apply 前查 example_clean_log 是否已有该批次（幂等防重删），且要求窗产物齐备。
|
||
单书失败不阻断后续书；重跑本脚本自动补缺。
|
||
"""
|
||
import json
|
||
import pathlib
|
||
import subprocess
|
||
import sys
|
||
|
||
import click
|
||
import psycopg
|
||
|
||
DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example"
|
||
"?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3")
|
||
TENANT = 1
|
||
OUT = pathlib.Path("/tmp/muse-clean")
|
||
HERE = pathlib.Path(__file__).resolve().parent
|
||
PY = str(pathlib.Path(sys.executable))
|
||
|
||
|
||
def run(args):
|
||
"""子进程跑 skill 脚本，回显输出；返回是否成功。"""
|
||
r = subprocess.run([PY, *args], capture_output=True, text=True)
|
||
for ln in (r.stdout + r.stderr).strip().splitlines():
|
||
click.echo(f" {ln}")
|
||
return r.returncode == 0
|
||
|
||
|
||
@click.command()
|
||
@click.option("--work-id", "work_ids", type=int, multiple=True, required=True)
|
||
@click.option("--batch", required=True, help="放量批次号（幂等判断依据，重跑请保持一致）")
|
||
@click.option("--report-dir", default="docs", show_default=True)
|
||
def main(work_ids, batch, report_dir):
|
||
with psycopg.connect(DSN) as conn:
|
||
titles = dict(conn.execute(
|
||
"SELECT id, title FROM muse_content_work WHERE tenant_id=%s AND deleted=FALSE AND id=ANY(%s)",
|
||
(TENANT, list(work_ids))).fetchall())
|
||
for wid in work_ids:
|
||
title = titles.get(wid, f"id{wid}")
|
||
click.echo(f"═══《{title}》(work={wid}) ═══")
|
||
d = OUT / str(wid)
|
||
# 1) 切窗（manifest 已存在则跳过——防覆盖已探测批次）
|
||
if not (d / "manifest.json").exists():
|
||
if not run([str(HERE / "clean_prep.py"), "--work-id", str(wid)]):
|
||
click.echo(f" [失败] prep，跳过本书")
|
||
continue
|
||
manifest = json.loads((d / "manifest.json").read_text())
|
||
# 2) 探测（detect 内部跳过已有产物窗；单窗失败中断，本书留待重跑补齐）
|
||
run([str(HERE / "clean_detect.py"), "--work-id", str(wid)])
|
||
missing = [m["win"] for m in manifest["windows"]
|
||
if not (d / f"deletions-{m['win']:03d}.json").exists()]
|
||
if missing:
|
||
click.echo(f" [缺窗] {missing}，本书暂不 apply（重跑本命令自动补）")
|
||
continue
|
||
# 3) 幂等防重删：该批次已落审计则跳过 apply
|
||
with psycopg.connect(DSN) as conn:
|
||
done = conn.execute(
|
||
"SELECT 1 FROM example_clean_log WHERE work_id=%s AND batch=%s LIMIT 1",
|
||
(wid, batch)).fetchone()
|
||
if done:
|
||
click.echo(f" [跳过] 批次 {batch} 已落库")
|
||
continue
|
||
# 4) 合并全窗建议 → 真删落库 + 质检报告
|
||
alls = []
|
||
for m in manifest["windows"]:
|
||
alls += json.loads((d / f"deletions-{m['win']:03d}.json").read_text())["deletions"]
|
||
(d / "deletions-all.json").write_text(
|
||
json.dumps({"deletions": alls}, ensure_ascii=False, indent=1))
|
||
report = pathlib.Path(report_dir) / f"清洗-{wid}-{title}.md"
|
||
ok = run([str(HERE / "clean_apply.py"), "--work-id", str(wid), "--batch", batch,
|
||
"--file", str(d / "deletions-all.json"), "--report-md", str(report)])
|
||
click.echo(f" {'✓ 完成' if ok else '[失败] apply'}《{title}》建议 {len(alls)} 条，报告 {report}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|