66 lines
2.9 KiB
Python
66 lines
2.9 KiB
Python
#!/usr/bin/env python3
|
||
"""clean skill：按章对齐切窗（LLM 探测输入面）。窗文件含章题标记行，便于 LLM 报 chapter_order。"""
|
||
import json
|
||
import pathlib
|
||
import re
|
||
import sys
|
||
|
||
import click
|
||
import psycopg
|
||
|
||
DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example"
|
||
"?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3")
|
||
TENANT = 1
|
||
OUT = pathlib.Path("/tmp/muse-clean")
|
||
|
||
|
||
@click.command()
|
||
@click.option("--work-id", type=int, required=True)
|
||
@click.option("--window", type=int, default=100000, show_default=True, help="每窗目标字符数（章对齐）")
|
||
@click.option("--from", "from_", type=int, default=1, help="起始章 order_no")
|
||
@click.option("--to", type=int, default=0, help="结束章 order_no（0=到末章）")
|
||
def main(work_id, window, from_, to):
|
||
with psycopg.connect(DSN) as conn:
|
||
title = conn.execute("SELECT title FROM muse_content_work WHERE id=%s", (work_id,)).fetchone()[0]
|
||
sql = """SELECT c.order_no, c.title, b.content_text
|
||
FROM muse_content_chapter c JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE
|
||
WHERE c.tenant_id=%s AND c.work_id=%s AND c.deleted=FALSE AND c.order_no>=%s"""
|
||
args = [TENANT, work_id, from_]
|
||
if to:
|
||
sql += " AND c.order_no<=%s"
|
||
args.append(to)
|
||
rows = conn.execute(sql + " ORDER BY c.order_no", args).fetchall()
|
||
d = OUT / str(work_id)
|
||
d.mkdir(parents=True, exist_ok=True)
|
||
wins, cur, cur_len, cur_span = [], [], 0, []
|
||
for no, ct, text in rows:
|
||
# 章题标记行：供 LLM 定位 chapter_order；clean_apply 不会删除标记行本身
|
||
cur.append(f"\n【第{no}章 | {ct}】\n{text}")
|
||
cur_len += len(text)
|
||
cur_span.append(no)
|
||
if cur_len >= window:
|
||
wins.append((cur_span[0], cur_span[-1], "".join(cur)))
|
||
cur, cur_len, cur_span = [], 0, []
|
||
if cur:
|
||
wins.append((cur_span[0], cur_span[-1], "".join(cur)))
|
||
manifest = []
|
||
for i, (a, b, text) in enumerate(wins, 1):
|
||
f = d / f"win-{i:03d}.txt"
|
||
f.write_text(f"《{title}》 第{a}–{b}章 原文窗口（章题标记行格式:【第N章 | 章题】）\n{text}")
|
||
manifest.append({"win": i, "file": str(f), "from": a, "to": b, "chars": len(text)})
|
||
(d / "manifest.json").write_text(json.dumps(
|
||
{"work_id": work_id, "title": title, "windows": manifest}, ensure_ascii=False, indent=1))
|
||
click.echo(f"《{title}》切窗完成: {len(wins)} 窗 → {d}/win-*.txt（manifest.json 就绪）")
|
||
for m in manifest[:5]:
|
||
click.echo(f" win-{m['win']:03d} 第{m['from']}–{m['to']}章 {m['chars']:,}字")
|
||
if len(manifest) > 5:
|
||
click.echo(f" … 共 {len(manifest)} 窗")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
main()
|
||
except psycopg.Error as e:
|
||
click.echo(f"[db错误] {type(e).__name__}: {e}", err=True)
|
||
sys.exit(1)
|