框架: 基础面验收轮——import解析器补双章区间号/分页尾巴/残破实体三规则(机战无限2901章找回巨章内71双章,超神机械师1455章清11分页空章);quality_report交付(字数/分章/章题/段落/垃圾嫌疑五面);clean skill交付(LLM探测+代码精确删除+守卫规则+审计表92);parse_ingest键名归一(StructuredOutput仅ASCII键教训);README§九刷2026-07-13快照(compact锚点)
This commit is contained in:
parent
6c189933dd
commit
11cdf93347
39
.claude/skills/clean/SKILL.md
Normal file
39
.claude/skills/clean/SKILL.md
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
---
|
||||||
|
name: clean
|
||||||
|
description: LLM 辅助正文清洗——LLM 按窗口(约10万字)只输出待删垃圾段原文与理由,代码做精确匹配删除并全程留审计(example_clean_log)。LLM 不改写正文,删不删由守卫规则最终裁决。
|
||||||
|
---
|
||||||
|
|
||||||
|
# clean —— LLM 检测 + 代码执行的正文清洗
|
||||||
|
|
||||||
|
分工(创始人方案 2026-07-13):**规则层**已在 import 解决结构性垃圾(水印正则/重贴章题/目录页/分页尾巴/残破实体);本 skill 处理**语义垃圾**——变体书站广告、作者拉票/PS 段、乱码水印等正则打不全的散落噪声。LLM 只当探测器(输出待删片段逐字原文),删除由脚本执行:零改写、可审计、可回放。
|
||||||
|
|
||||||
|
## 流程
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1) 备窗口:按章对齐切窗(默认 ~10万字/窗,写 /tmp/muse-clean/<work>/win-*.txt + manifest)
|
||||||
|
.venv/bin/python .claude/skills/clean/scripts/clean_prep.py --work-id 5 --window 100000
|
||||||
|
|
||||||
|
# 2) 探测:每窗一次 LLM 调用(haiku,编排层派发;prompt 模板见下)→ 每窗一份 deletions JSON
|
||||||
|
|
||||||
|
# 3) 执行:精确匹配删除 + 审计入库 + 字数刷新(守卫规则见下)
|
||||||
|
.venv/bin/python .claude/skills/clean/scripts/clean_apply.py --work-id 5 --batch <批次号> --file /tmp/muse-clean/5/deletions-all.json
|
||||||
|
|
||||||
|
# 4) 审查:删除对账(每章删了几段几字)+ 审计行抽查
|
||||||
|
.venv/bin/python .claude/skills/db/scripts/db.py query "SELECT reason, count(*), sum(length(removed_text)) FROM example_clean_log WHERE work_id=5 AND batch='<批次号>' GROUP BY reason"
|
||||||
|
```
|
||||||
|
|
||||||
|
## 探测 prompt 模板(编排层内联给 haiku)
|
||||||
|
|
||||||
|
> 你是网文正文清洗探测器。下面是《书名》第 A–B 章的原文(含章题行)。找出**所有非正文垃圾**:书站广告及变体(如「一秒记住♂粒÷小÷说→网」)、网址残留、求票拉票/打赏鸣谢、章尾作者话(ps:…)、上架感言类整段、乱码水印。输出 deletions 数组,每项 `{chapter_order, exact, reason}`:`exact` 必须与原文**逐字一致**(含标点空白,≤300字,一段一项);只报垃圾,拿不准的不报;**情节正文一个字都不许报**。章题行不报(即使含求票字样——章题另行处理)。
|
||||||
|
|
||||||
|
## 守卫规则(脚本硬闸,LLM 建议 ≠ 必删)
|
||||||
|
|
||||||
|
- `exact` 长度 4–500 字;在指定章内**逐字命中**才删(命中多次全删并记 occurrences);
|
||||||
|
- 不得命中/包含章题行;单章累计删除 ≤ 章长 20%(超限整章建议弃用并报告);
|
||||||
|
- 每次删除一行审计(原文/理由/模型/批次),源 txt 永远不动(可整书重导回滚);
|
||||||
|
- 删完刷新 block.word_count 与 work.word_count。
|
||||||
|
|
||||||
|
## 量级参考(2026-07-13 估)
|
||||||
|
|
||||||
|
- 拆书范围(5 书×前50章 ≈ 90万字)≈ 9–12 窗 ≈ 12 次 haiku 调用;
|
||||||
|
- 全语料(8 书 ≈ 3,300万字)≈ 340 窗 ≈ 350 次 haiku 调用(输入 ~35M token 量级)。
|
||||||
109
.claude/skills/clean/scripts/clean_apply.py
Normal file
109
.claude/skills/clean/scripts/clean_apply.py
Normal file
@ -0,0 +1,109 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""clean skill:精确匹配删除执行器(LLM 建议 ≠ 必删,守卫规则最终裁决)+ 审计入库。
|
||||||
|
|
||||||
|
输入 deletions JSON:[{chapter_order|chapter, exact, reason}](键名 ASCII,兼容中文键)。
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import click
|
||||||
|
import psycopg
|
||||||
|
|
||||||
|
DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example"
|
||||||
|
"?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3")
|
||||||
|
TENANT, ACTOR = 1, "1"
|
||||||
|
MIN_LEN, MAX_LEN = 4, 500
|
||||||
|
MAX_CH_RATIO = 0.20 # 单章累计删除上限(占章长比)
|
||||||
|
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
@click.option("--work-id", type=int, required=True)
|
||||||
|
@click.option("--batch", required=True, help="清洗批次号(审计用)")
|
||||||
|
@click.option("--file", "file_", type=click.Path(exists=True), required=True)
|
||||||
|
@click.option("--model", default="haiku", show_default=True, help="建议来源模型(审计标注)")
|
||||||
|
@click.option("--dry-run", is_flag=True, help="只对账不落库")
|
||||||
|
def main(work_id, batch, file_, model, dry_run):
|
||||||
|
raw = json.loads(pathlib.Path(file_).read_text())
|
||||||
|
if isinstance(raw, dict):
|
||||||
|
raw = raw.get("deletions") or raw.get("删除") or []
|
||||||
|
# 按章分组
|
||||||
|
by_ch = {}
|
||||||
|
for d in raw:
|
||||||
|
no = d.get("chapter_order") or d.get("chapter") or d.get("章序")
|
||||||
|
exact = d.get("exact") or d.get("原文")
|
||||||
|
if no is None or not exact:
|
||||||
|
continue
|
||||||
|
by_ch.setdefault(int(no), []).append((exact, d.get("reason") or d.get("理由") or ""))
|
||||||
|
|
||||||
|
stats = {"删除段": 0, "删除字数": 0, "拒绝": []}
|
||||||
|
with psycopg.connect(DSN) as conn:
|
||||||
|
for no, dels in sorted(by_ch.items()):
|
||||||
|
row = conn.execute(
|
||||||
|
"""SELECT c.id, c.title, b.id, b.content_text
|
||||||
|
FROM muse_content_chapter c JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE
|
||||||
|
WHERE c.tenant_id=%s AND c.work_id=%s AND c.order_no=%s AND c.deleted=FALSE""",
|
||||||
|
(TENANT, work_id, no)).fetchone()
|
||||||
|
if not row:
|
||||||
|
stats["拒绝"].append(f"#{no}: 章不存在")
|
||||||
|
continue
|
||||||
|
ch_id, ch_title, blk_id, text = row
|
||||||
|
new_text = text
|
||||||
|
ch_removed = 0
|
||||||
|
applied = []
|
||||||
|
for exact, reason in dels:
|
||||||
|
# 守卫1:长度界
|
||||||
|
if not (MIN_LEN <= len(exact) <= MAX_LEN):
|
||||||
|
stats["拒绝"].append(f"#{no}: 长度越界({len(exact)}) {exact[:24]!r}")
|
||||||
|
continue
|
||||||
|
# 守卫2:不得含/命中章题
|
||||||
|
if ch_title and ch_title in exact:
|
||||||
|
stats["拒绝"].append(f"#{no}: 含章题 {exact[:24]!r}")
|
||||||
|
continue
|
||||||
|
n = new_text.count(exact)
|
||||||
|
if n == 0:
|
||||||
|
stats["拒绝"].append(f"#{no}: 未命中 {exact[:24]!r}")
|
||||||
|
continue
|
||||||
|
new_text = new_text.replace(exact, "")
|
||||||
|
ch_removed += len(exact) * n
|
||||||
|
applied.append((exact, n, reason))
|
||||||
|
# 守卫3:单章删除比例硬顶
|
||||||
|
if ch_removed > len(text) * MAX_CH_RATIO:
|
||||||
|
stats["拒绝"].append(f"#{no}: 累计删除 {ch_removed} 字超章长20%上限,整章放弃")
|
||||||
|
continue
|
||||||
|
if not applied:
|
||||||
|
continue
|
||||||
|
new_text = re.sub(r'\n{3,}', '\n\n', new_text) # 删除后合并多余空行
|
||||||
|
stats["删除段"] += len(applied)
|
||||||
|
stats["删除字数"] += ch_removed
|
||||||
|
if dry_run:
|
||||||
|
continue
|
||||||
|
wc = len(re.sub(r'\s', '', new_text))
|
||||||
|
conn.execute("UPDATE muse_content_block SET content_text=%s, word_count=%s, updater=%s WHERE id=%s",
|
||||||
|
(new_text, wc, ACTOR, blk_id))
|
||||||
|
for exact, n, reason in applied:
|
||||||
|
conn.execute(
|
||||||
|
"""INSERT INTO example_clean_log (work_id, chapter_id, batch, removed_text, occurrences,
|
||||||
|
reason, model, creator, updater, tenant_id)
|
||||||
|
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||||
|
(work_id, ch_id, batch, exact, n, reason[:200], model, ACTOR, ACTOR, TENANT))
|
||||||
|
if not dry_run:
|
||||||
|
conn.execute(
|
||||||
|
"""UPDATE muse_content_work w SET word_count=(
|
||||||
|
SELECT COALESCE(sum(b.word_count),0) FROM muse_content_block b
|
||||||
|
WHERE b.tenant_id=%s AND b.work_id=w.id AND b.deleted=FALSE), updater=%s
|
||||||
|
WHERE w.id=%s""", (TENANT, ACTOR, work_id))
|
||||||
|
conn.commit()
|
||||||
|
mode = "(dry-run 未落库)" if dry_run else ""
|
||||||
|
click.echo(f"清洗执行{mode}: 删除 {stats['删除段']} 段 / {stats['删除字数']:,} 字;拒绝 {len(stats['拒绝'])} 条")
|
||||||
|
for r in stats["拒绝"][:12]:
|
||||||
|
click.echo(f" [拒] {r}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except psycopg.Error as e:
|
||||||
|
click.echo(f"[db错误] {type(e).__name__}: {e}", err=True)
|
||||||
|
sys.exit(1)
|
||||||
65
.claude/skills/clean/scripts/clean_prep.py
Normal file
65
.claude/skills/clean/scripts/clean_prep.py
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""clean skill:按章对齐切窗(LLM 探测输入面)。窗文件含章题标记行,便于 LLM 报 chapter_order。"""
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import click
|
||||||
|
import psycopg
|
||||||
|
|
||||||
|
DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example"
|
||||||
|
"?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3")
|
||||||
|
TENANT = 1
|
||||||
|
OUT = pathlib.Path("/tmp/muse-clean")
|
||||||
|
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
@click.option("--work-id", type=int, required=True)
|
||||||
|
@click.option("--window", type=int, default=100000, show_default=True, help="每窗目标字符数(章对齐)")
|
||||||
|
@click.option("--from", "from_", type=int, default=1, help="起始章 order_no")
|
||||||
|
@click.option("--to", type=int, default=0, help="结束章 order_no(0=到末章)")
|
||||||
|
def main(work_id, window, from_, to):
|
||||||
|
with psycopg.connect(DSN) as conn:
|
||||||
|
title = conn.execute("SELECT title FROM muse_content_work WHERE id=%s", (work_id,)).fetchone()[0]
|
||||||
|
sql = """SELECT c.order_no, c.title, b.content_text
|
||||||
|
FROM muse_content_chapter c JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE
|
||||||
|
WHERE c.tenant_id=%s AND c.work_id=%s AND c.deleted=FALSE AND c.order_no>=%s"""
|
||||||
|
args = [TENANT, work_id, from_]
|
||||||
|
if to:
|
||||||
|
sql += " AND c.order_no<=%s"
|
||||||
|
args.append(to)
|
||||||
|
rows = conn.execute(sql + " ORDER BY c.order_no", args).fetchall()
|
||||||
|
d = OUT / str(work_id)
|
||||||
|
d.mkdir(parents=True, exist_ok=True)
|
||||||
|
wins, cur, cur_len, cur_span = [], [], 0, []
|
||||||
|
for no, ct, text in rows:
|
||||||
|
# 章题标记行:供 LLM 定位 chapter_order;clean_apply 不会删除标记行本身
|
||||||
|
cur.append(f"\n【第{no}章 | {ct}】\n{text}")
|
||||||
|
cur_len += len(text)
|
||||||
|
cur_span.append(no)
|
||||||
|
if cur_len >= window:
|
||||||
|
wins.append((cur_span[0], cur_span[-1], "".join(cur)))
|
||||||
|
cur, cur_len, cur_span = [], 0, []
|
||||||
|
if cur:
|
||||||
|
wins.append((cur_span[0], cur_span[-1], "".join(cur)))
|
||||||
|
manifest = []
|
||||||
|
for i, (a, b, text) in enumerate(wins, 1):
|
||||||
|
f = d / f"win-{i:03d}.txt"
|
||||||
|
f.write_text(f"《{title}》 第{a}–{b}章 原文窗口(章题标记行格式:【第N章 | 章题】)\n{text}")
|
||||||
|
manifest.append({"win": i, "file": str(f), "from": a, "to": b, "chars": len(text)})
|
||||||
|
(d / "manifest.json").write_text(json.dumps(
|
||||||
|
{"work_id": work_id, "title": title, "windows": manifest}, ensure_ascii=False, indent=1))
|
||||||
|
click.echo(f"《{title}》切窗完成: {len(wins)} 窗 → {d}/win-*.txt(manifest.json 就绪)")
|
||||||
|
for m in manifest[:5]:
|
||||||
|
click.echo(f" win-{m['win']:03d} 第{m['from']}–{m['to']}章 {m['chars']:,}字")
|
||||||
|
if len(manifest) > 5:
|
||||||
|
click.echo(f" … 共 {len(manifest)} 窗")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except psycopg.Error as e:
|
||||||
|
click.echo(f"[db错误] {type(e).__name__}: {e}", err=True)
|
||||||
|
sys.exit(1)
|
||||||
@ -28,8 +28,8 @@ CN_UNITS = {"十": 10, "百": 100, "千": 1000}
|
|||||||
PAT_VOL_CH = re.compile( # 第X卷 第Y章 标题(含无空格变体)——恒启用
|
PAT_VOL_CH = re.compile( # 第X卷 第Y章 标题(含无空格变体)——恒启用
|
||||||
r'^\s*第\s*(?P<vol>[零〇一二两三四五六七八九十百千0-9]+)\s*卷\s*'
|
r'^\s*第\s*(?P<vol>[零〇一二两三四五六七八九十百千0-9]+)\s*卷\s*'
|
||||||
r'第\s*(?P<no>[零〇一二两三四五六七八九十百千0-9]+)\s*[章回]\s*(?P<title>\S.*)?$')
|
r'第\s*(?P<no>[零〇一二两三四五六七八九十百千0-9]+)\s*[章回]\s*(?P<title>\S.*)?$')
|
||||||
PAT_CH = re.compile( # 第X章 标题——恒启用
|
PAT_CH = re.compile( # 第X章 标题——恒启用;支持双章合一区间号「第2373-2374章」(机战无限实测)
|
||||||
r'^\s*第\s*(?P<no>[零〇一二两三四五六七八九十百千0-9]+)\s*[章回]\s*(?P<title>\S.*)?$')
|
r'^\s*第\s*(?P<no>[零〇一二两三四五六七八九十百千0-9]+(?:\s*[-—~-]\s*[0-9]+)?)\s*[章回]\s*(?P<title>\S.*)?$')
|
||||||
PAT_VOL_ONLY = re.compile( # 纯卷行(有卷无章)——不开章
|
PAT_VOL_ONLY = re.compile( # 纯卷行(有卷无章)——不开章
|
||||||
r'^\s*第\s*[零〇一二两三四五六七八九十百千0-9]+\s*卷\s*(?P<title>\S.*)?$')
|
r'^\s*第\s*[零〇一二两三四五六七八九十百千0-9]+\s*卷\s*(?P<title>\S.*)?$')
|
||||||
PAT_NUM = re.compile(r'^\s*(?P<no>\d{3,4})\s+(?P<title>\S.*)$') # 001 标题——自适应
|
PAT_NUM = re.compile(r'^\s*(?P<no>\d{3,4})\s+(?P<title>\S.*)$') # 001 标题——自适应
|
||||||
@ -38,8 +38,11 @@ PAT_CN_BARE = re.compile( # 一百零二 标题——自适应(限长防误
|
|||||||
|
|
||||||
|
|
||||||
def cn2int(s: str):
|
def cn2int(s: str):
|
||||||
"""中文数字→整数;混写/错写返回 None(调用方 fallback 前章+1)。"""
|
"""中文数字→整数;区间号「2373-2374」取首号;混写/错写返回 None(调用方 fallback 前章+1)。"""
|
||||||
s = s.strip()
|
s = s.strip()
|
||||||
|
m = re.match(r'^(\d+)\s*[-—~-]\s*\d+$', s)
|
||||||
|
if m:
|
||||||
|
return int(m.group(1))
|
||||||
if s.isdigit():
|
if s.isdigit():
|
||||||
return int(s)
|
return int(s)
|
||||||
total, section, num = 0, 0, 0
|
total, section, num = 0, 0, 0
|
||||||
@ -90,6 +93,8 @@ def parse_book(path: pathlib.Path):
|
|||||||
"""解析一本书 → (meta, chapters, stats)。修复规则见 SKILL.md。"""
|
"""解析一本书 → (meta, chapters, stats)。修复规则见 SKILL.md。"""
|
||||||
raw = path.read_text(encoding="utf-8", errors="replace")
|
raw = path.read_text(encoding="utf-8", errors="replace")
|
||||||
raw = html.unescape(raw) # 解 “ 等实体
|
raw = html.unescape(raw) # 解 “ 等实体
|
||||||
|
raw = re.sub(r'&[a-z]{2,6};', '', raw) # 残破实体(&bp; 等盗版损坏形态)直接剔除
|
||||||
|
raw = re.sub(r'\s*[((]第\s*\d+\s*/\s*\d+\s*页[))]', '', raw) # 分页尾巴「(第1/1页)」
|
||||||
lines = raw.split("\n")
|
lines = raw.split("\n")
|
||||||
|
|
||||||
# 头部 meta:# 《书名》 / # 作者: X / # 共 N 章
|
# 头部 meta:# 《书名》 / # 作者: X / # 共 N 章
|
||||||
|
|||||||
82
.claude/skills/import/scripts/quality_report.py
Normal file
82
.claude/skills/import/scripts/quality_report.py
Normal file
@ -0,0 +1,82 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""B1 基础面验收报告:字数/分章/章节名/段落切分/残留垃圾扫描(纯规则,审查面)。"""
|
||||||
|
import re
|
||||||
|
import statistics
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import click
|
||||||
|
import psycopg
|
||||||
|
|
||||||
|
DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example"
|
||||||
|
"?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3")
|
||||||
|
TENANT = 1
|
||||||
|
|
||||||
|
# 语义垃圾嫌疑模式(扫描计数用;真正删除走 clean skill 的 LLM 检测+代码执行)
|
||||||
|
JUNK_PATTERNS = {
|
||||||
|
"网址残留": re.compile(r'www\.|http|\.(?:com|net|cc|org|info)\b', re.I),
|
||||||
|
"一秒记住类": re.compile(r'一秒记住|天才.{0,3}记住|记住本站|首发域名|手机版阅读|无弹窗'),
|
||||||
|
"求票拉票": re.compile(r'求(?:月票|推荐票|订阅|收藏)|拉票|票票|投个|打赏|加更规则'),
|
||||||
|
"乱码符号": re.compile(r'[♂♀÷×]|&#\d+;|&[a-z]{2,6};'),
|
||||||
|
"作者话PS": re.compile(r'^PS[::]|^ps[::]|^附[::]|^作者的?话', re.M),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@click.command()
|
||||||
|
@click.option("--work-id", type=int, multiple=True, help="不给则全部")
|
||||||
|
@click.option("--sample-titles", default=5, show_default=True)
|
||||||
|
def main(work_id, sample_titles):
|
||||||
|
with psycopg.connect(DSN) as conn:
|
||||||
|
works = conn.execute(
|
||||||
|
f"""SELECT id, title FROM muse_content_work WHERE tenant_id=%s AND deleted=FALSE
|
||||||
|
{'AND id = ANY(%s)' if work_id else ''} ORDER BY id""",
|
||||||
|
(TENANT, list(work_id)) if work_id else (TENANT,)).fetchall()
|
||||||
|
for wid, title in works:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""SELECT c.order_no, c.title, b.content_text
|
||||||
|
FROM muse_content_chapter c JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE
|
||||||
|
WHERE c.tenant_id=%s AND c.work_id=%s AND c.deleted=FALSE ORDER BY c.order_no""",
|
||||||
|
(TENANT, wid)).fetchall()
|
||||||
|
if not rows:
|
||||||
|
continue
|
||||||
|
wc = [len(re.sub(r'\s', '', t)) for _, _, t in rows]
|
||||||
|
paras_per_ch, para_lens, fat_paras = [], [], 0
|
||||||
|
junk_hits = {k: 0 for k in JUNK_PATTERNS}
|
||||||
|
junk_samples = {}
|
||||||
|
for _, _, t in rows:
|
||||||
|
# 网文段落=非空行(一行一段是行业惯例;星环使命等源单换行分段,空行切分会误判粘连)
|
||||||
|
paras = [p for p in t.split("\n") if p.strip()]
|
||||||
|
paras_per_ch.append(len(paras))
|
||||||
|
for p in paras:
|
||||||
|
L = len(re.sub(r'\s', '', p))
|
||||||
|
para_lens.append(L)
|
||||||
|
if L > 2000:
|
||||||
|
fat_paras += 1
|
||||||
|
for k, pat in JUNK_PATTERNS.items():
|
||||||
|
ms = pat.findall(t)
|
||||||
|
if ms:
|
||||||
|
junk_hits[k] += len(ms)
|
||||||
|
if k not in junk_samples:
|
||||||
|
m = pat.search(t)
|
||||||
|
s = t[max(0, m.start() - 20):m.end() + 30].replace("\n", " ")
|
||||||
|
junk_samples[k] = s
|
||||||
|
print(f"═══ [{wid}] 《{title}》 ═══")
|
||||||
|
print(f" 章数 {len(rows)} | 总字数 {sum(wc):,} | 章字数 均{int(statistics.mean(wc))} 中位{int(statistics.median(wc))} 最短{min(wc)} 最长{max(wc)}")
|
||||||
|
shorts = sorted(zip(wc, (r[0] for r in rows), (r[1] for r in rows)))[:3]
|
||||||
|
print(f" 最短3章: " + " / ".join(f"#{o}《{t[:16]}》{w}字" for w, o, t in shorts))
|
||||||
|
print(f" 段落: 每章均{int(statistics.mean(paras_per_ch))}段 | 段均{int(statistics.mean(para_lens))}字 中位{int(statistics.median(para_lens))} | >2000字粘连段 {fat_paras}")
|
||||||
|
active = {k: v for k, v in junk_hits.items() if v}
|
||||||
|
print(f" 垃圾嫌疑: {active if active else '无命中'}")
|
||||||
|
for k, s in junk_samples.items():
|
||||||
|
print(f" [{k}] …{s}…")
|
||||||
|
heads = [f"#{r[0]}{r[1][:14]}" for r in rows[:sample_titles]]
|
||||||
|
tails = [f"#{r[0]}{r[1][:14]}" for r in rows[-3:]]
|
||||||
|
print(f" 章题样例: {' | '.join(heads)} … {' | '.join(tails)}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except psycopg.Error as e:
|
||||||
|
click.echo(f"[db错误] {type(e).__name__}: {e}", err=True)
|
||||||
|
sys.exit(1)
|
||||||
@ -22,6 +22,27 @@ PATTERN_TYPES = {"craft", "combat", "emotion", "scene_pattern", "trope"} # 拍
|
|||||||
NGRAM = 15 # 脱敏红线:≥15 连续字与原文重合=违规(parse-book skill)
|
NGRAM = 15 # 脱敏红线:≥15 连续字与原文重合=违规(parse-book skill)
|
||||||
|
|
||||||
|
|
||||||
|
def norm_scaffold(data: dict) -> dict:
|
||||||
|
"""键名归一:StructuredOutput 工具 schema 只允许 ASCII 键(API 硬约束 2026-07-13 实测),
|
||||||
|
库内 payload 统一中文键(供下游 prompt 消费)。两种键名都接受。"""
|
||||||
|
ents = []
|
||||||
|
for e in data.get("实体") or data.get("entities") or []:
|
||||||
|
ents.append({"型": e.get("型") or e.get("type"), "名称": e.get("名称") or e.get("name"),
|
||||||
|
"一句话摘要": e.get("一句话摘要") or e.get("brief")})
|
||||||
|
return {"细纲": data.get("细纲") or data.get("outline") or "", "实体": ents}
|
||||||
|
|
||||||
|
|
||||||
|
def norm_card(c: dict) -> dict:
|
||||||
|
"""范式卡键名归一(ASCII→中文)。"""
|
||||||
|
src = c.get("出处") or c.get("source") or {}
|
||||||
|
return {"型": c.get("型") or c.get("type"), "名称": c.get("名称") or c.get("name"),
|
||||||
|
"一句话摘要": c.get("一句话摘要") or c.get("brief"),
|
||||||
|
"字段": c.get("字段") or c.get("fields") or {},
|
||||||
|
"出处": {"书名": src.get("书名") or src.get("book"),
|
||||||
|
"回目": src.get("回目") or src.get("chapter"),
|
||||||
|
"定位": src.get("定位") or src.get("anchor")}}
|
||||||
|
|
||||||
|
|
||||||
def chapter_of(conn, work_id, order_no):
|
def chapter_of(conn, work_id, order_no):
|
||||||
"""取章 id 与正文。"""
|
"""取章 id 与正文。"""
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
@ -104,7 +125,7 @@ def init_tasks(work_id, from_, to):
|
|||||||
@click.option("--file", "file_", type=click.Path(exists=True), required=True)
|
@click.option("--file", "file_", type=click.Path(exists=True), required=True)
|
||||||
def scaffold(work_id, chapter_order, file_):
|
def scaffold(work_id, chapter_order, file_):
|
||||||
"""脚手架入库:{细纲, 实体:[{型,名称,一句话摘要,备注?}]};比例约束校验(3–5%,超标拒绝)。"""
|
"""脚手架入库:{细纲, 实体:[{型,名称,一句话摘要,备注?}]};比例约束校验(3–5%,超标拒绝)。"""
|
||||||
data = json.loads(pathlib.Path(file_).read_text())
|
data = norm_scaffold(json.loads(pathlib.Path(file_).read_text()))
|
||||||
with psycopg.connect(DSN) as conn:
|
with psycopg.connect(DSN) as conn:
|
||||||
ch_id, src = chapter_of(conn, work_id, chapter_order)
|
ch_id, src = chapter_of(conn, work_id, chapter_order)
|
||||||
outline = (data.get("细纲") or "").strip()
|
outline = (data.get("细纲") or "").strip()
|
||||||
@ -140,9 +161,12 @@ def scaffold(work_id, chapter_order, file_):
|
|||||||
def patterns(work_id, chapter_order, file_):
|
def patterns(work_id, chapter_order, file_):
|
||||||
"""范式卡入库:[{型∈五型, 名称, 一句话摘要, 字段{…}, 出处{书名,回目,定位}}] → draft(pending)。
|
"""范式卡入库:[{型∈五型, 名称, 一句话摘要, 字段{…}, 出处{书名,回目,定位}}] → draft(pending)。
|
||||||
机械硬阻断:归型合法、字段 key 合法(库内合同)、出处必填、15 连字脱敏检测。"""
|
机械硬阻断:归型合法、字段 key 合法(库内合同)、出处必填、15 连字脱敏检测。"""
|
||||||
cards = json.loads(pathlib.Path(file_).read_text())
|
raw = json.loads(pathlib.Path(file_).read_text())
|
||||||
if not isinstance(cards, list):
|
if isinstance(raw, dict): # 兼容 {cards:[…]}/{卡:[…]} 包装
|
||||||
|
raw = raw.get("cards") or raw.get("卡") or []
|
||||||
|
if not isinstance(raw, list):
|
||||||
raise click.ClickException("patterns 文件须为卡片数组")
|
raise click.ClickException("patterns 文件须为卡片数组")
|
||||||
|
cards = [norm_card(c) for c in raw]
|
||||||
with psycopg.connect(DSN) as conn:
|
with psycopg.connect(DSN) as conn:
|
||||||
ch_id, src = chapter_of(conn, work_id, chapter_order)
|
ch_id, src = chapter_of(conn, work_id, chapter_order)
|
||||||
contracts = {t: field_contract(conn, t) for t in PATTERN_TYPES}
|
contracts = {t: field_contract(conn, t) for t in PATTERN_TYPES}
|
||||||
|
|||||||
16
README.md
16
README.md
@ -182,7 +182,7 @@ flowchart LR
|
|||||||
|
|
||||||
| # | 场景锚 | 步骤 | 审查面 | 现状 |
|
| # | 场景锚 | 步骤 | 审查面 | 现状 |
|
||||||
|---|---|---|---|---|
|
|---|---|---|---|---|
|
||||||
| B1(原K3) | 参考书导入+静态分章 | import 工具(回目正则)→《封神演义》作品/章入库 | 作品/章行数+逐章抽查 | 12-14 回原文已备;全本未取 |
|
| B1(原K3) | 参考书导入+静态分章 | import 工具(回目正则)→ 8 本全本作品/章/块入库 | 作品/章行数对账+quality_report 基础面报告 | ✅ 基本收口(2026-07-13):8 本 ≈1.18 万章全入库;机战无限/超神机械师因源损坏格式(区间章号/分页尾巴/残破实体)用修复后解析器重导 |
|
||||||
| B2(原K4) | 拆书智能体 PG 版 | extractor 走 parse-book 2b:先逐章逆推细纲+抽实体作**脚手架**,再在其上按库内 schema 拆范式→知识行(draft)+嵌入 | 脚手架(细纲/实体)+知识行卡片打印+分型统计+出处链 | 文件版首轮流程已验(产物已清理,字段发现存 schema);PG 版从零起拆 |
|
| B2(原K4) | 拆书智能体 PG 版 | extractor 走 parse-book 2b:先逐章逆推细纲+抽实体作**脚手架**,再在其上按库内 schema 拆范式→知识行(draft)+嵌入 | 脚手架(细纲/实体)+知识行卡片打印+分型统计+出处链 | 文件版首轮流程已验(产物已清理,字段发现存 schema);PG 版从零起拆 |
|
||||||
| B3(原K5) | 检索验证 | 创作意图→相似度召回;授权与 aiContext 裁剪落查询层 | 检索结果+相关性人工评 | 未做 |
|
| B3(原K5) | 检索验证 | 创作意图→相似度召回;授权与 aiContext 裁剪落查询层 | 检索结果+相关性人工评 | 未做 |
|
||||||
| B4(原K6) | 联合优化环 | prompt × schema 字段 × 知识行质量,n=5 收敛 | 迭代前后对比+设计发现清单 | 未做 |
|
| B4(原K6) | 联合优化环 | prompt × schema 字段 × 知识行质量,n=5 收敛 | 迭代前后对比+设计发现清单 | 未做 |
|
||||||
@ -215,7 +215,19 @@ flowchart LR
|
|||||||
|
|
||||||
达标 prompt(→ 未来 agent_version.config 种子);schema 修订(→ 专题-06/后端-04/W1 种子);知识卡样式与上下文组装打法(→ 统一读取器 API 设计参考);`golden/` 样张与质量基线(→ 完整工程同场景对拍);阶段二的 API 缺口清单。
|
达标 prompt(→ 未来 agent_version.config 种子);schema 修订(→ 专题-06/后端-04/W1 种子);知识卡样式与上下文组装打法(→ 统一读取器 API 设计参考);`golden/` 样张与质量基线(→ 完整工程同场景对拍);阶段二的 API 缺口清单。
|
||||||
|
|
||||||
## 九、当前位置(2026-07-10 快照)
|
## 九、当前位置(2026-07-13 快照)
|
||||||
|
|
||||||
|
**执行态速览(compact 后从此恢复;细节看 git log 与库内状态表)**
|
||||||
|
|
||||||
|
- **A 阶段全收口**:muse-example 库+pgvector 0.8.5(容器重建恢复命令见 `db/连接信息.md`);24 表(主仓一致 20+example_* 5,含 92-清洗日志表;映射见 `db/表映射.md`);23 型 schema 入库(294 字段行,字段级 aiContext 在 visibility_policy.policy_snapshot)。
|
||||||
|
- **B1 基本收口**:8 本参考书全本入库(希泊尼战纪1496 / 星环使命1722 / 机动风暴673 / 机武风暴686 / 机破星河2250 / 深空之影594 / 超神机械师 / 机战无限2901 章;星环使命为创始人 2026-07-13 新增)。源损坏修复经历:双章区间号「第2373-2374章」、分页尾巴「(第1/1页)」、残破实体「&bp;」、纯中文数字章题+重贴+目录页(机动风暴)——全部进 import 解析器规则。基础面验收工具=`quality_report.py`(字数/分章/章题/段落/垃圾嫌疑扫描)。
|
||||||
|
- **书单拍板(2026-07-13 创始人)**:先试拆几本书前几章给创始人质检(试拆流水线=workflow 编排+逐章两 pass:scaffold(opus)→ingest(haiku)→patterns(opus)→ingest(haiku);脚本存 session workflows/,状态在 example_parse_task/scaffold/draft 表),质检过再放量。
|
||||||
|
- **清洗方案拍板(2026-07-13 创始人)**:语义垃圾走「LLM 探测输出待删段原文+代码精确匹配删除」=clean skill(窗口≈10万字/次调用,守卫规则+审计表 example_clean_log;机破星河最脏 879 求票+306 PS,win 文件已备 /tmp/muse-clean/7/)。
|
||||||
|
- **在飞任务(compact 时)**:超神机械师重导(修其 14 个百字以下章);试拆 workflow(机动风暴 1–3 章,ch1 脚手架已过校验:细纲 5.0%/实体 6)。
|
||||||
|
- **未决**:①试拆结果质检(含范式卡样张)②清洗范围拍板(仅拆书范围 ≈12 次 haiku vs 全语料 ≈350 次)③B2 放量确认闸(编排+成本呈报后创始人拍)。
|
||||||
|
- **教训入档**:StructuredOutput 工具 schema 属性名仅限 ASCII(中文键 API 400)——schema ASCII 键+入库脚本键名归一;Tailscale 长事务需 keepalive+批量写(逐行两万往返曾半死 16 分钟)。
|
||||||
|
|
||||||
|
## 九·旧(2026-07-10 快照,留档)
|
||||||
|
|
||||||
- 执行主线=§七闭环序,知识域(A/B)先行;创作域(C)后置,待 B5 过门后按 C 线走。
|
- 执行主线=§七闭环序,知识域(A/B)先行;创作域(C)后置,待 B5 过门后按 C 线走。
|
||||||
- **拍板台账(①–④已全拍)**:①B2 首轮公共面只拆 5 型范式,双层型二轮凭脚手架实据定;细纲字数比例已拍入 outline schema。②已拍(2026-07-10)——参考书=`../小说清单/` 7 本**全部导入**私有库;首轮拆书 4 本:超神机械师/机战无限/深空之影/机动风暴(每本 30–50 章;原文只入私有库,公共面只有脱敏范式)。③已拍——主仓表原样不改列+私货全进 `example_*`。④已拍——整序确认,A1 已起跑(B2 跑批前有成本确认闸)。
|
- **拍板台账(①–④已全拍)**:①B2 首轮公共面只拆 5 型范式,双层型二轮凭脚手架实据定;细纲字数比例已拍入 outline schema。②已拍(2026-07-10)——参考书=`../小说清单/` 7 本**全部导入**私有库;首轮拆书 4 本:超神机械师/机战无限/深空之影/机动风暴(每本 30–50 章;原文只入私有库,公共面只有脱敏范式)。③已拍——主仓表原样不改列+私货全进 `example_*`。④已拍——整序确认,A1 已起跑(B2 跑批前有成本确认闸)。
|
||||||
|
|||||||
20
db/ddl/92-example清洗日志.sql
Normal file
20
db/ddl/92-example清洗日志.sql
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
-- 实验私货:LLM 辅助清洗的审计日志(清洗方案见 .claude/skills/clean/SKILL.md)
|
||||||
|
-- 每删一段留一行:删了什么、为什么、哪个模型建议、哪个批次——可回放、可回滚(源 txt 是不动的 SoT)。
|
||||||
|
CREATE TABLE example_clean_log (
|
||||||
|
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||||
|
work_id BIGINT NOT NULL, -- → muse_content_work.id
|
||||||
|
chapter_id BIGINT NOT NULL, -- → muse_content_chapter.id
|
||||||
|
batch VARCHAR(64) NOT NULL, -- 清洗批次号(一次编排一个批次)
|
||||||
|
removed_text TEXT NOT NULL, -- 被删除的原文片段(逐字)
|
||||||
|
occurrences INT NOT NULL DEFAULT 1, -- 该片段在本章命中并删除的次数
|
||||||
|
reason VARCHAR(200), -- LLM 给出的删除理由
|
||||||
|
model VARCHAR(50), -- 建议来源模型
|
||||||
|
creator VARCHAR(64) NOT NULL DEFAULT '',
|
||||||
|
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updater VARCHAR(64) NOT NULL DEFAULT '',
|
||||||
|
update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
deleted BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
tenant_id BIGINT NOT NULL DEFAULT 0
|
||||||
|
);
|
||||||
|
CREATE INDEX idx_example_clean_log_work ON example_clean_log(tenant_id, work_id, batch);
|
||||||
|
CREATE TRIGGER trg_example_clean_log_updated_at BEFORE UPDATE ON example_clean_log FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||||
Loading…
x
Reference in New Issue
Block a user