diff --git a/.claude/skills/clean/SKILL.md b/.claude/skills/clean/SKILL.md new file mode 100644 index 0000000..cd272a4 --- /dev/null +++ b/.claude/skills/clean/SKILL.md @@ -0,0 +1,39 @@ +--- +name: clean +description: LLM 辅助正文清洗——LLM 按窗口(约10万字)只输出待删垃圾段原文与理由,代码做精确匹配删除并全程留审计(example_clean_log)。LLM 不改写正文,删不删由守卫规则最终裁决。 +--- + +# clean —— LLM 检测 + 代码执行的正文清洗 + +分工(创始人方案 2026-07-13):**规则层**已在 import 解决结构性垃圾(水印正则/重贴章题/目录页/分页尾巴/残破实体);本 skill 处理**语义垃圾**——变体书站广告、作者拉票/PS 段、乱码水印等正则打不全的散落噪声。LLM 只当探测器(输出待删片段逐字原文),删除由脚本执行:零改写、可审计、可回放。 + +## 流程 + +```bash +# 1) 备窗口:按章对齐切窗(默认 ~10万字/窗,写 /tmp/muse-clean//win-*.txt + manifest) +.venv/bin/python .claude/skills/clean/scripts/clean_prep.py --work-id 5 --window 100000 + +# 2) 探测:每窗一次 LLM 调用(haiku,编排层派发;prompt 模板见下)→ 每窗一份 deletions JSON + +# 3) 执行:精确匹配删除 + 审计入库 + 字数刷新(守卫规则见下) +.venv/bin/python .claude/skills/clean/scripts/clean_apply.py --work-id 5 --batch <批次号> --file /tmp/muse-clean/5/deletions-all.json + +# 4) 审查:删除对账(每章删了几段几字)+ 审计行抽查 +.venv/bin/python .claude/skills/db/scripts/db.py query "SELECT reason, count(*), sum(length(removed_text)) FROM example_clean_log WHERE work_id=5 AND batch='<批次号>' GROUP BY reason" +``` + +## 探测 prompt 模板(编排层内联给 haiku) + +> 你是网文正文清洗探测器。下面是《书名》第 A–B 章的原文(含章题行)。找出**所有非正文垃圾**:书站广告及变体(如「一秒记住♂粒÷小÷说→网」)、网址残留、求票拉票/打赏鸣谢、章尾作者话(ps:…)、上架感言类整段、乱码水印。输出 deletions 数组,每项 `{chapter_order, exact, reason}`:`exact` 必须与原文**逐字一致**(含标点空白,≤300字,一段一项);只报垃圾,拿不准的不报;**情节正文一个字都不许报**。章题行不报(即使含求票字样——章题另行处理)。 + +## 守卫规则(脚本硬闸,LLM 建议 ≠ 必删) + +- `exact` 长度 4–500 字;在指定章内**逐字命中**才删(命中多次全删并记 occurrences); +- 不得命中/包含章题行;单章累计删除 ≤ 章长 20%(超限整章建议弃用并报告); +- 每次删除一行审计(原文/理由/模型/批次),源 txt 永远不动(可整书重导回滚); +- 删完刷新 block.word_count 与 work.word_count。 + +## 量级参考(2026-07-13 估) + +- 拆书范围(5 书×前50章 ≈ 90万字)≈ 9–12 窗 ≈ 12 次 haiku 调用; +- 全语料(8 书 ≈ 3,300万字)≈ 340 窗 ≈ 350 次 haiku 调用(输入 ~35M token 量级)。 diff --git a/.claude/skills/clean/scripts/clean_apply.py b/.claude/skills/clean/scripts/clean_apply.py new file mode 100644 index 0000000..94ec064 --- /dev/null +++ b/.claude/skills/clean/scripts/clean_apply.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""clean skill:精确匹配删除执行器(LLM 建议 ≠ 必删,守卫规则最终裁决)+ 审计入库。 + +输入 deletions JSON:[{chapter_order|chapter, exact, reason}](键名 ASCII,兼容中文键)。 +""" +import json +import pathlib +import re +import sys + +import click +import psycopg + +DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example" + "?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3") +TENANT, ACTOR = 1, "1" +MIN_LEN, MAX_LEN = 4, 500 +MAX_CH_RATIO = 0.20 # 单章累计删除上限(占章长比) + + +@click.command() +@click.option("--work-id", type=int, required=True) +@click.option("--batch", required=True, help="清洗批次号(审计用)") +@click.option("--file", "file_", type=click.Path(exists=True), required=True) +@click.option("--model", default="haiku", show_default=True, help="建议来源模型(审计标注)") +@click.option("--dry-run", is_flag=True, help="只对账不落库") +def main(work_id, batch, file_, model, dry_run): + raw = json.loads(pathlib.Path(file_).read_text()) + if isinstance(raw, dict): + raw = raw.get("deletions") or raw.get("删除") or [] + # 按章分组 + by_ch = {} + for d in raw: + no = d.get("chapter_order") or d.get("chapter") or d.get("章序") + exact = d.get("exact") or d.get("原文") + if no is None or not exact: + continue + by_ch.setdefault(int(no), []).append((exact, d.get("reason") or d.get("理由") or "")) + + stats = {"删除段": 0, "删除字数": 0, "拒绝": []} + with psycopg.connect(DSN) as conn: + for no, dels in sorted(by_ch.items()): + row = conn.execute( + """SELECT c.id, c.title, b.id, b.content_text + FROM muse_content_chapter c JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE + WHERE c.tenant_id=%s AND c.work_id=%s AND c.order_no=%s AND c.deleted=FALSE""", + (TENANT, work_id, no)).fetchone() + if not row: + stats["拒绝"].append(f"#{no}: 章不存在") + continue + ch_id, ch_title, blk_id, text = row + new_text = text + ch_removed = 0 + applied = [] + for exact, reason in dels: + # 守卫1:长度界 + if not (MIN_LEN <= len(exact) <= MAX_LEN): + stats["拒绝"].append(f"#{no}: 长度越界({len(exact)}) {exact[:24]!r}") + continue + # 守卫2:不得含/命中章题 + if ch_title and ch_title in exact: + stats["拒绝"].append(f"#{no}: 含章题 {exact[:24]!r}") + continue + n = new_text.count(exact) + if n == 0: + stats["拒绝"].append(f"#{no}: 未命中 {exact[:24]!r}") + continue + new_text = new_text.replace(exact, "") + ch_removed += len(exact) * n + applied.append((exact, n, reason)) + # 守卫3:单章删除比例硬顶 + if ch_removed > len(text) * MAX_CH_RATIO: + stats["拒绝"].append(f"#{no}: 累计删除 {ch_removed} 字超章长20%上限,整章放弃") + continue + if not applied: + continue + new_text = re.sub(r'\n{3,}', '\n\n', new_text) # 删除后合并多余空行 + stats["删除段"] += len(applied) + stats["删除字数"] += ch_removed + if dry_run: + continue + wc = len(re.sub(r'\s', '', new_text)) + conn.execute("UPDATE muse_content_block SET content_text=%s, word_count=%s, updater=%s WHERE id=%s", + (new_text, wc, ACTOR, blk_id)) + for exact, n, reason in applied: + conn.execute( + """INSERT INTO example_clean_log (work_id, chapter_id, batch, removed_text, occurrences, + reason, model, creator, updater, tenant_id) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""", + (work_id, ch_id, batch, exact, n, reason[:200], model, ACTOR, ACTOR, TENANT)) + if not dry_run: + conn.execute( + """UPDATE muse_content_work w SET word_count=( + SELECT COALESCE(sum(b.word_count),0) FROM muse_content_block b + WHERE b.tenant_id=%s AND b.work_id=w.id AND b.deleted=FALSE), updater=%s + WHERE w.id=%s""", (TENANT, ACTOR, work_id)) + conn.commit() + mode = "(dry-run 未落库)" if dry_run else "" + click.echo(f"清洗执行{mode}: 删除 {stats['删除段']} 段 / {stats['删除字数']:,} 字;拒绝 {len(stats['拒绝'])} 条") + for r in stats["拒绝"][:12]: + click.echo(f" [拒] {r}") + + +if __name__ == "__main__": + try: + main() + except psycopg.Error as e: + click.echo(f"[db错误] {type(e).__name__}: {e}", err=True) + sys.exit(1) diff --git a/.claude/skills/clean/scripts/clean_prep.py b/.claude/skills/clean/scripts/clean_prep.py new file mode 100644 index 0000000..8a999c0 --- /dev/null +++ b/.claude/skills/clean/scripts/clean_prep.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""clean skill:按章对齐切窗(LLM 探测输入面)。窗文件含章题标记行,便于 LLM 报 chapter_order。""" +import json +import pathlib +import re +import sys + +import click +import psycopg + +DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example" + "?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3") +TENANT = 1 +OUT = pathlib.Path("/tmp/muse-clean") + + +@click.command() +@click.option("--work-id", type=int, required=True) +@click.option("--window", type=int, default=100000, show_default=True, help="每窗目标字符数(章对齐)") +@click.option("--from", "from_", type=int, default=1, help="起始章 order_no") +@click.option("--to", type=int, default=0, help="结束章 order_no(0=到末章)") +def main(work_id, window, from_, to): + with psycopg.connect(DSN) as conn: + title = conn.execute("SELECT title FROM muse_content_work WHERE id=%s", (work_id,)).fetchone()[0] + sql = """SELECT c.order_no, c.title, b.content_text + FROM muse_content_chapter c JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE + WHERE c.tenant_id=%s AND c.work_id=%s AND c.deleted=FALSE AND c.order_no>=%s""" + args = [TENANT, work_id, from_] + if to: + sql += " AND c.order_no<=%s" + args.append(to) + rows = conn.execute(sql + " ORDER BY c.order_no", args).fetchall() + d = OUT / str(work_id) + d.mkdir(parents=True, exist_ok=True) + wins, cur, cur_len, cur_span = [], [], 0, [] + for no, ct, text in rows: + # 章题标记行:供 LLM 定位 chapter_order;clean_apply 不会删除标记行本身 + cur.append(f"\n【第{no}章 | {ct}】\n{text}") + cur_len += len(text) + cur_span.append(no) + if cur_len >= window: + wins.append((cur_span[0], cur_span[-1], "".join(cur))) + cur, cur_len, cur_span = [], 0, [] + if cur: + wins.append((cur_span[0], cur_span[-1], "".join(cur))) + manifest = [] + for i, (a, b, text) in enumerate(wins, 1): + f = d / f"win-{i:03d}.txt" + f.write_text(f"《{title}》 第{a}–{b}章 原文窗口(章题标记行格式:【第N章 | 章题】)\n{text}") + manifest.append({"win": i, "file": str(f), "from": a, "to": b, "chars": len(text)}) + (d / "manifest.json").write_text(json.dumps( + {"work_id": work_id, "title": title, "windows": manifest}, ensure_ascii=False, indent=1)) + click.echo(f"《{title}》切窗完成: {len(wins)} 窗 → {d}/win-*.txt(manifest.json 就绪)") + for m in manifest[:5]: + click.echo(f" win-{m['win']:03d} 第{m['from']}–{m['to']}章 {m['chars']:,}字") + if len(manifest) > 5: + click.echo(f" … 共 {len(manifest)} 窗") + + +if __name__ == "__main__": + try: + main() + except psycopg.Error as e: + click.echo(f"[db错误] {type(e).__name__}: {e}", err=True) + sys.exit(1) diff --git a/.claude/skills/import/scripts/import_novel.py b/.claude/skills/import/scripts/import_novel.py index 4ada012..3efaaa8 100644 --- a/.claude/skills/import/scripts/import_novel.py +++ b/.claude/skills/import/scripts/import_novel.py @@ -28,8 +28,8 @@ CN_UNITS = {"十": 10, "百": 100, "千": 1000} PAT_VOL_CH = re.compile( # 第X卷 第Y章 标题(含无空格变体)——恒启用 r'^\s*第\s*(?P[零〇一二两三四五六七八九十百千0-9]+)\s*卷\s*' r'第\s*(?P[零〇一二两三四五六七八九十百千0-9]+)\s*[章回]\s*(?P\S.*)?$') -PAT_CH = re.compile( # 第X章 标题——恒启用 - r'^\s*第\s*(?P<no>[零〇一二两三四五六七八九十百千0-9]+)\s*[章回]\s*(?P<title>\S.*)?$') +PAT_CH = re.compile( # 第X章 标题——恒启用;支持双章合一区间号「第2373-2374章」(机战无限实测) + r'^\s*第\s*(?P<no>[零〇一二两三四五六七八九十百千0-9]+(?:\s*[-—~-]\s*[0-9]+)?)\s*[章回]\s*(?P<title>\S.*)?$') PAT_VOL_ONLY = re.compile( # 纯卷行(有卷无章)——不开章 r'^\s*第\s*[零〇一二两三四五六七八九十百千0-9]+\s*卷\s*(?P<title>\S.*)?$') PAT_NUM = re.compile(r'^\s*(?P<no>\d{3,4})\s+(?P<title>\S.*)$') # 001 标题——自适应 @@ -38,8 +38,11 @@ PAT_CN_BARE = re.compile( # 一百零二 标题——自适应(限长防误 def cn2int(s: str): - """中文数字→整数;混写/错写返回 None(调用方 fallback 前章+1)。""" + """中文数字→整数;区间号「2373-2374」取首号;混写/错写返回 None(调用方 fallback 前章+1)。""" s = s.strip() + m = re.match(r'^(\d+)\s*[-—~-]\s*\d+$', s) + if m: + return int(m.group(1)) if s.isdigit(): return int(s) total, section, num = 0, 0, 0 @@ -90,6 +93,8 @@ def parse_book(path: pathlib.Path): """解析一本书 → (meta, chapters, stats)。修复规则见 SKILL.md。""" raw = path.read_text(encoding="utf-8", errors="replace") raw = html.unescape(raw) # 解 “ 等实体 + raw = re.sub(r'&[a-z]{2,6};', '', raw) # 残破实体(&bp; 等盗版损坏形态)直接剔除 + raw = re.sub(r'\s*[((]第\s*\d+\s*/\s*\d+\s*页[))]', '', raw) # 分页尾巴「(第1/1页)」 lines = raw.split("\n") # 头部 meta:# 《书名》 / # 作者: X / # 共 N 章 diff --git a/.claude/skills/import/scripts/quality_report.py b/.claude/skills/import/scripts/quality_report.py new file mode 100644 index 0000000..9e355a1 --- /dev/null +++ b/.claude/skills/import/scripts/quality_report.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""B1 基础面验收报告:字数/分章/章节名/段落切分/残留垃圾扫描(纯规则,审查面)。""" +import re +import statistics +import sys + +import click +import psycopg + +DSN = ("postgresql://root:f6710e2d0294eb1c10e26a805a64bc54@100.64.0.8:5433/muse-example" + "?keepalives=1&keepalives_idle=15&keepalives_interval=5&keepalives_count=3") +TENANT = 1 + +# 语义垃圾嫌疑模式(扫描计数用;真正删除走 clean skill 的 LLM 检测+代码执行) +JUNK_PATTERNS = { + "网址残留": re.compile(r'www\.|http|\.(?:com|net|cc|org|info)\b', re.I), + "一秒记住类": re.compile(r'一秒记住|天才.{0,3}记住|记住本站|首发域名|手机版阅读|无弹窗'), + "求票拉票": re.compile(r'求(?:月票|推荐票|订阅|收藏)|拉票|票票|投个|打赏|加更规则'), + "乱码符号": re.compile(r'[♂♀÷×]|&#\d+;|&[a-z]{2,6};'), + "作者话PS": re.compile(r'^PS[::]|^ps[::]|^附[::]|^作者的?话', re.M), +} + + +@click.command() +@click.option("--work-id", type=int, multiple=True, help="不给则全部") +@click.option("--sample-titles", default=5, show_default=True) +def main(work_id, sample_titles): + with psycopg.connect(DSN) as conn: + works = conn.execute( + f"""SELECT id, title FROM muse_content_work WHERE tenant_id=%s AND deleted=FALSE + {'AND id = ANY(%s)' if work_id else ''} ORDER BY id""", + (TENANT, list(work_id)) if work_id else (TENANT,)).fetchall() + for wid, title in works: + rows = conn.execute( + """SELECT c.order_no, c.title, b.content_text + FROM muse_content_chapter c JOIN muse_content_block b ON b.chapter_id=c.id AND b.deleted=FALSE + WHERE c.tenant_id=%s AND c.work_id=%s AND c.deleted=FALSE ORDER BY c.order_no""", + (TENANT, wid)).fetchall() + if not rows: + continue + wc = [len(re.sub(r'\s', '', t)) for _, _, t in rows] + paras_per_ch, para_lens, fat_paras = [], [], 0 + junk_hits = {k: 0 for k in JUNK_PATTERNS} + junk_samples = {} + for _, _, t in rows: + # 网文段落=非空行(一行一段是行业惯例;星环使命等源单换行分段,空行切分会误判粘连) + paras = [p for p in t.split("\n") if p.strip()] + paras_per_ch.append(len(paras)) + for p in paras: + L = len(re.sub(r'\s', '', p)) + para_lens.append(L) + if L > 2000: + fat_paras += 1 + for k, pat in JUNK_PATTERNS.items(): + ms = pat.findall(t) + if ms: + junk_hits[k] += len(ms) + if k not in junk_samples: + m = pat.search(t) + s = t[max(0, m.start() - 20):m.end() + 30].replace("\n", " ") + junk_samples[k] = s + print(f"═══ [{wid}] 《{title}》 ═══") + print(f" 章数 {len(rows)} | 总字数 {sum(wc):,} | 章字数 均{int(statistics.mean(wc))} 中位{int(statistics.median(wc))} 最短{min(wc)} 最长{max(wc)}") + shorts = sorted(zip(wc, (r[0] for r in rows), (r[1] for r in rows)))[:3] + print(f" 最短3章: " + " / ".join(f"#{o}《{t[:16]}》{w}字" for w, o, t in shorts)) + print(f" 段落: 每章均{int(statistics.mean(paras_per_ch))}段 | 段均{int(statistics.mean(para_lens))}字 中位{int(statistics.median(para_lens))} | >2000字粘连段 {fat_paras}") + active = {k: v for k, v in junk_hits.items() if v} + print(f" 垃圾嫌疑: {active if active else '无命中'}") + for k, s in junk_samples.items(): + print(f" [{k}] …{s}…") + heads = [f"#{r[0]}{r[1][:14]}" for r in rows[:sample_titles]] + tails = [f"#{r[0]}{r[1][:14]}" for r in rows[-3:]] + print(f" 章题样例: {' | '.join(heads)} … {' | '.join(tails)}") + print() + + +if __name__ == "__main__": + try: + main() + except psycopg.Error as e: + click.echo(f"[db错误] {type(e).__name__}: {e}", err=True) + sys.exit(1) diff --git a/.claude/skills/parse-book/scripts/parse_ingest.py b/.claude/skills/parse-book/scripts/parse_ingest.py index 57a09f8..cf348ba 100644 --- a/.claude/skills/parse-book/scripts/parse_ingest.py +++ b/.claude/skills/parse-book/scripts/parse_ingest.py @@ -22,6 +22,27 @@ PATTERN_TYPES = {"craft", "combat", "emotion", "scene_pattern", "trope"} # 拍 NGRAM = 15 # 脱敏红线:≥15 连续字与原文重合=违规(parse-book skill) +def norm_scaffold(data: dict) -> dict: + """键名归一:StructuredOutput 工具 schema 只允许 ASCII 键(API 硬约束 2026-07-13 实测), + 库内 payload 统一中文键(供下游 prompt 消费)。两种键名都接受。""" + ents = [] + for e in data.get("实体") or data.get("entities") or []: + ents.append({"型": e.get("型") or e.get("type"), "名称": e.get("名称") or e.get("name"), + "一句话摘要": e.get("一句话摘要") or e.get("brief")}) + return {"细纲": data.get("细纲") or data.get("outline") or "", "实体": ents} + + +def norm_card(c: dict) -> dict: + """范式卡键名归一(ASCII→中文)。""" + src = c.get("出处") or c.get("source") or {} + return {"型": c.get("型") or c.get("type"), "名称": c.get("名称") or c.get("name"), + "一句话摘要": c.get("一句话摘要") or c.get("brief"), + "字段": c.get("字段") or c.get("fields") or {}, + "出处": {"书名": src.get("书名") or src.get("book"), + "回目": src.get("回目") or src.get("chapter"), + "定位": src.get("定位") or src.get("anchor")}} + + def chapter_of(conn, work_id, order_no): """取章 id 与正文。""" row = conn.execute( @@ -104,7 +125,7 @@ def init_tasks(work_id, from_, to): @click.option("--file", "file_", type=click.Path(exists=True), required=True) def scaffold(work_id, chapter_order, file_): """脚手架入库:{细纲, 实体:[{型,名称,一句话摘要,备注?}]};比例约束校验(3–5%,超标拒绝)。""" - data = json.loads(pathlib.Path(file_).read_text()) + data = norm_scaffold(json.loads(pathlib.Path(file_).read_text())) with psycopg.connect(DSN) as conn: ch_id, src = chapter_of(conn, work_id, chapter_order) outline = (data.get("细纲") or "").strip() @@ -140,9 +161,12 @@ def scaffold(work_id, chapter_order, file_): def patterns(work_id, chapter_order, file_): """范式卡入库:[{型∈五型, 名称, 一句话摘要, 字段{…}, 出处{书名,回目,定位}}] → draft(pending)。 机械硬阻断:归型合法、字段 key 合法(库内合同)、出处必填、15 连字脱敏检测。""" - cards = json.loads(pathlib.Path(file_).read_text()) - if not isinstance(cards, list): + raw = json.loads(pathlib.Path(file_).read_text()) + if isinstance(raw, dict): # 兼容 {cards:[…]}/{卡:[…]} 包装 + raw = raw.get("cards") or raw.get("卡") or [] + if not isinstance(raw, list): raise click.ClickException("patterns 文件须为卡片数组") + cards = [norm_card(c) for c in raw] with psycopg.connect(DSN) as conn: ch_id, src = chapter_of(conn, work_id, chapter_order) contracts = {t: field_contract(conn, t) for t in PATTERN_TYPES} diff --git a/README.md b/README.md index 11bd2e1..cd1d34f 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ flowchart LR | # | 场景锚 | 步骤 | 审查面 | 现状 | |---|---|---|---|---| -| B1(原K3) | 参考书导入+静态分章 | import 工具(回目正则)→《封神演义》作品/章入库 | 作品/章行数+逐章抽查 | 12-14 回原文已备;全本未取 | +| B1(原K3) | 参考书导入+静态分章 | import 工具(回目正则)→ 8 本全本作品/章/块入库 | 作品/章行数对账+quality_report 基础面报告 | ✅ 基本收口(2026-07-13):8 本 ≈1.18 万章全入库;机战无限/超神机械师因源损坏格式(区间章号/分页尾巴/残破实体)用修复后解析器重导 | | B2(原K4) | 拆书智能体 PG 版 | extractor 走 parse-book 2b:先逐章逆推细纲+抽实体作**脚手架**,再在其上按库内 schema 拆范式→知识行(draft)+嵌入 | 脚手架(细纲/实体)+知识行卡片打印+分型统计+出处链 | 文件版首轮流程已验(产物已清理,字段发现存 schema);PG 版从零起拆 | | B3(原K5) | 检索验证 | 创作意图→相似度召回;授权与 aiContext 裁剪落查询层 | 检索结果+相关性人工评 | 未做 | | B4(原K6) | 联合优化环 | prompt × schema 字段 × 知识行质量,n=5 收敛 | 迭代前后对比+设计发现清单 | 未做 | @@ -215,7 +215,19 @@ flowchart LR 达标 prompt(→ 未来 agent_version.config 种子);schema 修订(→ 专题-06/后端-04/W1 种子);知识卡样式与上下文组装打法(→ 统一读取器 API 设计参考);`golden/` 样张与质量基线(→ 完整工程同场景对拍);阶段二的 API 缺口清单。 -## 九、当前位置(2026-07-10 快照) +## 九、当前位置(2026-07-13 快照) + +**执行态速览(compact 后从此恢复;细节看 git log 与库内状态表)** + +- **A 阶段全收口**:muse-example 库+pgvector 0.8.5(容器重建恢复命令见 `db/连接信息.md`);24 表(主仓一致 20+example_* 5,含 92-清洗日志表;映射见 `db/表映射.md`);23 型 schema 入库(294 字段行,字段级 aiContext 在 visibility_policy.policy_snapshot)。 +- **B1 基本收口**:8 本参考书全本入库(希泊尼战纪1496 / 星环使命1722 / 机动风暴673 / 机武风暴686 / 机破星河2250 / 深空之影594 / 超神机械师 / 机战无限2901 章;星环使命为创始人 2026-07-13 新增)。源损坏修复经历:双章区间号「第2373-2374章」、分页尾巴「(第1/1页)」、残破实体「&bp;」、纯中文数字章题+重贴+目录页(机动风暴)——全部进 import 解析器规则。基础面验收工具=`quality_report.py`(字数/分章/章题/段落/垃圾嫌疑扫描)。 +- **书单拍板(2026-07-13 创始人)**:先试拆几本书前几章给创始人质检(试拆流水线=workflow 编排+逐章两 pass:scaffold(opus)→ingest(haiku)→patterns(opus)→ingest(haiku);脚本存 session workflows/,状态在 example_parse_task/scaffold/draft 表),质检过再放量。 +- **清洗方案拍板(2026-07-13 创始人)**:语义垃圾走「LLM 探测输出待删段原文+代码精确匹配删除」=clean skill(窗口≈10万字/次调用,守卫规则+审计表 example_clean_log;机破星河最脏 879 求票+306 PS,win 文件已备 /tmp/muse-clean/7/)。 +- **在飞任务(compact 时)**:超神机械师重导(修其 14 个百字以下章);试拆 workflow(机动风暴 1–3 章,ch1 脚手架已过校验:细纲 5.0%/实体 6)。 +- **未决**:①试拆结果质检(含范式卡样张)②清洗范围拍板(仅拆书范围 ≈12 次 haiku vs 全语料 ≈350 次)③B2 放量确认闸(编排+成本呈报后创始人拍)。 +- **教训入档**:StructuredOutput 工具 schema 属性名仅限 ASCII(中文键 API 400)——schema ASCII 键+入库脚本键名归一;Tailscale 长事务需 keepalive+批量写(逐行两万往返曾半死 16 分钟)。 + +## 九·旧(2026-07-10 快照,留档) - 执行主线=§七闭环序,知识域(A/B)先行;创作域(C)后置,待 B5 过门后按 C 线走。 - **拍板台账(①–④已全拍)**:①B2 首轮公共面只拆 5 型范式,双层型二轮凭脚手架实据定;细纲字数比例已拍入 outline schema。②已拍(2026-07-10)——参考书=`../小说清单/` 7 本**全部导入**私有库;首轮拆书 4 本:超神机械师/机战无限/深空之影/机动风暴(每本 30–50 章;原文只入私有库,公共面只有脱敏范式)。③已拍——主仓表原样不改列+私货全进 `example_*`。④已拍——整序确认,A1 已起跑(B2 跑批前有成本确认闸)。 diff --git a/db/ddl/92-example清洗日志.sql b/db/ddl/92-example清洗日志.sql new file mode 100644 index 0000000..7ac7177 --- /dev/null +++ b/db/ddl/92-example清洗日志.sql @@ -0,0 +1,20 @@ +-- 实验私货:LLM 辅助清洗的审计日志(清洗方案见 .claude/skills/clean/SKILL.md) +-- 每删一段留一行:删了什么、为什么、哪个模型建议、哪个批次——可回放、可回滚(源 txt 是不动的 SoT)。 +CREATE TABLE example_clean_log ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + work_id BIGINT NOT NULL, -- → muse_content_work.id + chapter_id BIGINT NOT NULL, -- → muse_content_chapter.id + batch VARCHAR(64) NOT NULL, -- 清洗批次号(一次编排一个批次) + removed_text TEXT NOT NULL, -- 被删除的原文片段(逐字) + occurrences INT NOT NULL DEFAULT 1, -- 该片段在本章命中并删除的次数 + reason VARCHAR(200), -- LLM 给出的删除理由 + model VARCHAR(50), -- 建议来源模型 + creator VARCHAR(64) NOT NULL DEFAULT '', + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updater VARCHAR(64) NOT NULL DEFAULT '', + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted BOOLEAN NOT NULL DEFAULT FALSE, + tenant_id BIGINT NOT NULL DEFAULT 0 +); +CREATE INDEX idx_example_clean_log_work ON example_clean_log(tenant_id, work_id, batch); +CREATE TRIGGER trg_example_clean_log_updated_at BEFORE UPDATE ON example_clean_log FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();