feat(资料研究): 场景切片派生缓存与向量代次索引——参考书原文确定性切窗入检索池
- V0061：切片台账（窗计划哈希锁切窗方案、禁原地改写）、切片向量索引与代次台账，代次守卫仿 V0059 - 场景切片.py：段落聚合切窗纯函数（目标1800字符、末窗并邻）、确定性切片身份、全量重建命令与授权读取 - 只纳入 reference_reading 用途来源的导入者视角；用途撤回即退出候选，切片行保留 - 底座种子表登记切片代次单例；单元4例+集成3例全过
This commit is contained in:
zizi
parent
97f2e4cb95
commit
fea8f44327
@ -88,7 +88,7 @@ S02检查点是执行证据而非第二分析读取路径。运行中的完成
|
||||
|
||||
## 场景片段切片与索引
|
||||
|
||||
参考书原文可确定性切片入检索池，为参考改写提供场景原文语料。切片是纯函数：按段落边界聚合至目标窗长（1200～2400 字符，不足并入邻窗），切片参数与窗口清单记入窗计划哈希；同一来源版本只认一个窗计划，重算不一致即拒绝为陈旧。切片只追加不改删，原文以 `muse_source_version` 为权威，切片表只存定位与检索文本。
|
||||
参考书原文可确定性切片入检索池，为参考改写提供场景原文语料。切片是纯函数：按段落边界聚合至目标窗长（1200～2400 字符，不足并入邻窗），切片参数与窗口清单记入窗计划哈希；同一来源版本只认一个窗计划，重算不一致即拒绝为陈旧。切片与索引是可全量重建的派生缓存（原文以 `muse_source_version` 为权威），禁原地改写，重建走整批替换；代次守卫随事务变更递增。
|
||||
|
||||
片段向量索引与方法索引同构：哈希 bigram embedder（零模型成本、确定性）、代次台账 active 唯一、检索时对不上即拒绝为陈旧。索引构建是显式命令（CLI/HTTP），不耦合导入入口；构建入口按当前具备 `reference_reading` 用途的来源版本全量对账，来源版本更新或用途撤回后重算。切片身份与授权解耦：授权变化只影响检索候选与材料读取，不删历史切片。
|
||||
|
||||
|
||||
221
src/muse/资料研究/场景切片.py
Normal file
221
src/muse/资料研究/场景切片.py
Normal file
@ -0,0 +1,221 @@
|
||||
"""场景切片：参考书原文的确定性切窗与向量索引。
|
||||
|
||||
原文以 muse_source_version 为权威；切片与索引是可全量重建的派生缓存。
|
||||
切窗是纯函数（同文本同参数同窗计划），嵌入用哈希 bigram，零模型成本。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from muse.基础设施.模型.嵌入 import 特征哈希嵌入器
|
||||
from muse.正式变更.接口 import 固定哈希
|
||||
from muse.资料研究.存储 import 资料存储
|
||||
from muse.资料研究.模型 import 资料错误
|
||||
|
||||
目标窗长 = 1800
|
||||
最短窗长 = 1200
|
||||
|
||||
|
||||
def 段落切窗(文本: str, *, 窗长: int = 目标窗长) -> list[dict[str, int]]:
|
||||
"""按段落边界聚合至目标窗长；末窗过短并入邻窗。纯函数，同输入同输出。"""
|
||||
if not isinstance(文本, str) or not 文本:
|
||||
raise 资料错误("切片需要非空原文")
|
||||
if type(窗长) is not int or 窗长 < 最短窗长:
|
||||
raise 资料错误(f"窗长必须不小于 {最短窗长}")
|
||||
段落: list[tuple[int, int]] = []
|
||||
起点 = 0
|
||||
for i, 字符 in enumerate(文本):
|
||||
if 字符 == "\n":
|
||||
if i > 起点:
|
||||
段落.append((起点, i))
|
||||
起点 = i + 1
|
||||
if 起点 < len(文本):
|
||||
段落.append((起点, len(文本)))
|
||||
if not 段落:
|
||||
raise 资料错误("原文没有可切片的段落")
|
||||
窗口: list[dict[str, int]] = []
|
||||
窗起: int | None = None
|
||||
窗终 = 0
|
||||
for 段起, 段终 in 段落:
|
||||
if 窗起 is None:
|
||||
窗起 = 段起
|
||||
窗终 = 段终
|
||||
if 窗终 - 窗起 >= 窗长:
|
||||
窗口.append({"起点": 窗起, "终点": 窗终})
|
||||
窗起 = None
|
||||
if 窗起 is not None:
|
||||
if 窗口 and 窗终 - 窗起 < 最短窗长:
|
||||
窗口[-1]["终点"] = 窗终
|
||||
else:
|
||||
窗口.append({"起点": 窗起, "终点": 窗终})
|
||||
return 窗口
|
||||
|
||||
|
||||
def 切片计划哈希(窗口: list[dict[str, int]], 窗长: int) -> str:
|
||||
return 固定哈希(
|
||||
[窗长, [[w["起点"], w["终点"]] for w in 窗口]],
|
||||
)
|
||||
|
||||
|
||||
def 切片身份(source_id: str, revision: int, 计划哈希: str, 起点: int) -> str:
|
||||
return str(
|
||||
uuid5(
|
||||
NAMESPACE_URL,
|
||||
"muse:B03:slice:" + 固定哈希([source_id, revision, 计划哈希, 起点]),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def 计算切片(
|
||||
文本: str, source_id: str, revision: int, *, 窗长: int = 目标窗长
|
||||
) -> list[dict[str, Any]]:
|
||||
"""全文→切片行（身份、定位、检索文本与哈希）；纯函数供构建与测试复用。"""
|
||||
窗口 = 段落切窗(文本, 窗长=窗长)
|
||||
计划 = 切片计划哈希(窗口, 窗长)
|
||||
行 = []
|
||||
for w in 窗口:
|
||||
检索文本 = 文本[w["起点"] : w["终点"]].strip()
|
||||
if not 检索文本:
|
||||
continue
|
||||
行.append(
|
||||
{
|
||||
"slice_id": 切片身份(source_id, revision, 计划, w["起点"]),
|
||||
"source_id": source_id,
|
||||
"revision": revision,
|
||||
"起点": w["起点"],
|
||||
"终点": w["终点"],
|
||||
"窗计划哈希": 计划,
|
||||
"检索文本": 检索文本,
|
||||
"检索文本哈希": 固定哈希([检索文本]),
|
||||
}
|
||||
)
|
||||
if not 行:
|
||||
raise 资料错误("原文切片后没有非空检索文本")
|
||||
return 行
|
||||
|
||||
|
||||
def 构建场景索引(连, 作者: str, *, 嵌入器: 特征哈希嵌入器 | None = None) -> dict[str, Any]:
|
||||
"""按当前具备 reference_reading 用途的来源版本全量重建切片与向量索引。"""
|
||||
嵌入器 = 嵌入器 or 特征哈希嵌入器()
|
||||
存储 = 资料存储(连)
|
||||
版本集 = 存储.查询(
|
||||
"SELECT s.source_id, s.title, MAX(v.revision) AS revision, "
|
||||
"(ARRAY_AGG(v.content ORDER BY v.revision DESC))[1] AS content, "
|
||||
"(ARRAY_AGG(v.content_hash ORDER BY v.revision DESC))[1] AS content_hash, "
|
||||
"(ARRAY_AGG(v.import_result->>'导入者' ORDER BY v.revision DESC))[1] AS author_id "
|
||||
"FROM muse_source s JOIN muse_source_version v ON v.source_id=s.source_id "
|
||||
"WHERE s.authorized_uses ? 'reference_reading' "
|
||||
"GROUP BY s.source_id, s.title"
|
||||
).fetchall()
|
||||
切片行: list[dict[str, Any]] = []
|
||||
来源数 = 0
|
||||
for 版 in 版本集:
|
||||
if 版["author_id"] != 作者:
|
||||
continue
|
||||
来源数 += 1
|
||||
切片行.extend(计算切片(版["content"], str(版["source_id"]), int(版["revision"])))
|
||||
索引行 = [
|
||||
(
|
||||
行["slice_id"],
|
||||
行["检索文本"],
|
||||
行["检索文本哈希"],
|
||||
嵌入器.嵌入(行["检索文本"]),
|
||||
嵌入器.身份,
|
||||
)
|
||||
for 行 in 切片行
|
||||
]
|
||||
with 连.transaction():
|
||||
连.execute("LOCK TABLE public.muse_source_slice IN SHARE ROW EXCLUSIVE MODE")
|
||||
连.execute("DELETE FROM public.muse_source_slice")
|
||||
for 行 in 切片行:
|
||||
连.execute(
|
||||
"INSERT INTO public.muse_source_slice "
|
||||
"(slice_id,source_id,revision,起点,终点,窗计划哈希,检索文本,检索文本哈希) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s)",
|
||||
(
|
||||
行["slice_id"],
|
||||
行["source_id"],
|
||||
行["revision"],
|
||||
行["起点"],
|
||||
行["终点"],
|
||||
行["窗计划哈希"],
|
||||
行["检索文本"],
|
||||
行["检索文本哈希"],
|
||||
),
|
||||
)
|
||||
for 身份, 文本, 文本哈希, 向量, embedder in 索引行:
|
||||
连.execute(
|
||||
"INSERT INTO public.muse_source_slice_index "
|
||||
"(slice_id,检索文本,检索文本哈希,vector,embedder) VALUES (%s,%s,%s,%s,%s)",
|
||||
(身份, 文本, 文本哈希, 向量, embedder),
|
||||
)
|
||||
连.execute("UPDATE public.muse_source_slice_index_version SET active=false WHERE active")
|
||||
指纹 = 固定哈希([嵌入器.身份, [(行[0], 行[2]) for 行 in 索引行]])
|
||||
头 = (
|
||||
连.cursor(row_factory=dict_row)
|
||||
.execute(
|
||||
"INSERT INTO public.muse_source_slice_index_version "
|
||||
"(索引身份,embedder,slice_count,active) VALUES (%s,%s,%s,true) "
|
||||
"RETURNING ledger_id",
|
||||
(指纹, 嵌入器.身份, len(索引行)),
|
||||
)
|
||||
.fetchone()
|
||||
)
|
||||
return {
|
||||
"ledger_id": int(头["ledger_id"]),
|
||||
"索引身份": 指纹,
|
||||
"embedder": 嵌入器.身份,
|
||||
"切片数": len(索引行),
|
||||
"来源数": 来源数,
|
||||
}
|
||||
|
||||
|
||||
def 读取场景切片(连, 作者: str, slice_ids: list[str]) -> list[dict[str, Any]]:
|
||||
"""按身份读取切片与来源版本哈希；核对来源仍具 reference_reading 用途。"""
|
||||
if not slice_ids:
|
||||
return []
|
||||
占位 = ",".join(["%s"] * len(slice_ids))
|
||||
行 = (
|
||||
资料存储(连)
|
||||
.查询(
|
||||
"SELECT x.slice_id, x.source_id, x.revision, x.起点, x.终点, x.检索文本, "
|
||||
"v.content_hash, s.title, s.authorized_uses "
|
||||
"FROM muse_source_slice x "
|
||||
"JOIN muse_source_version v ON v.source_id=x.source_id AND v.revision=x.revision "
|
||||
"JOIN muse_source s ON s.source_id=x.source_id "
|
||||
f"WHERE x.slice_id IN ({占位})",
|
||||
tuple(slice_ids),
|
||||
)
|
||||
.fetchall()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"slice_id": str(条["slice_id"]),
|
||||
"source_id": str(条["source_id"]),
|
||||
"revision": int(条["revision"]),
|
||||
"起点": int(条["起点"]),
|
||||
"终点": int(条["终点"]),
|
||||
"检索文本": 条["检索文本"],
|
||||
"内容哈希": 条["content_hash"],
|
||||
"书名": 条["title"],
|
||||
"用途": 条["authorized_uses"],
|
||||
}
|
||||
for 条 in 行
|
||||
if 条["authorized_uses"] is not None and "reference_reading" in 条["authorized_uses"]
|
||||
]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"目标窗长",
|
||||
"最短窗长",
|
||||
"段落切窗",
|
||||
"切片计划哈希",
|
||||
"切片身份",
|
||||
"计算切片",
|
||||
"构建场景索引",
|
||||
"读取场景切片",
|
||||
]
|
||||
@ -11,6 +11,7 @@ from muse.资料研究.分析保存 import 参考分析请求, 呈现分析, 登
|
||||
from muse.资料研究.分析版本 import 承接历史分析, 版本台账, 版本身份, 登记模型分析, 读取版本状态
|
||||
from muse.资料研究.分章切窗 import 切窗, 漂移检查, 窗口, 窗口载荷, 窗计划哈希, 覆盖核对
|
||||
from muse.资料研究.原文版本 import 读取证据
|
||||
from muse.资料研究.场景切片 import 构建场景索引, 计算切片, 读取场景切片
|
||||
from muse.资料研究.存储 import 资料存储
|
||||
from muse.资料研究.实体归并 import 归并, 收集实体
|
||||
from muse.资料研究.拆书分析 import 执行窗口提取
|
||||
@ -234,4 +235,7 @@ __all__ = [
|
||||
"收集实体",
|
||||
"执行窗口提取",
|
||||
"选书交接单",
|
||||
"构建场景索引",
|
||||
"计算切片",
|
||||
"读取场景切片",
|
||||
]
|
||||
|
||||
80
tests/单元/test_场景切片计算.py
Normal file
80
tests/单元/test_场景切片计算.py
Normal file
@ -0,0 +1,80 @@
|
||||
"""场景切片纯函数：段落聚合切窗与确定性身份；同输入同输出。"""
|
||||
|
||||
import pytest
|
||||
|
||||
from muse.资料研究.场景切片 import 切片计划哈希, 切片身份, 段落切窗
|
||||
from muse.资料研究.接口 import 计算切片
|
||||
|
||||
|
||||
@pytest.mark.case_id(
|
||||
"NC-w33-a10025",
|
||||
environment="离线纯函数",
|
||||
given="含多段落的参考书原文与目标窗长",
|
||||
when="执行段落聚合切窗",
|
||||
then=["窗内长度达目标即闭合；末窗过短并入邻窗；窗口首尾相接覆盖全文"],
|
||||
contract="docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
)
|
||||
def test_段落聚合切窗覆盖全文且末窗并邻__a10025():
|
||||
段 = "机甲的合金装甲在月光下泛着冷光。\n"
|
||||
文本 = 段 * 200 # 每段约 17 字，200 段约 3400 字
|
||||
窗口 = 段落切窗(文本, 窗长=1800)
|
||||
assert 窗口[0]["起点"] == 0
|
||||
assert 窗口[-1]["终点"] == len(文本) - 1 # 尾部换行不是内容，不入窗
|
||||
for 甲, 乙 in zip(窗口, 窗口[1:], strict=False):
|
||||
assert 乙["起点"] > 甲["终点"] # 段间换行符不入窗
|
||||
assert 甲["终点"] - 甲["起点"] >= 1800
|
||||
assert len(窗口) == 2
|
||||
|
||||
|
||||
@pytest.mark.case_id(
|
||||
"NC-w33-a10026",
|
||||
environment="离线纯函数",
|
||||
given="短于最短窗长的原文",
|
||||
when="执行段落切窗",
|
||||
then=["整文成单窗，不因短而拒绝"],
|
||||
contract="docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
)
|
||||
def test_短文成单窗__a10026():
|
||||
文本 = "短打斗。\n一刀封喉。\n"
|
||||
窗口 = 段落切窗(文本, 窗长=1800)
|
||||
assert len(窗口) == 1
|
||||
assert 窗口[0] == {"起点": 0, "终点": len(文本) - 1}
|
||||
|
||||
|
||||
@pytest.mark.case_id(
|
||||
"NC-w33-a10027",
|
||||
environment="离线纯函数",
|
||||
given="同一原文、来源身份与窗参数",
|
||||
when="重复计算切片",
|
||||
then=["切片身份、窗计划哈希与检索文本哈希逐次一致（确定性）"],
|
||||
contract="docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
)
|
||||
def test_切片计算确定性__a10027():
|
||||
文本 = "刀锋擦过装甲。\n火花四散。\n" * 300
|
||||
甲 = 计算切片(文本, "11111111-1111-1111-1111-111111111111", 1)
|
||||
乙 = 计算切片(文本, "11111111-1111-1111-1111-111111111111", 1)
|
||||
assert [r["slice_id"] for r in 甲] == [r["slice_id"] for r in 乙]
|
||||
assert 甲[0]["窗计划哈希"] == 切片计划哈希(段落切窗(文本, 窗长=1800), 1800)
|
||||
assert 甲[0]["slice_id"] == 切片身份(
|
||||
"11111111-1111-1111-1111-111111111111", 1, 甲[0]["窗计划哈希"], 甲[0]["起点"]
|
||||
)
|
||||
# 版本号变化即身份变化：同文本不同版本不同身份。
|
||||
丙 = 计算切片(文本, "11111111-1111-1111-1111-111111111111", 2)
|
||||
assert 甲[0]["slice_id"] != 丙[0]["slice_id"]
|
||||
|
||||
|
||||
@pytest.mark.case_id(
|
||||
"NC-w33-a10028",
|
||||
environment="离线纯函数",
|
||||
given="空原文或非法窗长",
|
||||
when="执行段落切窗",
|
||||
then=["拒绝为资料错误，不产出空窗"],
|
||||
contract="docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
)
|
||||
def test_空文与非法窗长拒绝__a10028():
|
||||
from muse.资料研究.模型 import 资料错误
|
||||
|
||||
with pytest.raises(资料错误):
|
||||
段落切窗("", 窗长=1800)
|
||||
with pytest.raises(资料错误):
|
||||
段落切窗("有字。\n", 窗长=100)
|
||||
@ -35,6 +35,7 @@ from muse.配置 import 数据库引用
|
||||
{
|
||||
("public", "muse_migration"), # 迁移账本，由迁移执行器维护
|
||||
("public", "muse_method_index_guard"), # V0059 检索索引代次单例
|
||||
("public", "muse_source_slice_guard"), # V0061 场景切片代次单例
|
||||
("public", "muse_raw_namespace"), # V0004 原文命名空间单例
|
||||
("evaluation", "muse_raw_namespace"), # V0004 原文命名空间单例（evaluation 库）
|
||||
("metadata", "type_registry"), # 内置结构种子，由导入种子写入
|
||||
|
||||
193
tests/用例清单.json
193
tests/用例清单.json
@ -27745,6 +27745,199 @@
|
||||
"数据库"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "NC-w33-a10025",
|
||||
"environment": "离线纯函数",
|
||||
"given": "含多段落的参考书原文与目标窗长",
|
||||
"when": "执行段落聚合切窗",
|
||||
"then": [
|
||||
"窗内长度达目标即闭合；末窗过短并入邻窗；窗口首尾相接覆盖全文"
|
||||
],
|
||||
"contract": "docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
"file": "tests/单元/test_场景切片计算.py",
|
||||
"symbol": "test_段落聚合切窗覆盖全文且末窗并邻__a10025",
|
||||
"parameter_ids": [],
|
||||
"node_ids": [
|
||||
"tests/单元/test_场景切片计算.py::test_段落聚合切窗覆盖全文且末窗并邻__a10025"
|
||||
],
|
||||
"fixtures": [
|
||||
"request",
|
||||
"测试资源接缝",
|
||||
"源码资源",
|
||||
"离线防护"
|
||||
],
|
||||
"markers": []
|
||||
},
|
||||
{
|
||||
"case_id": "NC-w33-a10026",
|
||||
"environment": "离线纯函数",
|
||||
"given": "短于最短窗长的原文",
|
||||
"when": "执行段落切窗",
|
||||
"then": [
|
||||
"整文成单窗，不因短而拒绝"
|
||||
],
|
||||
"contract": "docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
"file": "tests/单元/test_场景切片计算.py",
|
||||
"symbol": "test_短文成单窗__a10026",
|
||||
"parameter_ids": [],
|
||||
"node_ids": [
|
||||
"tests/单元/test_场景切片计算.py::test_短文成单窗__a10026"
|
||||
],
|
||||
"fixtures": [
|
||||
"request",
|
||||
"测试资源接缝",
|
||||
"源码资源",
|
||||
"离线防护"
|
||||
],
|
||||
"markers": []
|
||||
},
|
||||
{
|
||||
"case_id": "NC-w33-a10027",
|
||||
"environment": "离线纯函数",
|
||||
"given": "同一原文、来源身份与窗参数",
|
||||
"when": "重复计算切片",
|
||||
"then": [
|
||||
"切片身份、窗计划哈希与检索文本哈希逐次一致（确定性）"
|
||||
],
|
||||
"contract": "docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
"file": "tests/单元/test_场景切片计算.py",
|
||||
"symbol": "test_切片计算确定性__a10027",
|
||||
"parameter_ids": [],
|
||||
"node_ids": [
|
||||
"tests/单元/test_场景切片计算.py::test_切片计算确定性__a10027"
|
||||
],
|
||||
"fixtures": [
|
||||
"request",
|
||||
"测试资源接缝",
|
||||
"源码资源",
|
||||
"离线防护"
|
||||
],
|
||||
"markers": []
|
||||
},
|
||||
{
|
||||
"case_id": "NC-w33-a10028",
|
||||
"environment": "离线纯函数",
|
||||
"given": "空原文或非法窗长",
|
||||
"when": "执行段落切窗",
|
||||
"then": [
|
||||
"拒绝为资料错误，不产出空窗"
|
||||
],
|
||||
"contract": "docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
"file": "tests/单元/test_场景切片计算.py",
|
||||
"symbol": "test_空文与非法窗长拒绝__a10028",
|
||||
"parameter_ids": [],
|
||||
"node_ids": [
|
||||
"tests/单元/test_场景切片计算.py::test_空文与非法窗长拒绝__a10028"
|
||||
],
|
||||
"fixtures": [
|
||||
"request",
|
||||
"测试资源接缝",
|
||||
"源码资源",
|
||||
"离线防护"
|
||||
],
|
||||
"markers": []
|
||||
},
|
||||
{
|
||||
"case_id": "NC-w33-a10029",
|
||||
"environment": "隔离PG与生成环境",
|
||||
"given": "一个 reference_reading 授权来源与一个仅 analysis 来源",
|
||||
"when": "构建场景索引并读取切片",
|
||||
"then": [
|
||||
"仅授权来源入池，切片数与确定性计算一致",
|
||||
"切片行携带书名、定位与来源版本内容哈希",
|
||||
"重复构建为幂等重建，索引身份一致"
|
||||
],
|
||||
"contract": "docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
"file": "tests/集成/test_场景切片.py",
|
||||
"symbol": "test_构建场景索引只纳入授权来源__a10029",
|
||||
"parameter_ids": [],
|
||||
"node_ids": [
|
||||
"tests/集成/test_场景切片.py::test_构建场景索引只纳入授权来源__a10029"
|
||||
],
|
||||
"fixtures": [
|
||||
"request",
|
||||
"tmp_path",
|
||||
"tmp_path_factory",
|
||||
"内置种子方案",
|
||||
"内置结构测试库",
|
||||
"切片环境",
|
||||
"数据库底座",
|
||||
"测试资源接缝",
|
||||
"源码资源",
|
||||
"生成环境",
|
||||
"离线防护",
|
||||
"隔离数据库URL"
|
||||
],
|
||||
"markers": [
|
||||
"数据库"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "NC-w33-a10030",
|
||||
"environment": "隔离PG与生成环境",
|
||||
"given": "已入池的授权来源",
|
||||
"when": "撤回 reference_reading 用途后读取切片",
|
||||
"then": [
|
||||
"读取返回空，切片行保留（派生缓存不删历史），重建后来源数归零"
|
||||
],
|
||||
"contract": "docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
"file": "tests/集成/test_场景切片.py",
|
||||
"symbol": "test_用途撤回后切片退出候选__a10030",
|
||||
"parameter_ids": [],
|
||||
"node_ids": [
|
||||
"tests/集成/test_场景切片.py::test_用途撤回后切片退出候选__a10030"
|
||||
],
|
||||
"fixtures": [
|
||||
"request",
|
||||
"tmp_path",
|
||||
"tmp_path_factory",
|
||||
"内置种子方案",
|
||||
"内置结构测试库",
|
||||
"切片环境",
|
||||
"数据库底座",
|
||||
"测试资源接缝",
|
||||
"源码资源",
|
||||
"生成环境",
|
||||
"离线防护",
|
||||
"隔离数据库URL"
|
||||
],
|
||||
"markers": [
|
||||
"数据库"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "NC-w33-a10031",
|
||||
"environment": "隔离PG与生成环境",
|
||||
"given": "他人导入的授权来源",
|
||||
"when": "以当前作者构建场景索引",
|
||||
"then": [
|
||||
"来源不计入切片池：切片只对导入者视角构建"
|
||||
],
|
||||
"contract": "docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
"file": "tests/集成/test_场景切片.py",
|
||||
"symbol": "test_非导入者来源不入池__a10031",
|
||||
"parameter_ids": [],
|
||||
"node_ids": [
|
||||
"tests/集成/test_场景切片.py::test_非导入者来源不入池__a10031"
|
||||
],
|
||||
"fixtures": [
|
||||
"request",
|
||||
"tmp_path",
|
||||
"tmp_path_factory",
|
||||
"内置种子方案",
|
||||
"内置结构测试库",
|
||||
"切片环境",
|
||||
"数据库底座",
|
||||
"测试资源接缝",
|
||||
"源码资源",
|
||||
"生成环境",
|
||||
"离线防护",
|
||||
"隔离数据库URL"
|
||||
],
|
||||
"markers": [
|
||||
"数据库"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "NC-work-current-binding",
|
||||
"environment": "隔离 PostgreSQL；结构升级为明确的已完成状态夹具，不代替作者升级旅程",
|
||||
|
||||
120
tests/集成/test_场景切片.py
Normal file
120
tests/集成/test_场景切片.py
Normal file
@ -0,0 +1,120 @@
|
||||
"""场景切片构建与读取：授权来源入池，用途撤回即退出候选；切片可全量重建。"""
|
||||
|
||||
import pytest
|
||||
import test_模型修订任务 as 修订测试
|
||||
|
||||
from muse.资料研究.场景切片 import 计算切片
|
||||
from muse.资料研究.接口 import (
|
||||
导入请求,
|
||||
构建场景索引,
|
||||
读取场景切片,
|
||||
资料服务,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.数据库
|
||||
生成环境 = 修订测试.生成环境
|
||||
|
||||
|
||||
def _导入(env, 连, *, 标题: str, origin: str, 用途: tuple[str, ...]) -> str:
|
||||
来源 = 资料服务().导入(
|
||||
连,
|
||||
env["作者"].作者,
|
||||
导入请求(
|
||||
kind="reference",
|
||||
title=标题,
|
||||
origin=origin,
|
||||
content="刀锋擦过装甲，反震顺着掌心爬上来。\n火花在真空中凝成细线。\n" * 400,
|
||||
authorized_uses=用途,
|
||||
),
|
||||
)
|
||||
return str(来源.source_id)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def 切片环境(生成环境):
|
||||
env = 生成环境
|
||||
with env["装配"].要求数据库().连接() as 连:
|
||||
env["授权来源"] = _导入(
|
||||
env, 连, 标题="机甲打斗参考", origin="seed://slice-ref", 用途=("reference_reading",)
|
||||
)
|
||||
env["未授权来源"] = _导入(
|
||||
env, 连, 标题="分析专用资料", origin="seed://slice-analysis", 用途=("analysis",)
|
||||
)
|
||||
return env
|
||||
|
||||
|
||||
@pytest.mark.case_id(
|
||||
"NC-w33-a10029",
|
||||
environment="隔离PG与生成环境",
|
||||
given="一个 reference_reading 授权来源与一个仅 analysis 来源",
|
||||
when="构建场景索引并读取切片",
|
||||
then=[
|
||||
"仅授权来源入池，切片数与确定性计算一致",
|
||||
"切片行携带书名、定位与来源版本内容哈希",
|
||||
"重复构建为幂等重建，索引身份一致",
|
||||
],
|
||||
contract="docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
)
|
||||
def test_构建场景索引只纳入授权来源__a10029(切片环境):
|
||||
env = 切片环境
|
||||
库 = env["装配"].要求数据库()
|
||||
with 库.连接() as 连:
|
||||
回执 = 构建场景索引(连, env["作者"].作者)
|
||||
assert 回执["来源数"] == 1
|
||||
assert 回执["切片数"] > 0
|
||||
assert 回执["embedder"].startswith("hash-bigram-")
|
||||
再次 = 构建场景索引(连, env["作者"].作者)
|
||||
assert 再次["切片数"] == 回执["切片数"]
|
||||
assert 再次["索引身份"] == 回执["索引身份"]
|
||||
|
||||
raw = 连.execute("SELECT slice_id FROM muse_source_slice").fetchall()
|
||||
切片集 = [str(r[0]) for r in raw]
|
||||
明细 = 读取场景切片(连, env["作者"].作者, 切片集)
|
||||
assert len(明细) == 回执["切片数"]
|
||||
assert all(条["书名"] == "机甲打斗参考" for 条 in 明细)
|
||||
assert all(条["内容哈希"] for 条 in 明细)
|
||||
# 未授权来源切片不出现。
|
||||
assert all(条["source_id"] == env["授权来源"] for 条 in 明细)
|
||||
|
||||
|
||||
@pytest.mark.case_id(
|
||||
"NC-w33-a10030",
|
||||
environment="隔离PG与生成环境",
|
||||
given="已入池的授权来源",
|
||||
when="撤回 reference_reading 用途后读取切片",
|
||||
then=["读取返回空，切片行保留（派生缓存不删历史），重建后来源数归零"],
|
||||
contract="docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
)
|
||||
def test_用途撤回后切片退出候选__a10030(切片环境):
|
||||
env = 切片环境
|
||||
库 = env["装配"].要求数据库()
|
||||
with 库.连接() as 连:
|
||||
构建场景索引(连, env["作者"].作者)
|
||||
当前 = 连.execute("SELECT slice_id FROM muse_source_slice LIMIT 1").fetchone()
|
||||
# 测试关注读侧授权过滤，直接置空用途模拟撤回后的来源状态。
|
||||
连.execute(
|
||||
"UPDATE muse_source SET authorized_uses='[]'::jsonb WHERE source_id=%s",
|
||||
(env["授权来源"],),
|
||||
)
|
||||
明细 = 读取场景切片(连, env["作者"].作者, [str(当前[0])])
|
||||
assert 明细 == []
|
||||
保留 = 连.execute("SELECT COUNT(*) FROM muse_source_slice").fetchone()[0]
|
||||
assert 保留 > 0
|
||||
回执 = 构建场景索引(连, env["作者"].作者)
|
||||
assert 回执["来源数"] == 0 and 回执["切片数"] == 0
|
||||
|
||||
|
||||
@pytest.mark.case_id(
|
||||
"NC-w33-a10031",
|
||||
environment="隔离PG与生成环境",
|
||||
given="他人导入的授权来源",
|
||||
when="以当前作者构建场景索引",
|
||||
then=["来源不计入切片池：切片只对导入者视角构建"],
|
||||
contract="docs/系统架构/新版设计/模块设计/B03-资料研究.md#场景片段切片与索引",
|
||||
)
|
||||
def test_非导入者来源不入池__a10031(切片环境):
|
||||
env = 切片环境
|
||||
with env["装配"].要求数据库().连接() as 连:
|
||||
回执 = 构建场景索引(连, "other-author")
|
||||
assert 回执["来源数"] == 0 and 回执["切片数"] == 0
|
||||
assert 计算切片("有字。\n", env["授权来源"], 1)
|
||||
87
数据库/迁移/V0061__场景切片索引.sql
Normal file
87
数据库/迁移/V0061__场景切片索引.sql
Normal file
@ -0,0 +1,87 @@
|
||||
-- B03：场景切片派生缓存与向量代次索引——参考书原文的确定性切窗。
|
||||
-- 原文以 muse_source_version 为权威；切片与索引是可全量重建的派生数据，禁原地改写。
|
||||
DO $migration$
|
||||
BEGIN
|
||||
-- 切片台账：来源版本的定位窗；窗计划哈希锁定切窗方案，同版本只认一个窗计划。
|
||||
CREATE TABLE public.muse_source_slice (
|
||||
slice_id uuid PRIMARY KEY,
|
||||
source_id uuid NOT NULL REFERENCES public.muse_source(source_id),
|
||||
revision bigint NOT NULL CHECK (revision > 0),
|
||||
起点 int NOT NULL CHECK (起点 >= 0),
|
||||
终点 int NOT NULL CHECK (终点 > 起点),
|
||||
窗计划哈希 text NOT NULL,
|
||||
检索文本 text NOT NULL,
|
||||
检索文本哈希 text NOT NULL CHECK (检索文本哈希 ~ '^[0-9a-f]{64}$'),
|
||||
built_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
-- 同版本同窗计划内起点唯一；不同窗计划可并存，重建时整计划替换。
|
||||
CREATE UNIQUE INDEX muse_source_slice_window
|
||||
ON public.muse_source_slice (source_id, revision, 窗计划哈希, 起点);
|
||||
CREATE INDEX muse_source_slice_by_source ON public.muse_source_slice (source_id, revision);
|
||||
|
||||
-- 派生数据禁原地改写；删除仅允许全量重建事务成批进行。
|
||||
CREATE FUNCTION public.muse_guard_source_slice() RETURNS trigger LANGUAGE plpgsql AS $guard$
|
||||
BEGIN
|
||||
RAISE EXCEPTION '场景切片是派生数据，禁止原地改写；重建请整批删除后重算';
|
||||
END;
|
||||
$guard$;
|
||||
CREATE TRIGGER muse_source_slice_guard BEFORE UPDATE ON public.muse_source_slice
|
||||
FOR EACH ROW EXECUTE FUNCTION public.muse_guard_source_slice();
|
||||
|
||||
-- 切片向量索引：与 muse_method_index 同构（哈希 bigram，内存余弦）。
|
||||
CREATE TABLE public.muse_source_slice_index (
|
||||
slice_id uuid PRIMARY KEY REFERENCES public.muse_source_slice(slice_id) ON DELETE CASCADE,
|
||||
检索文本 text NOT NULL,
|
||||
检索文本哈希 text NOT NULL,
|
||||
vector double precision[] NOT NULL CHECK (array_length(vector, 1) > 0),
|
||||
embedder text NOT NULL,
|
||||
built_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
-- 代次台账：当前有效索引身份；检索时对不上即拒绝为陈旧。
|
||||
CREATE TABLE public.muse_source_slice_index_version (
|
||||
ledger_id bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
|
||||
索引身份 text NOT NULL,
|
||||
embedder text NOT NULL,
|
||||
slice_count int NOT NULL CHECK (slice_count >= 0),
|
||||
built_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
active boolean NOT NULL DEFAULT false
|
||||
);
|
||||
CREATE UNIQUE INDEX muse_source_slice_index_active
|
||||
ON public.muse_source_slice_index_version (active) WHERE active;
|
||||
|
||||
-- 代次守卫：切片与索引的事务变更递增代次，检索按代次判新鲜（仿 V0059）。
|
||||
CREATE TABLE public.muse_source_slice_guard (
|
||||
singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton),
|
||||
generation bigint NOT NULL DEFAULT 0 CHECK (generation >= 0)
|
||||
);
|
||||
INSERT INTO public.muse_source_slice_guard (singleton, generation) VALUES (true, 0);
|
||||
|
||||
CREATE FUNCTION public.muse_source_slice_changed() RETURNS trigger
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog AS $guard$
|
||||
BEGIN
|
||||
UPDATE public.muse_source_slice_guard
|
||||
SET generation = generation + 1
|
||||
WHERE singleton;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'source slice mutation guard is missing';
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$guard$;
|
||||
|
||||
CREATE TRIGGER muse_source_slice_changed
|
||||
AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON public.muse_source_slice
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.muse_source_slice_changed();
|
||||
CREATE TRIGGER muse_source_slice_index_changed
|
||||
AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON public.muse_source_slice_index
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.muse_source_slice_changed();
|
||||
CREATE TRIGGER muse_source_slice_head_changed
|
||||
AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON public.muse_source_slice_index_version
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.muse_source_slice_changed();
|
||||
|
||||
REVOKE ALL ON public.muse_source_slice_guard FROM PUBLIC, muse_app, muse_eval;
|
||||
GRANT SELECT ON public.muse_source_slice_guard TO muse_app, muse_eval;
|
||||
REVOKE ALL ON FUNCTION public.muse_source_slice_changed() FROM PUBLIC;
|
||||
END
|
||||
$migration$;
|
||||
Loading…
x
Reference in New Issue
Block a user