d909d1bd1b
实现侧： - 上下文：任务范围拆分为 范围校验/范围授权；索引按可发现口径重建、索引新鲜度改对称差；依赖校验统一快照漂移说明。 - 知识方法：方法与材料读取口径统一；超限方法材料按可选省略，核对路径不再二次计费；删除无合同的读时重算。 - 任务运行：新增 context.usage/tool.denied 事件类型；连接池常驻并在装配生命周期内开关；调用结算与核对分列。 - 效果评测/审校修订/交付连载/作者经验/作品规划：凭据冻结、标定消费、导出补证、事实引文核对等收尾修复。 - 资源加载：能力正文不再夹带索引用的导航注记（该注记此前进入角色与技能的模型提示）。 - 元数据：受保护骨架与代码保护属性对齐；字段校验与内置结构口径同步。 - 基础设施：环境预检进入装配生命周期；数据库连接运行期字段不参与相等比较；索引指纹归一化 jsonb 浮点。 - 删除被替代实现：7 份旧提示词模板与空壳 资料来源 读取器。 用例侧： - 用例身份与导航元信息迁移；夹具补生命周期、同库暴露与模板封存； - 本轮定向修复：方法材料省略、事实引文、迁移回执、额度与暂停用例、慢用例超时预算等。
321 lines
13 KiB
Python
321 lines
13 KiB
Python
"""旧源只读加载：仅合成SQLite和显式隔离PG；不访问真实旧库或导入业务目标。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import base64
|
||
import hashlib
|
||
import json
|
||
import sqlite3
|
||
import subprocess
|
||
import sys
|
||
from pathlib import Path
|
||
from uuid import uuid4
|
||
|
||
import psycopg
|
||
import pytest
|
||
from psycopg import sql
|
||
|
||
from 建立映射 import 迁移错误
|
||
from 读取旧PG import _打开只读连接 as 打开只读PG
|
||
from 读取旧PG import 读取旧库 as 读取PG
|
||
from 读取旧SQLite import _打开只读连接 as 打开只读SQLite
|
||
from 读取旧SQLite import 读取旧库 as 读取SQLite
|
||
|
||
|
||
@pytest.fixture
|
||
def sqlite副本(tmp_path):
|
||
路径 = tmp_path / "合成旧库.db"
|
||
with sqlite3.connect(路径) as 连:
|
||
连.executescript("""
|
||
CREATE TABLE runs(id TEXT PRIMARY KEY, output_text TEXT NOT NULL);
|
||
CREATE TABLE events(run_id TEXT NOT NULL,seq INTEGER NOT NULL,
|
||
payload_json TEXT,PRIMARY KEY(run_id,seq));
|
||
CREATE TABLE cards(id TEXT PRIMARY KEY,kind TEXT,payload_json TEXT,embedding BLOB);
|
||
INSERT INTO runs VALUES('r1','仅供测试的原运行输出');
|
||
INSERT INTO events VALUES('r1',1,'{"片段":"甲"}'),('r1',2,'{"片段":"乙"}');
|
||
""")
|
||
连.execute("INSERT INTO cards VALUES(?,?,?,?)", ("c1", "embedding", "{}", b"\x00\xff"))
|
||
return 路径
|
||
|
||
|
||
def _sqlite(路径, **参数):
|
||
return 读取SQLite(
|
||
路径, 数据集="合成旧源", 表=("runs", "events", "cards"), 快照版本="export-v1", **参数
|
||
)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w21-21b003",
|
||
environment="离线合成JSON/SQLite与CLI；不含目标PG导入",
|
||
given="合成SQLite运行、复合键事件及二进制镜像",
|
||
when="对明确合成源执行只读加载、纯分流、身份核对或独立CLI",
|
||
then=["原行、复合主键及字节可恢复，只读连接拒绝写入，源文件不变"],
|
||
contract="docs/系统架构/新版设计/数据模型/旧知识记录分流.md",
|
||
)
|
||
def test_SQLite原行复合主键字节与只读性__21b003(sqlite副本):
|
||
原哈希 = hashlib.sha256(sqlite副本.read_bytes()).hexdigest()
|
||
结果 = _sqlite(sqlite副本)
|
||
assert len(结果) == 4
|
||
事件 = [r for r in 结果 if r.source.table == "events"]
|
||
assert [json.loads(r.source.id) for r in 事件] == [
|
||
{"run_id": "r1", "seq": 1},
|
||
{"run_id": "r1", "seq": 2},
|
||
]
|
||
assert all(r.source.revision == "export-v1" for r in 结果)
|
||
assert all(str(sqlite副本) not in r.source.database for r in 结果)
|
||
卡 = next(r for r in 结果 if r.source.table == "cards")
|
||
assert base64.b64decode(卡.原行["embedding"]["__bytea_base64__"]) == b"\x00\xff"
|
||
with 打开只读SQLite(sqlite副本) as 连:
|
||
assert 连.execute("PRAGMA query_only").fetchone()[0] == 1
|
||
with pytest.raises(sqlite3.OperationalError, match="readonly"):
|
||
连.execute("DELETE FROM runs")
|
||
assert hashlib.sha256(sqlite副本.read_bytes()).hexdigest() == 原哈希
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w21-21b004",
|
||
environment="离线合成JSON/SQLite与CLI；不含目标PG导入",
|
||
given="缺失副本、未登记表或视图",
|
||
when="对明确合成源执行只读加载、纯分流、身份核对或独立CLI",
|
||
then=["读取明确失败，不创建文件或暗读未允许来源"],
|
||
contract="docs/系统架构/新版设计/数据模型/旧知识记录分流.md",
|
||
)
|
||
def test_SQLite缺文件未知表和视图不能暗读__21b004(sqlite副本, tmp_path):
|
||
缺失 = tmp_path / "不存在.db"
|
||
with pytest.raises(FileNotFoundError):
|
||
_sqlite(缺失)
|
||
assert not 缺失.exists()
|
||
with pytest.raises(迁移错误, match="source_table_denied"):
|
||
读取SQLite(sqlite副本, 数据集="合成旧源", 表=("private_keys",), 快照版本="x")
|
||
with sqlite3.connect(sqlite副本) as 连:
|
||
连.execute("DROP TABLE cards")
|
||
连.execute("CREATE VIEW cards AS SELECT id FROM runs")
|
||
with pytest.raises(迁移错误, match="source_table_denied"):
|
||
_sqlite(sqlite副本)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w21-21b005",
|
||
environment="离线合成JSON/SQLite与CLI；不含目标PG导入",
|
||
given="复合主键范围或超出快照配额",
|
||
when="对明确合成源执行只读加载、纯分流、身份核对或独立CLI",
|
||
then=["只读指定范围，错误范围拒绝，行数和字节超限不截断"],
|
||
contract="docs/系统架构/新版设计/数据模型/旧知识记录分流.md",
|
||
)
|
||
def test_SQLite范围行数与字节上限不静默截断__21b005(sqlite副本):
|
||
结果 = 读取SQLite(
|
||
sqlite副本,
|
||
数据集="合成旧源",
|
||
表=("events",),
|
||
快照版本="x",
|
||
范围={"events": {"start": ["r1", 2], "end": ["r1", 3]}},
|
||
)
|
||
assert len(结果) == 1 and 结果[0].原行["seq"] == 2
|
||
with pytest.raises(迁移错误, match="source_range_invalid"):
|
||
_sqlite(sqlite副本, 范围={"events": {"start": [1]}})
|
||
with pytest.raises(迁移错误, match="snapshot_limit"):
|
||
_sqlite(sqlite副本, 最大行数=1)
|
||
with sqlite3.connect(sqlite副本) as 连:
|
||
连.execute("UPDATE runs SET output_text=?", ("x" * (16 * 1024 * 1024),))
|
||
with pytest.raises(迁移错误, match="snapshot_limit"):
|
||
_sqlite(sqlite副本)
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w21-21b006",
|
||
environment="离线合成JSON/SQLite与CLI；不含目标PG导入",
|
||
given="CLI从合成SQLite导出并交给分流",
|
||
when="对明确合成源执行只读加载、纯分流、身份核对或独立CLI",
|
||
then=["快照可重放且私有，不覆盖副本；运行原文不进stdout，分流仅保历史"],
|
||
contract="docs/系统架构/新版设计/数据模型/旧知识记录分流.md",
|
||
)
|
||
def test_读取命令接纯分流且不覆盖源__21b006(sqlite副本, tmp_path):
|
||
脚本 = Path(__file__).resolve().parents[2] / "数据库/旧库迁移/入口.py"
|
||
输出, 映射, 计划 = [tmp_path / n for n in ("快照.json", "映射.json", "计划.json")]
|
||
命令 = [
|
||
sys.executable,
|
||
str(脚本),
|
||
"读取SQLite",
|
||
"--副本",
|
||
str(sqlite副本),
|
||
"--数据集",
|
||
"合成旧源",
|
||
"--表",
|
||
"runs",
|
||
"--表",
|
||
"events",
|
||
"--快照版本",
|
||
"export-v1",
|
||
"--输出",
|
||
str(输出),
|
||
]
|
||
for _ in range(2):
|
||
运行 = subprocess.run(命令, capture_output=True, text=True, timeout=15)
|
||
assert 运行.returncode == 0, 运行.stderr
|
||
assert "仅供测试的原运行输出" not in 运行.stdout
|
||
assert len(json.loads(输出.read_text())) == 3 and 输出.stat().st_mode & 0o777 == 0o600
|
||
映射.write_text('{"works": []}', encoding="utf-8")
|
||
运行 = subprocess.run(
|
||
[
|
||
sys.executable,
|
||
str(脚本),
|
||
"分流",
|
||
"--快照",
|
||
str(输出),
|
||
"--映射",
|
||
str(映射),
|
||
"--输出",
|
||
str(计划),
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=15,
|
||
)
|
||
assert 运行.returncode == 0, 运行.stderr
|
||
assert json.loads(计划.read_text())["dispositions"] == {"历史保留": 3}
|
||
哈希 = hashlib.sha256(sqlite副本.read_bytes()).hexdigest()
|
||
assert subprocess.run([*命令[:-1], str(sqlite副本)], capture_output=True).returncode == 1
|
||
assert hashlib.sha256(sqlite副本.read_bytes()).hexdigest() == 哈希
|
||
|
||
|
||
@pytest.fixture
|
||
def pg旧源(隔离数据库URL, tmp_path):
|
||
schema = "w21_source_" + uuid4().hex
|
||
with psycopg.connect(隔离数据库URL) as 连:
|
||
连.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema)))
|
||
连.execute(
|
||
sql.SQL(
|
||
"CREATE TABLE {}(id bigint PRIMARY KEY, tenant_id bigint NOT NULL, "
|
||
"revision int NOT NULL, description text)"
|
||
).format(sql.Identifier(schema, "muse_knowledge_entity"))
|
||
)
|
||
连.execute(
|
||
sql.SQL(
|
||
"INSERT INTO {} VALUES (1,7,3,'合成甲'),(2,8,4,'合成乙'),(3,7,5,'合成丙')"
|
||
).format(sql.Identifier(schema, "muse_knowledge_entity"))
|
||
)
|
||
连.execute(
|
||
sql.SQL(
|
||
"CREATE TABLE {}(run_id text, seq int, data text, PRIMARY KEY(run_id,seq))"
|
||
).format(sql.Identifier(schema, "example_run_receipt"))
|
||
)
|
||
连.execute(
|
||
sql.SQL("INSERT INTO {} VALUES ('r1',1,'合成事件')").format(
|
||
sql.Identifier(schema, "example_run_receipt")
|
||
)
|
||
)
|
||
文件 = tmp_path / "连接说明.txt"
|
||
文件.write_text(隔离数据库URL, encoding="utf-8")
|
||
文件.chmod(0o600)
|
||
try:
|
||
yield 文件, schema
|
||
finally:
|
||
with psycopg.connect(隔离数据库URL) as 连:
|
||
连.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema)))
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w21-21b007",
|
||
environment="隔离PG合成源；不含真实旧库或目标业务导入",
|
||
given="隔离PG中不同租户、真实修订和复合主键源",
|
||
when="对明确合成源执行只读加载、纯分流、身份核对或独立CLI",
|
||
then=["读取前按租户限定，真实修订不替换，范围与缺修订快照版本可核对"],
|
||
contract="docs/系统架构/新版设计/数据模型/旧知识记录分流.md",
|
||
)
|
||
@pytest.mark.数据库
|
||
def test_PG按租户范围读取且保留真实修订__21b007(pg旧源):
|
||
文件, schema = pg旧源
|
||
参数 = {
|
||
"数据集": "合成PG",
|
||
"schema": schema,
|
||
"表": ("muse_knowledge_entity",),
|
||
"快照版本": "export-v1",
|
||
}
|
||
一 = 读取PG(文件, 租户=7, **参数)
|
||
二 = 读取PG(文件, 租户=8, **参数)
|
||
assert [r.原行["id"] for r in 一] == [1, 3] and [r.原行["id"] for r in 二] == [2]
|
||
assert [r.source.revision for r in 一] == ["3", "5"]
|
||
assert 一[0].source.database != 二[0].source.database
|
||
限定 = 读取PG(文件, 租户=7, 范围={"muse_knowledge_entity": {"start": [3], "end": [4]}}, **参数)
|
||
assert len(限定) == 1 and 限定[0].原行["id"] == 3
|
||
with pytest.raises(迁移错误, match="source_scope_invalid"):
|
||
读取PG(文件, 租户=None, **参数)
|
||
with pytest.raises(迁移错误, match="snapshot_limit"):
|
||
读取PG(文件, 租户=7, 最大行数=1, **参数)
|
||
全局 = 读取PG(
|
||
文件,
|
||
数据集="合成PG",
|
||
schema=schema,
|
||
租户=None,
|
||
表=("example_run_receipt",),
|
||
快照版本="export-v1",
|
||
)
|
||
assert json.loads(全局[0].source.id) == {"run_id": "r1", "seq": 1}
|
||
assert 全局[0].source.revision == "export-v1"
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w21-21b008",
|
||
environment="隔离PG合成源；不含真实旧库或目标业务导入",
|
||
given="隔离PG服务器只读事务",
|
||
when="对明确合成源执行只读加载、纯分流、身份核对或独立CLI",
|
||
then=["服务器实际拒绝删除，隔离级别固定；视图不读取"],
|
||
contract="docs/系统架构/新版设计/数据模型/旧知识记录分流.md",
|
||
)
|
||
@pytest.mark.数据库
|
||
def test_PG源事务真正只读且声明错误不读取__21b008(pg旧源):
|
||
文件, schema = pg旧源
|
||
with pytest.raises(迁移错误, match="25006"):
|
||
with 打开只读PG(文件) as 连:
|
||
assert (
|
||
连.execute("SHOW transaction_read_only").fetchone()["transaction_read_only"] == "on"
|
||
)
|
||
assert (
|
||
连.execute("SHOW transaction_isolation").fetchone()["transaction_isolation"]
|
||
== "repeatable read"
|
||
)
|
||
连.execute(
|
||
sql.SQL("DELETE FROM {}").format(sql.Identifier(schema, "muse_knowledge_entity"))
|
||
)
|
||
with psycopg.connect(文件.read_text()) as 连:
|
||
assert (
|
||
连.execute(
|
||
sql.SQL("SELECT count(*) FROM {}").format(
|
||
sql.Identifier(schema, "muse_knowledge_entity")
|
||
)
|
||
).fetchone()[0]
|
||
== 3
|
||
)
|
||
连.execute(
|
||
sql.SQL("CREATE VIEW {} AS SELECT 1 AS id").format(
|
||
sql.Identifier(schema, "example_run")
|
||
)
|
||
)
|
||
with pytest.raises(迁移错误, match="source_table_denied"):
|
||
读取PG(文件, 数据集="合成PG", schema=schema, 租户=None, 表=("example_run",), 快照版本="v1")
|
||
|
||
|
||
@pytest.mark.case_id(
|
||
"NC-w21-21b009",
|
||
environment="离线合成JSON/SQLite与CLI；不含目标PG导入",
|
||
given="连接说明缺明确参数或含无效敏感内容",
|
||
when="对明确合成源执行只读加载、纯分流、身份核对或独立CLI",
|
||
then=["不连接默认环境，不把驱动原错误中的凭据写出"],
|
||
contract="docs/系统架构/新版设计/数据模型/旧知识记录分流.md",
|
||
)
|
||
def test_连接说明不回落环境且错误不泄露凭据__21b009(tmp_path, monkeypatch):
|
||
文件 = tmp_path / "连接说明.txt"
|
||
monkeypatch.setattr(
|
||
psycopg.Connection, "connect", lambda *a, **k: pytest.fail("配置非法时不得连接")
|
||
)
|
||
for 内容 in ("", "dbname=somewhere", "host=localhost user=someone"):
|
||
文件.write_text(内容, encoding="utf-8")
|
||
with pytest.raises(迁移错误, match="source_connection_invalid"):
|
||
with 打开只读PG(文件):
|
||
pytest.fail("不得取得连接")
|
||
文件.write_text("password='never-print-this", encoding="utf-8")
|
||
with pytest.raises(迁移错误) as 错:
|
||
with 打开只读PG(文件):
|
||
pytest.fail("不得取得连接")
|
||
assert "never-print-this" not in str(错.value)
|