范围(不含 design-story-foundation、docs/、humanization/README.md 等进行中改动):
1. 新增 harness/ 控制平面
- skill_harness.py 静态审计:32 个运行时 Skill 的 frontmatter/manifest/文档污染,当前 0 问题
- run_selected.py 选择性执行器:manifest 与磁盘一一对账、依赖阻断、
空跑与 skip-only 失败关闭、AST 测试形状门
- manifests/skills.json:32 个 Skill 的合同责任方与协作领域登记
- manifests/test-inventory.json:81 个测试资产登记
- specs/skill-testing.md 与 README.md:测试分层、证据边界与 harness 职责
2. 实现测试从 .claude/skills/*/scripts/ 迁至 tests/skills/<skill>/
- 71 个测试文件迁移并修复项目根与临时目录运行导入
- 数据库触发器测试宽泛异常收窄为 psycopg.errors.RaiseException
- 抽取离线大测试拆出真实 PG smoke(默认阻断,不计入离线通过)
- 抽取 presence 去重边界拆出独立测试:493 + 78 = 571 项检查不变
3. 运行时文档清理
- 13 个 SKILL.md 移除自测/离线验证段落、测试命令与测试文件事实源表述,
只保留运行时合同;业务运行合同、额度、授权与离线模式均保留
4. SoT 同步
- AGENTS.md:新增 Skill 领域索引(7 个合同责任方分组,覆盖 32 个运行时 Skill)
- 领域 07:测试入口改由 harness/manifests/ 登记,SKILL.md 不承载测试命令
- humanization 覆盖矩阵:活动测试路径同步迁移
验证证据: harness 自测 15 项 + runner 自测 13 项通过;静态审计 32 Skill / 0 问题;
73 个非数据库测试通过;8 个集成条目中 6 个 PostgreSQL 项被依赖门明确阻断;
py_compile 与 git diff --check 通过。未连接 PostgreSQL、网络、真实模型或额度。
已知边界: 真正 skill_behavior_eval 仍为 0,尚未验证任何 Skill 自然语言行为;
evaluate-frozen-replay 的 raw 存储边界冲突留待单独治理。
142 lines
4.1 KiB
Python
142 lines
4.1 KiB
Python
#!/usr/bin/env python3
|
|
"""模型调用持久化合同的离线测试。
|
|
|
|
这些测试不连网、不连库,只固定共享 LLM 入口必须向持久化适配器提供的证据形状。
|
|
"""
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
import types
|
|
|
|
|
|
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
SCRIPT_DIR = PROJECT_ROOT / ".claude" / "skills" / "call-content-model" / "scripts"
|
|
sys.path.insert(0, str(SCRIPT_DIR))
|
|
import llm # noqa: E402
|
|
|
|
|
|
USAGE = {
|
|
"prompt_tokens": 12,
|
|
"completion_tokens": 7,
|
|
"prompt_tokens_details": {"cached_tokens": 3},
|
|
}
|
|
|
|
|
|
class FakeResponse:
|
|
status_code = 200
|
|
text = ""
|
|
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
def json(self):
|
|
return {
|
|
"choices": [{
|
|
"message": {"content": "模型输出"},
|
|
"finish_reason": "stop",
|
|
}],
|
|
"usage": USAGE,
|
|
"id": "completion-1",
|
|
}
|
|
|
|
|
|
def test_chat_emits_a_complete_persistence_event():
|
|
"""成功调用应把完整请求/响应和审计字段交给原子落库适配器。"""
|
|
events = []
|
|
|
|
class FakeSession:
|
|
trust_env = True
|
|
|
|
def post(self, url, headers=None, json=None, timeout=None):
|
|
return FakeResponse()
|
|
|
|
old_session, old_time = llm.requests.Session, llm.time
|
|
try:
|
|
llm.requests.Session = FakeSession
|
|
llm.time = types.SimpleNamespace(time=lambda: 100.0, sleep=lambda _: None)
|
|
content, usage = llm.chat(
|
|
"用户提示",
|
|
model="MiniMax-M3",
|
|
system="系统提示",
|
|
retries=0,
|
|
run_id="run-1",
|
|
caller="parse-book",
|
|
persist_call=events.append,
|
|
)
|
|
finally:
|
|
llm.requests.Session, llm.time = old_session, old_time
|
|
|
|
assert (content, usage) == ("模型输出", USAGE)
|
|
assert len(events) == 1
|
|
event = events[0]
|
|
assert event["run_id"] == "run-1"
|
|
assert event["caller"] == "parse-book"
|
|
assert event["requested_model_id"] == "MiniMax-M3"
|
|
assert event["actual_model_id"] == "MiniMax-M3"
|
|
assert event["usage"] == USAGE
|
|
assert event["duration_ms"] == 0
|
|
assert event["stop_reason"] == "stop"
|
|
assert json.loads(event["prompt"]) == {
|
|
"messages": [
|
|
{"role": "system", "content": "系统提示"},
|
|
{"role": "user", "content": "用户提示"},
|
|
],
|
|
"model": "MiniMax-M3",
|
|
"max_tokens": 512000,
|
|
"temperature": 0.2,
|
|
}
|
|
assert json.loads(event["response"])["choices"][0]["message"]["content"] == "模型输出"
|
|
|
|
|
|
def test_governed_forwards_persistence_context_to_actual_model_call():
|
|
"""额度治理选出的实际模型必须继续携带 run/caller/持久化适配器。"""
|
|
calls = []
|
|
|
|
def fake_chat(prompt, model=None, **kwargs):
|
|
calls.append((prompt, model, kwargs))
|
|
return "ok", {"prompt_tokens": 1, "completion_tokens": 1}
|
|
|
|
old_now = llm._now
|
|
old_read = llm._read_window
|
|
old_bump = llm._bump_window
|
|
old_chat = llm.chat
|
|
old_pricing = llm._PRICING_CACHE
|
|
try:
|
|
llm._now = lambda: __import__("datetime").datetime(2026, 7, 16, 12, 0)
|
|
llm._read_window = lambda _: (0.0, 0)
|
|
llm._bump_window = lambda *_: (0.0, 1)
|
|
llm._PRICING_CACHE = dict(llm.PRICING_FALLBACK)
|
|
llm.chat = fake_chat
|
|
marker = object()
|
|
content, _, used = llm.chat_governed(
|
|
"prompt",
|
|
run_id="run-2",
|
|
caller="extract-knowledge",
|
|
persist_call=marker,
|
|
)
|
|
finally:
|
|
llm._now = old_now
|
|
llm._read_window = old_read
|
|
llm._bump_window = old_bump
|
|
llm.chat = old_chat
|
|
llm._PRICING_CACHE = old_pricing
|
|
|
|
assert (content, used) == ("ok", "MiniMax-M3")
|
|
assert calls[0][2]["run_id"] == "run-2"
|
|
assert calls[0][2]["caller"] == "extract-knowledge"
|
|
assert calls[0][2]["persist_call"] is marker
|
|
|
|
|
|
def main():
|
|
for test in (
|
|
test_chat_emits_a_complete_persistence_event,
|
|
test_governed_forwards_persistence_context_to_actual_model_call,
|
|
):
|
|
test()
|
|
print(f" ✓ {test.__name__}")
|
|
print("全部通过")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|