dc89701fcc
治「真相层缺失」病根(历史上模型文本/工具返回/门反馈零落盘、失败 run 被同 gid 重跑 覆盖,归因只能建在聚合数字上): - cheap_turns_sink.py:逐 turn 全文落 amgen-<gid>/turns.jsonl(模型文本/工具全参/ 工具返回/门续修反馈);observe-only middleware、best-effort,接线失败绝不阻断生成; CLI 与 Service 双路接线(cheap_run/cheap_service_app) - per-run 归档:同 gid 重跑前工程整体归档 _amgen-archive/<gid>-<ts>/,失败现场不再被覆盖 - evidence 清单化清理:按白名单保留,跨 run 残留(旧截图/旧 verdict)开跑即清,防证据串局 - failureLayer 三层归因:run-summary 落 generation/gate/gameplay 失败层字段, 聚合报表可分「生成没跑完/机械门挂/玩法层拒」 主会话亲验:两局真跑归档 179KB、turns 含门反馈全文;pytest 新增三个测试档全绿。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
251 lines
12 KiB
Python
251 lines
12 KiB
Python
"""test_cheap_turns_sink.py — W-AXIS 波1 真相层逐 turn 全文落盘单测(TDD:聚合 / 脱敏 / 轮转 / 开关)。
|
|
|
|
覆盖 cheap_turns_sink 的纯逻辑件(脱 agentscope,系统 Python 即可跑):
|
|
1. _redact:new-api key / Bearer / 带键名赋值 / 进程 env 真实凭据 全脱敏;
|
|
2. _TurnAggregator:一轮 ReAct 事件流聚合成整段 model_turn(模型文本 + 工具全参 + 工具返回文本 + token);
|
|
3. 门反馈/harness 输入两条路(note_reply_inputs / scan_context_gate)落 gate_feedback/harness_input + 去重;
|
|
4. CheapTurnsWriter:JSONL 落盘 + 单文件上限轮转 turns.<n>.jsonl;
|
|
5. turns_enabled 开关(默认开;CHEAP_TURNS_ENABLED=0 关)、build_turns_middleware 关时返回 None。
|
|
|
|
跑法:cheap-worker/.venv/bin/python -m pytest cheap-worker/tests/test_cheap_turns_sink.py -q
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
_CW = os.path.dirname(_HERE)
|
|
_REPO_ROOT = os.path.dirname(_CW)
|
|
_GEN_WORKER = os.path.join(_REPO_ROOT, "tier2", "gen-worker")
|
|
for _p in (_CW, _GEN_WORKER):
|
|
if _p not in sys.path:
|
|
sys.path.insert(0, _p)
|
|
|
|
import cheap_turns_sink as cts # noqa: E402
|
|
|
|
|
|
# ── 一:密钥脱敏 ─────────────────────────────────────────────────────────────
|
|
def test_redact_sk_bearer_labeled():
|
|
assert "sk-ABCDEF123456" not in cts._redact("凭据 sk-ABCDEF123456 已用")
|
|
assert cts._REDACTED in cts._redact("凭据 sk-ABCDEF123456 已用")
|
|
out = cts._redact('headers={"Authorization":"Bearer tok_9f8e7d6c5b4a"}')
|
|
assert "tok_9f8e7d6c5b4a" not in out and cts._REDACTED in out
|
|
out2 = cts._redact('api_key=deadbeefcafebabe0123')
|
|
assert "deadbeefcafebabe0123" not in out2 and cts._REDACTED in out2
|
|
|
|
|
|
def test_redact_env_real_key(monkeypatch):
|
|
monkeypatch.setenv("NEWAPI_KEY", "REALKEY-abcdef1234567890")
|
|
out = cts._redact("工具回显了 REALKEY-abcdef1234567890 这条")
|
|
assert "REALKEY-abcdef1234567890" not in out and cts._REDACTED in out
|
|
|
|
|
|
def test_redact_none_and_plain():
|
|
assert cts._redact(None) == ""
|
|
assert cts._redact("普通游戏代码 createGame({plugins})") == "普通游戏代码 createGame({plugins})"
|
|
|
|
|
|
# ── 二:一轮 ReAct 事件聚合成 model_turn ─────────────────────────────────────
|
|
def _collect(events, *, inputs=None, context=None):
|
|
"""把事件序列喂聚合器,返回落下的记录列表(emit 收集器)。"""
|
|
recs = []
|
|
agg = cts._TurnAggregator("g-demo", "g-demo", recs.append)
|
|
if inputs is not None:
|
|
agg.note_reply_inputs(inputs)
|
|
for e in events:
|
|
if cts._evt_type(e) == "ModelCallStartEvent":
|
|
agg.flush()
|
|
if context is not None:
|
|
agg.scan_context_gate(context)
|
|
agg.ingest_event(e)
|
|
agg.close()
|
|
return recs
|
|
|
|
|
|
def test_aggregate_one_round_full_text():
|
|
events = [
|
|
{"type": "ModelCallStartEvent", "model_name": "MiniMax-M3"},
|
|
{"type": "TextBlockDeltaEvent", "delta": "我先读手册"},
|
|
{"type": "TextBlockDeltaEvent", "delta": "再写 game-logic"},
|
|
{"type": "ToolCallStartEvent", "tool_call_id": "t1", "tool_call_name": "write_file"},
|
|
{"type": "ToolCallDeltaEvent", "tool_call_id": "t1", "delta": '{"path":"src/game-logic.js",'},
|
|
{"type": "ToolCallDeltaEvent", "tool_call_id": "t1", "delta": '"content":"createGame(){}"}'},
|
|
{"type": "ModelCallEndEvent", "input_tokens": 1200, "output_tokens": 800},
|
|
{"type": "ToolResultStartEvent", "tool_call_id": "t1", "tool_call_name": "write_file"},
|
|
{"type": "ToolResultTextDeltaEvent", "tool_call_id": "t1", "delta": '{"ok":true,"bytes":512}'},
|
|
{"type": "ToolResultEndEvent", "tool_call_id": "t1", "tool_call_name": "write_file", "state": "success"},
|
|
{"type": "ReplyEndEvent", "reply_id": "r1"},
|
|
]
|
|
recs = _collect(events)
|
|
turns = [r for r in recs if r["kind"] == "model_turn"]
|
|
assert len(turns) == 1, f"应聚合成一条 model_turn,实得 {recs}"
|
|
t = turns[0]
|
|
# 模型文本聚合全(不再碎成逐 delta)。
|
|
assert t["text"] == "我先读手册再写 game-logic"
|
|
# 工具调用:名 + 全参(argsDelta 聚合)。
|
|
assert t["toolCalls"][0]["name"] == "write_file"
|
|
assert '"path":"src/game-logic.js"' in t["toolCalls"][0]["args"]
|
|
assert '"content":"createGame(){}"' in t["toolCalls"][0]["args"]
|
|
# 工具返回文本(to_trace_step 丢的那段,这里补上)+ 状态。
|
|
assert t["toolResults"][0]["text"] == '{"ok":true,"bytes":512}'
|
|
assert t["toolResults"][0]["state"] == "success"
|
|
# token。
|
|
assert t["tokens"] == {"in": 1200, "out": 800}
|
|
|
|
|
|
def test_two_rounds_split_into_two_turns():
|
|
events = [
|
|
{"type": "ModelCallStartEvent", "model_name": "m"},
|
|
{"type": "TextBlockDeltaEvent", "delta": "第一轮"},
|
|
{"type": "ModelCallEndEvent", "input_tokens": 1, "output_tokens": 1},
|
|
{"type": "ModelCallStartEvent", "model_name": "m"},
|
|
{"type": "TextBlockDeltaEvent", "delta": "第二轮"},
|
|
{"type": "ModelCallEndEvent", "input_tokens": 2, "output_tokens": 2},
|
|
{"type": "ReplyEndEvent"},
|
|
]
|
|
turns = [r for r in _collect(events) if r["kind"] == "model_turn"]
|
|
assert [t["turn"] for t in turns] == [1, 2]
|
|
assert turns[0]["text"] == "第一轮" and turns[1]["text"] == "第二轮"
|
|
|
|
|
|
def test_empty_turn_not_written():
|
|
# 只有 ModelCallStart 无任何内容 → 不落空 turn。
|
|
turns = [r for r in _collect([
|
|
{"type": "ModelCallStartEvent", "model_name": "m"},
|
|
{"type": "ReplyEndEvent"},
|
|
]) if r["kind"] == "model_turn"]
|
|
assert turns == []
|
|
|
|
|
|
# ── 三:门反馈 / harness 输入两条路 ───────────────────────────────────────────
|
|
def test_harness_input_and_gate_feedback():
|
|
# CLI 路:kick(name=user)→ harness_input;门续修反馈同理(CLI 用 name=user 的 kick 承载 feedback)。
|
|
recs = _collect(
|
|
[{"type": "ModelCallStartEvent", "model_name": "m"},
|
|
{"type": "TextBlockDeltaEvent", "delta": "改"},
|
|
{"type": "ModelCallEndEvent", "input_tokens": 1, "output_tokens": 1},
|
|
{"type": "ReplyEndEvent"}],
|
|
inputs=[{"name": "user", "content": "请按 brief 造游戏"}],
|
|
)
|
|
kinds = [r["kind"] for r in recs]
|
|
assert "harness_input" in kinds
|
|
hi = next(r for r in recs if r["kind"] == "harness_input")
|
|
assert hi["text"] == "请按 brief 造游戏"
|
|
|
|
|
|
def test_gate_feedback_from_context_and_dedup():
|
|
# Service 路:mid-reply 注入 name=gate 续修,每见 ModelCallStart 扫 context;同内容只落一次。
|
|
ctx = [{"name": "user", "content": "kick"},
|
|
{"name": "gate", "content": "以下真玩验收门未通过,请修 H_progress"}]
|
|
recs = _collect(
|
|
[{"type": "ModelCallStartEvent", "model_name": "m"},
|
|
{"type": "TextBlockDeltaEvent", "delta": "第一轮"},
|
|
{"type": "ModelCallEndEvent", "input_tokens": 1, "output_tokens": 1},
|
|
{"type": "ModelCallStartEvent", "model_name": "m"}, # 第二次 MCS 再扫一遍 context
|
|
{"type": "TextBlockDeltaEvent", "delta": "第二轮"},
|
|
{"type": "ModelCallEndEvent", "input_tokens": 1, "output_tokens": 1},
|
|
{"type": "ReplyEndEvent"}],
|
|
context=ctx,
|
|
)
|
|
gate_recs = [r for r in recs if r["kind"] == "gate_feedback"]
|
|
assert len(gate_recs) == 1, "同内容门续修只落一次(去重)"
|
|
assert "H_progress" in gate_recs[0]["text"]
|
|
|
|
|
|
def test_gate_feedback_ordered_after_prior_turn():
|
|
# 时间序(真实注入序):门续修在【第一轮结束后】才注入 context,故第二轮 ModelCallStart 才扫到。
|
|
# 手动驱动以复现「先冲上一 turn → 再落 gate → 再起新 turn」的中间件顺序(_collect 静态 context 无法表达)。
|
|
recs = []
|
|
agg = cts._TurnAggregator("g", "g", recs.append)
|
|
ctx = [] # 起手 context 无 gate(第一轮时判卷还没跑)
|
|
for e in [{"type": "ModelCallStartEvent", "model_name": "m"},
|
|
{"type": "TextBlockDeltaEvent", "delta": "第一轮"},
|
|
{"type": "ModelCallEndEvent", "input_tokens": 1, "output_tokens": 1}]:
|
|
if cts._evt_type(e) == "ModelCallStartEvent":
|
|
agg.flush(); agg.scan_context_gate(ctx)
|
|
agg.ingest_event(e)
|
|
ctx.append({"name": "gate", "content": "门未通过"}) # 第一轮后 RepairMiddleware 注入门续修
|
|
for e in [{"type": "ModelCallStartEvent", "model_name": "m"},
|
|
{"type": "TextBlockDeltaEvent", "delta": "第二轮"},
|
|
{"type": "ModelCallEndEvent", "input_tokens": 1, "output_tokens": 1},
|
|
{"type": "ReplyEndEvent"}]:
|
|
if cts._evt_type(e) == "ModelCallStartEvent":
|
|
agg.flush(); agg.scan_context_gate(ctx)
|
|
agg.ingest_event(e)
|
|
agg.close()
|
|
seq_kinds = [r["kind"] for r in recs]
|
|
# 期望序:model_turn(第一轮) → gate_feedback → model_turn(第二轮)
|
|
i_turn1 = seq_kinds.index("model_turn")
|
|
i_gate = seq_kinds.index("gate_feedback")
|
|
assert i_turn1 < i_gate, "门续修须落在上一 turn 之后(时间序正确)"
|
|
assert i_gate < seq_kinds.index("model_turn", i_gate), "门续修须在下一 turn 之前"
|
|
|
|
|
|
def test_input_redacted():
|
|
recs = _collect(
|
|
[{"type": "ModelCallStartEvent", "model_name": "m"},
|
|
{"type": "TextBlockDeltaEvent", "delta": "x"},
|
|
{"type": "ModelCallEndEvent", "input_tokens": 1, "output_tokens": 1},
|
|
{"type": "ReplyEndEvent"}],
|
|
inputs=[{"name": "user", "content": "key=sk-SECRET1234567 造游戏"}],
|
|
)
|
|
hi = next(r for r in recs if r["kind"] == "harness_input")
|
|
assert "sk-SECRET1234567" not in hi["text"] and cts._REDACTED in hi["text"]
|
|
|
|
|
|
# ── 四:落盘 + 轮转 ──────────────────────────────────────────────────────────
|
|
def test_writer_appends_jsonl():
|
|
with tempfile.TemporaryDirectory() as td:
|
|
p = Path(td) / "turns.jsonl"
|
|
w = cts.CheapTurnsWriter(p)
|
|
w.write_record({"kind": "model_turn", "turn": 1, "text": "a"})
|
|
w.write_record({"kind": "model_turn", "turn": 2, "text": "b"})
|
|
lines = p.read_text(encoding="utf-8").strip().splitlines()
|
|
assert len(lines) == 2
|
|
assert json.loads(lines[0])["turn"] == 1 and json.loads(lines[1])["turn"] == 2
|
|
|
|
|
|
def test_writer_rotates_on_cap():
|
|
with tempfile.TemporaryDirectory() as td:
|
|
p = Path(td) / "turns.jsonl"
|
|
w = cts.CheapTurnsWriter(p, max_bytes=200) # 极小上限逼轮转
|
|
for i in range(20):
|
|
w.write_record({"kind": "model_turn", "turn": i, "pad": "x" * 40})
|
|
rotated = list(Path(td).glob("turns.*.jsonl"))
|
|
assert rotated, "超单文件上限应轮转出 turns.<n>.jsonl"
|
|
# 轮转分片 + 现文件的行数总和 = 写入条数(不丢记录)。
|
|
total = 0
|
|
for f in [p] + rotated:
|
|
total += len(f.read_text(encoding="utf-8").strip().splitlines())
|
|
assert total == 20
|
|
|
|
|
|
def test_writer_write_failure_is_silent(monkeypatch):
|
|
# 落盘失败(路径不可写)只告警、绝不抛(best-effort 铁律)。
|
|
w = cts.CheapTurnsWriter("/nonexistent-root-xyz/deep/turns.jsonl")
|
|
w.write_record({"kind": "model_turn", "turn": 1}) # 不抛即通过
|
|
|
|
|
|
# ── 五:开关 ─────────────────────────────────────────────────────────────────
|
|
def test_enabled_default_on_and_off(monkeypatch):
|
|
monkeypatch.delenv(cts.ENV_ENABLED, raising=False)
|
|
assert cts.turns_enabled() is True
|
|
for off in ("0", "false", "no", "off"):
|
|
monkeypatch.setenv(cts.ENV_ENABLED, off)
|
|
assert cts.turns_enabled() is False
|
|
monkeypatch.setenv(cts.ENV_ENABLED, "1")
|
|
assert cts.turns_enabled() is True
|
|
|
|
|
|
def test_build_middleware_none_when_disabled(monkeypatch):
|
|
monkeypatch.setenv(cts.ENV_ENABLED, "0")
|
|
assert cts.build_turns_middleware("g1") is None
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import pytest
|
|
sys.exit(pytest.main([__file__, "-q"]))
|