e6525aaa3d
接 sim-business 设计指导到生成 agent(cheap_roles+prompt.mjs 双源,先设计后写码+MVP-first);新建纯 LLM 丰富度验证 agent cheap_verify(非阻塞·不进 verdict·零 code-presence 断言,红线落地);bake_off 加 richness 列;cheap_roles 三级回落配置热取(C2a 加载器);cheap_studio trace 接线(C1b)。富游戏 M1 三品类各 5/5=100% 过九门达标。测试 test_roles 12 / test_cheap_verify 11 / test_bake_off 17 / test_cheap_trace 4 全绿。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
173 lines
8.2 KiB
Python
173 lines
8.2 KiB
Python
"""test_cheap_trace.py — C1b cheap-worker trace 落盘单测（TDD：先红后绿）。
|
||
|
||
验证 cheap-worker 生成轨迹落盘：
|
||
1. make_jsonl_sink + TraceAdapter → emit 事件 → trace.jsonl 落盘；
|
||
2. 每行是合法 JSON；
|
||
3. 五字段（traceId / step / cost / verdict / timestamp）齐全；
|
||
4. sink 写失败 → 只告警不抛（非阻塞铁律）；
|
||
5. cheap_studio.run_studio 通过 Tier2TraceMiddleware(sink=...) 接线后，
|
||
产物目录 games/amgen-<id>/trace.jsonl 存在（可选 e2e，此处为集成桩）。
|
||
|
||
复用 tier2 observability/trace.py 的 JsonlFileSink / make_jsonl_sink / TraceAdapter，
|
||
不另造 sink（最小改动原则；cheap-worker 已通过 _bootstrap 把 tier2/gen-worker 加进 sys.path）。
|
||
|
||
跑法（无 agentscope 依赖，系统 Python 即可）：
|
||
cheap-worker/.venv/bin/python cheap-worker/tests/test_cheap_trace.py
|
||
"""
|
||
|
||
import sys
|
||
import os
|
||
import json
|
||
import tempfile
|
||
from pathlib import Path
|
||
|
||
# sys.path：把 cheap-worker/ 和 tier2/gen-worker/ 加进去
|
||
# （tier2/gen-worker 含 observability 包，cheap-worker 含 cheap_studio 等）
|
||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||
_CW = os.path.dirname(_HERE) # cheap-worker/
|
||
_REPO_ROOT = os.path.dirname(_CW) # repo root
|
||
_GEN_WORKER = os.path.join(_REPO_ROOT, "tier2", "gen-worker")
|
||
|
||
for _p in (_CW, _GEN_WORKER):
|
||
if _p not in sys.path:
|
||
sys.path.insert(0, _p)
|
||
|
||
# C1b 复用 tier2 trace 基础件（cheap-worker 不另造 sink）
|
||
from observability.trace import ( # noqa: E402
|
||
JsonlFileSink, make_jsonl_sink, TraceAdapter,
|
||
)
|
||
|
||
|
||
# ── 测试一：核心落盘 + 五字段完整性（happy path）────────────────────────────────
|
||
def test_cheap_trace_creates_file_and_five_fields():
|
||
"""emit 三条桩事件 → trace.jsonl 落盘；每行合法 JSON；五字段齐；内容正确。"""
|
||
with tempfile.TemporaryDirectory() as tmpdir:
|
||
out = Path(tmpdir) / "trace.jsonl"
|
||
adapter = TraceAdapter(
|
||
trace_id="c1btrace-test",
|
||
sink=make_jsonl_sink(out),
|
||
)
|
||
|
||
# 便宜档生成中会产生的典型事件形态（dict 形式，无需真 AgentScope 环境）
|
||
fake_events = [
|
||
# ReplyStartEvent：一轮 ReAct 回复开始
|
||
{"type": "ReplyStartEvent", "created_at": "2026-06-30T10:00:00",
|
||
"reply_id": "r1", "name": "cheap-writer"},
|
||
# ModelCallEndEvent：模型调用结束，带 token 用量
|
||
{"type": "ModelCallEndEvent", "created_at": "2026-06-30T10:00:01",
|
||
"input_tokens": 200, "output_tokens": 80},
|
||
# ToolResultEndEvent：工具调用结果，带 verdict
|
||
{"type": "ToolResultEndEvent", "created_at": "2026-06-30T10:00:02",
|
||
"state": "success"},
|
||
]
|
||
for e in fake_events:
|
||
adapter.ingest(e)
|
||
|
||
# ① 文件已落盘
|
||
assert out.exists(), f"trace.jsonl 应已落盘，但不存在：{out}"
|
||
|
||
lines = out.read_text(encoding="utf-8").strip().split("\n")
|
||
# ② 三条事件对应三行
|
||
assert len(lines) == 3, f"应有 3 行，实际 {len(lines)} 行：\n" + "\n".join(lines)
|
||
|
||
# ③ 每行合法 JSON + 五字段齐全
|
||
CORE_FIELDS = {"traceId", "step", "cost", "verdict", "timestamp"}
|
||
for i, line in enumerate(lines):
|
||
obj = json.loads(line) # 若非合法 JSON 此处抛 JSONDecodeError
|
||
missing = CORE_FIELDS - set(obj.keys())
|
||
assert not missing, f"第 {i} 行缺五字段 {missing}：{line}"
|
||
assert obj["traceId"] == "c1btrace-test", (
|
||
f"第 {i} 行 traceId 错（期望 c1btrace-test，实际 {obj['traceId']}）")
|
||
assert obj["step"] == i, (
|
||
f"第 {i} 行 step 错（期望 {i}，实际 {obj['step']}）")
|
||
|
||
# ④ ModelCallEnd 步 cost.tokens 正确
|
||
second = json.loads(lines[1])
|
||
assert second["cost"] is not None, "ModelCallEnd 步 cost 应非空"
|
||
assert second["cost"]["tokens"]["in"] == 200, "input_tokens 应为 200"
|
||
assert second["cost"]["tokens"]["out"] == 80, "output_tokens 应为 80"
|
||
|
||
# ⑤ ToolResultEnd 步 verdict=success
|
||
third = json.loads(lines[2])
|
||
assert third["verdict"] == "success", (
|
||
f"ToolResultEnd 步 verdict 应为 success，实际 {third['verdict']}")
|
||
|
||
print(f" [ok] traceId={json.loads(lines[0])['traceId']}, {len(lines)} 行, 五字段齐")
|
||
|
||
|
||
# ── 测试二：嵌套目录自动创建 ─────────────────────────────────────────────────────
|
||
def test_cheap_trace_auto_creates_dir():
|
||
"""游戏目录若不存在，sink 应自动创建并写入（amgen-<id>/ 由 scaffold 建，但 best-effort 时序容错）。"""
|
||
with tempfile.TemporaryDirectory() as tmpdir:
|
||
out = Path(tmpdir) / "subdir" / "amgen-c1b" / "trace.jsonl"
|
||
sink = make_jsonl_sink(out)
|
||
step = {
|
||
"traceId": "c1b-auto", "step": 0, "cost": None,
|
||
"verdict": None, "timestamp": "2026-06-30T00:00:00",
|
||
"ext": {"raw": {"event": "ReplyStartEvent"}},
|
||
}
|
||
sink(step)
|
||
assert out.exists(), f"嵌套目录下 trace.jsonl 应自动创建：{out}"
|
||
obj = json.loads(out.read_text(encoding="utf-8").strip())
|
||
assert obj["traceId"] == "c1b-auto"
|
||
print(f" [ok] auto-mkdir: {out}")
|
||
|
||
|
||
# ── 测试三：写失败只告警不抛（非阻塞铁律）────────────────────────────────────────
|
||
def test_cheap_trace_write_fail_no_raise():
|
||
"""sink 写失败（不可写路径）→ 只 print 告警，绝不抛，不阻断调用方（非阻塞铁律）。"""
|
||
bad = Path("/proc/nonexistent_cheap_trace_test/trace.jsonl")
|
||
sink = JsonlFileSink(bad)
|
||
try:
|
||
sink({"traceId": "t-fail", "step": 0, "cost": None, "verdict": None,
|
||
"timestamp": "", "ext": {"raw": {"event": "X"}}})
|
||
print(" [ok] 写失败只告警不抛")
|
||
except Exception as e:
|
||
raise AssertionError(f"sink 写失败时不应抛异常，但抛了：{e}") from e
|
||
|
||
|
||
# ── 测试四：sink 内部抛 → adapter.ingest best-effort 不传播（非阻塞）──────────────
|
||
def test_cheap_trace_adapter_sink_fail_no_raise():
|
||
"""adapter 内层 sink 崩溃 → adapter.ingest 只记 dropped，不向外抛（non-blocking 铁律）。"""
|
||
def _bad_sink(step):
|
||
raise IOError("模拟磁盘满写失败")
|
||
|
||
adapter = TraceAdapter(trace_id="t-sink-fail", sink=_bad_sink)
|
||
# 正常 emit 一条（sink 内部崩溃）
|
||
result = adapter.ingest(
|
||
{"type": "ReplyStartEvent", "created_at": "2026-06-30T10:00:00", "reply_id": "r1"}
|
||
)
|
||
# ingest 应返回 TraceStep（映射成功），只是 sink 写失败记 dropped
|
||
assert result is not None or adapter.dropped > 0, (
|
||
"sink 崩溃时要么返回 step（sink 失败只计 dropped），要么 dropped>0")
|
||
print(f" [ok] sink 崩溃 → dropped={adapter.dropped}，adapter.ingest 不抛")
|
||
|
||
|
||
# ── 入口 ────────────────────────────────────────────────────────────────────────
|
||
if __name__ == "__main__":
|
||
tests = [
|
||
("test_cheap_trace_creates_file_and_five_fields",
|
||
test_cheap_trace_creates_file_and_five_fields),
|
||
("test_cheap_trace_auto_creates_dir",
|
||
test_cheap_trace_auto_creates_dir),
|
||
("test_cheap_trace_write_fail_no_raise",
|
||
test_cheap_trace_write_fail_no_raise),
|
||
("test_cheap_trace_adapter_sink_fail_no_raise",
|
||
test_cheap_trace_adapter_sink_fail_no_raise),
|
||
]
|
||
passed = failed = 0
|
||
for name, fn in tests:
|
||
try:
|
||
print(f"[RUN] {name}")
|
||
fn()
|
||
print(f"[PASS] {name}")
|
||
passed += 1
|
||
except Exception as exc:
|
||
import traceback
|
||
print(f"[FAIL] {name}: {exc}")
|
||
traceback.print_exc()
|
||
failed += 1
|
||
print(f"\n结果：{passed} 通过 / {failed} 失败")
|
||
import sys as _sys
|
||
_sys.exit(0 if failed == 0 else 1)
|