"""test_dedup.py — M3b U3:D9 反同质化(vendored dedup)单测。 守的不变量(镜像 wg1 agent_loop/dedup.py,唯一改动 = DEDUP_REGISTRY 指 cheap-worker/results/): · 首次 (title,theme) 不撞 dupHit=False;第二次同 (title,theme) 撞重 dupHit=True、dupWith 指首款; · 不同 title 不撞 + sig 不同;norm_text 去空白+casefold+去标点;compute_sig 可复现 64-hex; · 登记表坏行容错(降级不抛 dupHit ∈ {False,None},不阻断); · DEDUP_REGISTRY 落 cheap-worker/results/、绝不写 wg1(路径隔离铁律); · 接线:process_job 把 similarity 塞 summary → trace.similarity;撞重不改 status(仍按九门判)。 全 stdlib、零网络/零 LLM(隔离临时 registry,不污染真 cheap-worker/results/)。 跑:cheap-worker/.venv/bin/python cheap-worker/tests/test_dedup.py """ import importlib import sys import tempfile from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # → cheap-worker/ import dedup as D # noqa: E402 import worker_service as W # noqa: E402 _GOOD_BUNDLE = "var __GameBundle=(function(){return{bootGameHost(){}}})();" def _iso_registry(): """隔离临时 registry(不污染真 cheap-worker/results/_dedup_registry.jsonl);两个模块引用同一文件。""" p = Path(tempfile.mkdtemp(prefix="dedup-test-")) / "_dedup_registry.jsonl" D.DEDUP_REGISTRY = p W.dedup.DEDUP_REGISTRY = p # worker_service 内的 dedup 同模块,接线测试用同一隔离表 return p def test_first_not_dup_second_same_is_dup(): """首次不撞、第二次同 (title,theme) 撞重 + dupWith 指首款。""" _iso_registry() s1 = D.check_similarity("点击得分小游戏", "generic", "trace-1") assert s1["dupHit"] is False, s1 s2 = D.check_similarity("点击得分小游戏", "generic", "trace-2") assert s2["dupHit"] is True, s2 assert s2["dupWith"] == "trace-1" def test_different_title_not_dup(): """不同 title → 不撞 + sig 不同。""" _iso_registry() a = D.check_similarity("躲避陨石", "generic", "t1") b = D.check_similarity("消除方块", "generic", "t2") assert a["dupHit"] is False and b["dupHit"] is False assert a["sig"] != b["sig"] def test_norm_text_and_compute_sig(): """norm_text 去空白/英文标点+casefold;compute_sig 可复现 + 64-hex。""" assert D.norm_text("点点乐 ") == D.norm_text("点点乐") assert D.norm_text("Hello, World!") == D.norm_text("helloworld") assert D.compute_sig("a", "b", "c") == D.compute_sig("a", "b", "c") assert len(D.compute_sig("a", "b", "c")) == 64 def test_registry_corrupt_tolerance(): """登记表坏行 → 降级不抛(dupHit ∈ {False,None}),不阻断。""" p = _iso_registry() p.parent.mkdir(parents=True, exist_ok=True) p.write_text("not-json\n{bad\n", encoding="utf-8") s = D.check_similarity("某游戏", "generic", "t9") assert s["dupHit"] in (False, None), s def test_registry_path_in_cheap_worker_not_wg1(): """DEDUP_REGISTRY 默认落 cheap-worker/results/、不写 wg1(路径隔离铁律)。""" d = importlib.reload(D) # 重载取默认路径(前面测试改过 DEDUP_REGISTRY) p = str(d.DEDUP_REGISTRY) assert p.endswith("cheap-worker/results/_dedup_registry.jsonl"), p assert "wg1" not in p def test_wiring_similarity_into_trace_and_status_unchanged(): """接线:process_job 把 similarity 塞 trace.similarity;撞重不改 status(仍按九门判)。""" _iso_registry() gd = Path(tempfile.mkdtemp(prefix="dedup-gd-")) / "amgen-d1" (gd / "src").mkdir(parents=True) (gd / "bundle.iife.js").write_text(_GOOD_BUNDLE, encoding="utf-8") captured = [] state = W.WorkerState( run_fn=lambda job: ({"verdict": {"pass": True}, "attempts": 1, "costRmb": 0.02}, gd), send_fn=lambda url, payload, secret: captured.append(payload) or (200, ""), profile_fn=lambda j, g: None, ) job1 = {"job_id": "d1", "traceId": "d1", "templateId": "generic", "brief": "撞重点点乐", "callback": {"target": "http://cb"}} job2 = dict(job1, job_id="d2", traceId="d2") # 同 brief → 第二次撞重 W.process_job(state, job1) W.process_job(state, job2) sim1 = captured[0]["trace"]["similarity"] sim2 = captured[1]["trace"]["similarity"] assert sim1["dupHit"] is False, sim1 assert sim2["dupHit"] is True, sim2 # 同 brief 撞重 assert captured[1]["status"] == "succeeded" # 撞重不阻断、不改 status(仍按九门判) assert "__GameBundle" in captured[1]["engineBundle"] if __name__ == "__main__": _fns = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] _failed = 0 for _fn in _fns: try: _fn() print(f" PASS {_fn.__name__}") except Exception as e: # noqa: BLE001 _failed += 1 print(f" FAIL {_fn.__name__}: {type(e).__name__}: {e}") print(f"\n{len(_fns) - _failed}/{len(_fns)} passed") sys.exit(1 if _failed else 0)