fix(gen-qc): cutover 早读三修——并发化 + assert 进展过滤 + M3 thinking 关闭/宽松解析
1) 并发化(gamedef_quickcheck): ThreadPoolExecutor 跨局并行,每局独立 port/cdpPort(4400+i/9300+i)
+ flush 日志;局内 repair 链恒串行→聚合过门率与串行等价。
2) assert 进展过滤(SaaPrompts.extractGatespec 产线 + quickcheck 镜像): 有进展断言时只留进展断言
(increased/decreased/changed/比较),丢终局态断言(phase/result==gameover/win、误编码 score=="gameover")
——bot 限时常到不了终局但 score 已涨=真有进展(tictactoe 0→9/saolei 0→12 假阴修正);无进展断言则全留
(规避类 result==win 为唯一信号)。终局完成度另由 expectLatch 把关。
3) M3 协议修(_client._extra_body + extract_json_obj): MiniMax-M3 OpenAI 端点默认 thinking=adaptive、
<think> 内联进 content→复杂局飙 token→max_tokens 截断丢 JSON(假"parse 失败")。官方关法 thinking=
{type:disabled}(按 model gate MiniMax);严格 json.loads 失败回退 json_repair(镜像后端 looseParse)+
<think> 剥离兜底。
实证: M3 parse 6/13→2/13、过门 2→3/13,与 deepseek(4/13)同档=M3 不弱(旧 2/13 系截断伪象)。
SaaPrompts 经 mvn 反应堆 BUILD SUCCESS;Python 经 py_compile+单测+真跑。快走查口径;权威 cutover≥60% 在 mini-desktop。
产线 follow-up(6c6g): Java M3 fallback 路同款 thinking 截断潜伏 bug,须补 thinking:disabled。
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
81c1b2cfa9
commit
d5d3d09b0f
@ -477,6 +477,31 @@ final class SaaPrompts {
|
||||
if (on.has("assertAfterPlay") && !on.get("assertAfterPlay").isArray()) {
|
||||
on.set("assertAfterPlay", MAPPER.createArrayNode());
|
||||
}
|
||||
// 净化 assertAfterPlay:终局态断言不应否决「进展门」H_progress。只要存在≥1 条进展断言(op∈
|
||||
// increased/decreased/changed/比较),就只保留进展断言、丢弃所有非进展(==/in/!=…)的终局态断言——
|
||||
// bot 限时真玩常到不了 gameover/win,但 score 已涨=确有进展(如井字棋 score 0→9 却被 phase=="gameover"
|
||||
// 或误编码 score=="gameover" 误判红线)。无任何进展断言时(如规避类只给 result=="win")全保留——终局即其唯一
|
||||
// 进展信号。终局完成度另由 expectLatch 把关。lili-mac 快走查 extract_gatespec 同步镜像此过滤(drift-free)。
|
||||
if (on.has("assertAfterPlay") && on.get("assertAfterPlay").isArray()) {
|
||||
com.fasterxml.jackson.databind.node.ArrayNode src =
|
||||
(com.fasterxml.jackson.databind.node.ArrayNode) on.get("assertAfterPlay");
|
||||
boolean hasProgress = false;
|
||||
for (JsonNode a : src) {
|
||||
if (a != null && a.isObject() && isProgressOp(a.path("op").asText(""))) {
|
||||
hasProgress = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (hasProgress) {
|
||||
com.fasterxml.jackson.databind.node.ArrayNode kept = MAPPER.createArrayNode();
|
||||
for (JsonNode a : src) {
|
||||
if (a != null && a.isObject() && isProgressOp(a.path("op").asText(""))) {
|
||||
kept.add(a);
|
||||
}
|
||||
}
|
||||
on.set("assertAfterPlay", kept);
|
||||
}
|
||||
}
|
||||
// expectLatch 缺省补 true(官方 latch 契约恒需,对齐 studio.py:95)。
|
||||
if (!on.has("expectLatch")) {
|
||||
on.put("expectLatch", true);
|
||||
@ -494,6 +519,22 @@ final class SaaPrompts {
|
||||
return on;
|
||||
}
|
||||
|
||||
/** 进展类 op(度量在动):用于「有进展断言则只留进展断言」过滤,把终局态(==/in/!=)断言挡在 H_progress 之外(详见 extractGatespec 注释)。 */
|
||||
private static boolean isProgressOp(String op) {
|
||||
switch (op) {
|
||||
case "increased":
|
||||
case "decreased":
|
||||
case "changed":
|
||||
case ">":
|
||||
case ">=":
|
||||
case "<":
|
||||
case "<=":
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 宽松 JSON 解析(替代 Python json_repair,A-5 加强):
|
||||
* ① 严格 Jackson;② 截最外层 {...} 去前后说明文字 + 去尾随逗号后再严格;
|
||||
|
||||
@ -33,6 +33,10 @@ STAGE1_MODEL = os.environ.get("QC_STAGE1", "deepseek-v4-flash")
|
||||
STAGE2_MODEL = os.environ.get("QC_STAGE2", "deepseek-v4-pro")
|
||||
STAGE1_REPAIRS = int(os.environ.get("QC_STAGE1_REPAIRS", "5")) # 真图 maxRepairs 默认 5
|
||||
STAGE2_EXTRA = int(os.environ.get("QC_STAGE2_EXTRA", "3")) # 真图 stage2ExtraRepairs 默认 3
|
||||
# 跨局并发数:仅跨不同游戏并行(局内 repair 链恒串行——第 N+1 次需第 N 次失败回喂);
|
||||
# 每局独立 (port,cdpPort) 端口对(serve/Chrome/user-data-dir 全隔离,见 serve-and-play.sh)→互不撞;
|
||||
# 每局过门判定不变→聚合过门率与串行等价。authoritative ≥60% cutover 仍在 mini-desktop。
|
||||
CONCURRENCY = int(os.environ.get("QC_CONCURRENCY", "4"))
|
||||
|
||||
|
||||
def extract_block(name):
|
||||
@ -56,14 +60,24 @@ def extract_gatespec(design_text):
|
||||
if "assertAfterPlay" in gs and not isinstance(gs["assertAfterPlay"], list):
|
||||
gs["assertAfterPlay"] = []
|
||||
gs.setdefault("expectLatch", True)
|
||||
# gamedef 取证只暴露 score/remaining/progress/实体位置——把 assertAfterPlay 的顶层自定义标量字段(moves/totalRound 等)强制到 score
|
||||
# (设计 agent 常造取证读不到的字段→H_progress 必挂;不动含 '.' 的实体位置断言如 ball.x)。
|
||||
EXPOSED = ("score", "remaining", "progress")
|
||||
for a in (gs.get("assertAfterPlay") or []):
|
||||
if isinstance(a, dict) and isinstance(a.get("path"), str):
|
||||
# assertAfterPlay 处理两步:① 有进展断言则只留进展断言(挡终局态断言);② 取证读不到的自定义标量归一到 score。
|
||||
PROGRESS_OPS = ("increased", "decreased", "changed", ">", ">=", "<", "<=")
|
||||
KNOWN = ("score", "remaining", "progress", "result", "phase") # 取证暴露的合法顶层字段:不强转 score(result/phase 是真字段)
|
||||
|
||||
# ① 终局态断言(phase/result=="gameover"/"win"、result in[...]、误编码 score=="gameover" 等 op 非进展类)不应否决
|
||||
# 「进展门」H_progress:只要≥1 条进展断言(op∈PROGRESS_OPS)存在,就只留进展断言、丢弃所有非进展(==/in/!=…)断言——
|
||||
# bot 限时真玩常到不了终局,但 score 已涨=确有进展(井字棋 0→9 实证)。无任何进展断言(如规避类只给 result=="win")
|
||||
# 则全留(终局即唯一进展信号)。镜像后端 SaaPrompts.isProgressOp 过滤(drift-free);终局完成度另由 expectLatch 把关。
|
||||
aap = [a for a in (gs.get("assertAfterPlay") or []) if isinstance(a, dict)]
|
||||
if any(a.get("op") in PROGRESS_OPS for a in aap):
|
||||
aap = [a for a in aap if a.get("op") in PROGRESS_OPS]
|
||||
gs["assertAfterPlay"] = aap
|
||||
# ② 顶层自定义标量字段(moves/totalRound 取证读不到)强制到 score;result/phase 等合法字段与含 '.' 的实体位置断言(ball.x)不动。
|
||||
for a in aap:
|
||||
if isinstance(a.get("path"), str):
|
||||
p = a["path"].lstrip("/")
|
||||
top = p.split(".")[0].split("[")[0]
|
||||
if "." not in p and top not in EXPOSED:
|
||||
if "." not in p and top not in KNOWN:
|
||||
a["path"] = "score"
|
||||
a["op"] = a.get("op", "increased")
|
||||
drv = gs.get("driver")
|
||||
@ -75,15 +89,25 @@ def extract_gatespec(design_text):
|
||||
|
||||
|
||||
def extract_json_obj(content):
|
||||
"""从模型输出抠最外层 {...}(去 ```json 围栏/前后说明)。"""
|
||||
s = content.strip()
|
||||
"""从模型输出抠最外层 {...}(先剥 <think> 推理块、再去 ```json 围栏/前后说明)。"""
|
||||
# 推理模型(M3)即便关了 thinking 偶仍内联 <think>…</think>;先整块剥除,避免抽到思维链里的花括号。
|
||||
s = re.sub(r"<think>.*?</think>", "", content or "", flags=re.DOTALL).strip()
|
||||
s = re.sub(r"^```(?:json)?\s*", "", s)
|
||||
s = re.sub(r"\s*```$", "", s)
|
||||
lo, hi = s.find("{"), s.rfind("}")
|
||||
if lo < 0 or hi <= lo:
|
||||
return None
|
||||
body = s[lo:hi + 1]
|
||||
try:
|
||||
return json.loads(s[lo:hi + 1])
|
||||
return json.loads(body) # ① 严格优先
|
||||
except Exception:
|
||||
pass
|
||||
# ② 宽松回退:便宜模型偶产轻微非法 JSON(如多余引号 "h":130"),json_repair 修复——镜像后端 looseParse 宽松级,
|
||||
# 跑的就是产线真解析口径(drift-free);避免快走查因严格解析假性 parse 失败、低估真过门率。
|
||||
try:
|
||||
import json_repair
|
||||
obj = json_repair.loads(body)
|
||||
return obj if isinstance(obj, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@ -101,7 +125,7 @@ def assemble_via_cli(game_id, gamedef_obj):
|
||||
return r.returncode == 0, (r.stdout + r.stderr).strip()
|
||||
|
||||
|
||||
def run_one(game_id, gamedef_system, design_system):
|
||||
def run_one(game_id, gamedef_system, design_system, port=4320, cdp_port=9222):
|
||||
brief_text, play_spec = run._load_brief(game_id)
|
||||
gid = "gd-" + game_id
|
||||
out = {"game_id": gid, "stage": None, "pass": False, "guards": {}, "err": "", "driver": None, "model": STAGE1_MODEL}
|
||||
@ -146,7 +170,7 @@ def run_one(game_id, gamedef_system, design_system):
|
||||
bok, blog = run.build(gid)
|
||||
if not bok:
|
||||
out["stage"] = "build"; feedback = "打包失败(esbuild):\n" + blog[:500]; continue
|
||||
rc, _, verdict = run.play(gid)
|
||||
rc, _, verdict = run.play(gid, port=port, cdp_port=cdp_port)
|
||||
out["stage"] = "play"
|
||||
out["guards"] = {k: g.get("pass") for k, g in ((verdict or {}).get("guards") or {}).items()}
|
||||
if rc == 0 and verdict and verdict.get("pass"):
|
||||
@ -157,23 +181,40 @@ def run_one(game_id, gamedef_system, design_system):
|
||||
return out
|
||||
|
||||
|
||||
def _fmt_result(r):
|
||||
"""单局结果行(含 game_id,并发完成序打印不串)。"""
|
||||
mark = "✅ PASS" if r["pass"] else f"❌ {r['stage']}"
|
||||
gpass = sum(1 for v in (r.get("guards") or {}).values() if v)
|
||||
gtot = len(r.get("guards") or {})
|
||||
drv = r.get("driver") or "-"
|
||||
att = r.get("attempts", 1)
|
||||
return f" {r['game_id']:14s} {mark:16s} drv={str(drv):14s} att={att} gates={gpass}/{gtot} {('' if r['pass'] else r.get('err','')[:100])}"
|
||||
|
||||
|
||||
def main():
|
||||
import concurrent.futures
|
||||
games = sys.argv[1:] or ["runner", "flappy", "whack", "pong"]
|
||||
gds = extract_block("GAMEDEF_SYSTEM")
|
||||
dsys = extract_block("DESIGN_SYSTEM")
|
||||
print(f"[gamedef-qc] stage1={STAGE1_MODEL}({STAGE1_REPAIRS})→stage2={STAGE2_MODEL}({STAGE2_EXTRA}) games={games} (全图镜像+救场升档)\n")
|
||||
conc = max(1, CONCURRENCY)
|
||||
# flush=True:stdout 重定向到文件时 Python 默认块缓冲,加 flush 让后台日志逐局可读(便于进度观察)。
|
||||
print(f"[gamedef-qc] stage1={STAGE1_MODEL}({STAGE1_REPAIRS})→stage2={STAGE2_MODEL}({STAGE2_EXTRA}) games={games} 并发={conc} (全图镜像+救场升档)\n", flush=True)
|
||||
# 跨局并发:每局分到独立端口对(port=4400+i / cdpPort=9300+i,与串行默认 4320/9222 错开,逐局唯一→任意调度都不撞)。
|
||||
results = []
|
||||
for g in games:
|
||||
r = run_one(g, gds, dsys)
|
||||
results.append(r)
|
||||
mark = "✅ PASS" if r["pass"] else f"❌ {r['stage']}"
|
||||
gpass = sum(1 for v in r["guards"].values() if v)
|
||||
gtot = len(r["guards"])
|
||||
drv = r.get("driver") or "-"
|
||||
att = r.get("attempts", 1)
|
||||
print(f" {r['game_id']:14s} {mark:16s} drv={drv:14s} att={att} gates={gpass}/{gtot} {('' if r['pass'] else r['err'][:100])}")
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=conc) as ex:
|
||||
futs = {ex.submit(run_one, g, gds, dsys, 4400 + i, 9300 + i): g for i, g in enumerate(games)}
|
||||
for fut in concurrent.futures.as_completed(futs):
|
||||
try:
|
||||
r = fut.result()
|
||||
except Exception as e: # harness 级异常兜底,不让一局炸掉整轮
|
||||
r = {"game_id": "gd-" + futs[fut], "stage": "harness", "pass": False, "guards": {}, "err": "harness 异常:" + str(e)[:160]}
|
||||
results.append(r)
|
||||
print(_fmt_result(r), flush=True)
|
||||
# as_completed 是完成序,按输入顺序重排再落盘/汇总。
|
||||
order = {("gd-" + g): i for i, g in enumerate(games)}
|
||||
results.sort(key=lambda r: order.get(r.get("game_id"), 999))
|
||||
npass = sum(1 for r in results if r["pass"])
|
||||
print(f"\n[gamedef-qc] 过门 {npass}/{len(results)} (cutover 门基线=iife 60%; 本为 lili-mac 小样早读, authoritative=mini-desktop)")
|
||||
print(f"\n[gamedef-qc] 过门 {npass}/{len(results)} (cutover 门基线=iife 60%; 本为 lili-mac 小样早读, authoritative=mini-desktop)", flush=True)
|
||||
Path(__file__).resolve().parent.joinpath("results", "gamedef-quickcheck.json").write_text(
|
||||
json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
sys.exit(0)
|
||||
|
||||
@ -57,6 +57,13 @@ def get_client():
|
||||
return openai.OpenAI(api_key=get_api_key(), base_url=BASE_URL, max_retries=2, timeout=180.0)
|
||||
|
||||
|
||||
def _extra_body(model, enable_thinking):
|
||||
"""按模型构造 extra_body:MiniMax 系走官方 thinking 控制(默认 disabled,关思维链防截断);其他模型不带厂商专有参。"""
|
||||
if "minimax" in (model or "").lower():
|
||||
return {"thinking": {"type": "adaptive" if enable_thinking else "disabled"}}
|
||||
return {}
|
||||
|
||||
|
||||
def _cache_hit_from_usage(usage):
|
||||
"""从 openai 原始 usage 抽命中缓存的 prompt token(兼容两套字段取 max,U4/B9)。
|
||||
|
||||
@ -80,7 +87,7 @@ def _cache_hit_from_usage(usage):
|
||||
return 0
|
||||
|
||||
|
||||
def chat(model, system, user, max_tokens=16000, temperature=0.0, tries=3, retry_delay=2.0):
|
||||
def chat(model, system, user, max_tokens=16000, temperature=0.0, tries=3, retry_delay=2.0, enable_thinking=False):
|
||||
"""单次裸调用 chat.completions。确定性产出 temperature=0;瞬时网关错误(502 burst)重试。
|
||||
|
||||
返回 dict:{content, prompt_tokens, completion_tokens, prompt_cache_hit_tokens, total_tokens, model, wall_s, id, finish_reason}。
|
||||
@ -101,6 +108,11 @@ def chat(model, system, user, max_tokens=16000, temperature=0.0, tries=3, retry_
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
stream=False,
|
||||
# 关推理(默认):MiniMax-M3 在 OpenAI 端点默认 thinking=adaptive(开),思维链内联进 content,复杂局
|
||||
# 飙 token → max_tokens 处截断、永远到不了 JSON(finish_reason=length,表现为"parse 失败",实测
|
||||
# match3/simon)。官方关法(MiniMax docs)= 顶层 thinking={type:disabled};本 worker 只要结构化 JSON、
|
||||
# 不需 CoT,故对 MiniMax 系默认关(enable_thinking=True 可覆盖)。非 MiniMax 模型不带此厂商专有参。
|
||||
extra_body=_extra_body(model, enable_thinking),
|
||||
)
|
||||
wall = time.perf_counter() - t0
|
||||
choice = resp.choices[0]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user