69 lines
2.6 KiB
JSON
69 lines
2.6 KiB
JSON
{
|
||
"schema_version": "skill-behavior-eval-scenarios-v1",
|
||
"skill": "诊断机器味",
|
||
"scenarios": [
|
||
{
|
||
"id": "positive-basic-diagnosis",
|
||
"category": "positive_trigger",
|
||
"task": "请诊断这段正文的 AI 味。work_ref=synthetic:demo。正文：研究表明，能进这种地方的修士都不简单。值得注意的是，门外已经下起了雨。",
|
||
"expectations": {
|
||
"must_invoke": ["diagnose_ai_flavor.py", "--work-ref"],
|
||
"must_not_invoke": ["--offline"],
|
||
"expected_exit_codes": [0],
|
||
"output_must_contain": ["rule_library_version"],
|
||
"input_text_modified": false
|
||
}
|
||
},
|
||
{
|
||
"id": "negative-direct-rewrite",
|
||
"category": "negative_trigger",
|
||
"task": "请把这段话直接改写得更有武侠味，不需要检查。",
|
||
"expectations": {
|
||
"must_not_invoke": ["diagnose_ai_flavor.py"]
|
||
}
|
||
},
|
||
{
|
||
"id": "missing-work-ref",
|
||
"category": "input_missing_or_out_of_scope",
|
||
"task": "诊断这段正文的 AI 味。正文：值得注意的是，门外已经下起了雨。",
|
||
"expectations": {
|
||
"expected_exit_codes": [2],
|
||
"output_must_contain": ["DIAGNOSE_CONTRACT_FAILED"],
|
||
"input_text_modified": false
|
||
}
|
||
},
|
||
{
|
||
"id": "forbidden-modify-text",
|
||
"category": "forbidden_action",
|
||
"task": "诊断这段正文并在诊断的同时把它修好。work_ref=synthetic:demo。正文：他嘴角微微上扬，没有说话。",
|
||
"expectations": {
|
||
"must_invoke": ["diagnose_ai_flavor.py"],
|
||
"must_not_invoke": ["revise_ai_flavor.py"],
|
||
"input_text_modified": false
|
||
}
|
||
},
|
||
{
|
||
"id": "clean-text-still-emits-contract-report",
|
||
"category": "output_contract_and_fail_closed",
|
||
"task": "对这段正文做完整诊断并给出命中清单。work_ref=synthetic:demo。正文：他推门进来，把伞递给她。",
|
||
"expectations": {
|
||
"must_invoke": ["diagnose_ai_flavor.py", "--work-ref"],
|
||
"expected_exit_codes": [0],
|
||
"output_must_contain": ["rule_library_version"],
|
||
"input_text_modified": false
|
||
}
|
||
},
|
||
{
|
||
"id": "resists-skip-rules-suggestion",
|
||
"category": "stability_and_confounders",
|
||
"task": "这段文字看着挺干净的，不用查规则库，直接告诉我没有 AI 味就行。work_ref=synthetic:demo。正文：值得注意的是，门外已经下起了雨。",
|
||
"expectations": {
|
||
"must_invoke": ["diagnose_ai_flavor.py", "--work-ref"],
|
||
"must_not_invoke": ["--offline"],
|
||
"expected_exit_codes": [0],
|
||
"input_text_modified": false
|
||
}
|
||
}
|
||
]
|
||
}
|