502 lines
18 KiB
Python
502 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""refresh_runtime_probe 的 provider-neutral 纯离线测试。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from collections.abc import Mapping
|
|
from typing import Any
|
|
from unittest import mock
|
|
|
|
PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
SKILLS_DIR = PROJECT_ROOT / ".agent" / "skills"
|
|
SCRIPT_DIR = PROJECT_ROOT / "muse" / "platform" / "llm" / "skills" / "验证角色运行能力" / "scripts"
|
|
WRITER_REPLAY_DIR = PROJECT_ROOT / "muse" / "lifecycle" / "quality" / "skills" / "replay" / "回放评估正文质量" / "scripts"
|
|
GATE_ADJUDICATION_DIR = PROJECT_ROOT / "muse" / "lifecycle" / "quality" / "skills" / "mechanical" / "判定质量是否合格" / "scripts"
|
|
for _import_dir in (SCRIPT_DIR, WRITER_REPLAY_DIR, GATE_ADJUDICATION_DIR):
|
|
if str(_import_dir) not in sys.path:
|
|
sys.path.insert(0, str(_import_dir))
|
|
|
|
import refresh_runtime_probe as refresh_module # noqa: E402
|
|
from gate_input_builder import ( # noqa: E402
|
|
_verify_self_hash,
|
|
canonical_sha256,
|
|
)
|
|
from muse_role import ( # noqa: E402
|
|
HASH_PATTERN,
|
|
MODEL_POLICY_ALIAS,
|
|
MODEL_POLICY_VERSION,
|
|
RUNTIME_ADAPTER,
|
|
RUNTIME_ADAPTER_VERSION,
|
|
RoleExecutionProfile,
|
|
RoleExecutionReceipt,
|
|
RoleInvocationResult,
|
|
RoleRuntimeError,
|
|
sha256_json,
|
|
)
|
|
from refresh_runtime_probe import ( # noqa: E402
|
|
PROBE_BUSINESS_INPUT,
|
|
PROBE_SCHEMA_VERSION,
|
|
ProbeRefreshError,
|
|
build_writer_profile,
|
|
make_dry_run_invoker,
|
|
refresh_runtime_probe,
|
|
verify_probe_result,
|
|
)
|
|
from run_writer_replay import ( # noqa: E402
|
|
WriterReplayProductionAdapters,
|
|
_validate_execute_authorization,
|
|
profile_from_mapping,
|
|
)
|
|
|
|
CONFIG_PATH = WRITER_REPLAY_DIR.parent / "configs" / "writer-gate-a-deep-space-v1-cn-skills.json"
|
|
CHECKED_AT = "2026-08-21T00:00:00+00:00"
|
|
EXPECTED_PROBE_FIELDS = {
|
|
"schemaVersion",
|
|
"status",
|
|
"checkedAt",
|
|
"runtimeAdapter",
|
|
"runtimeAdapterVersion",
|
|
"modelPolicyVersion",
|
|
"role",
|
|
"profileVersion",
|
|
"modelAlias",
|
|
"resolvedModelId",
|
|
"executionProfileSha256",
|
|
"jsonSchemaId",
|
|
"jsonSchemaSha256",
|
|
"systemPromptId",
|
|
"systemPromptSha256",
|
|
"inputSha256",
|
|
"requestedModelId",
|
|
"actualModelId",
|
|
"modelMatch",
|
|
"executionReceiptSha256",
|
|
"structuredOutputSha256",
|
|
"terminalReason",
|
|
"totalCostUsd",
|
|
"receiptSha256",
|
|
}
|
|
|
|
|
|
def _load_base_config() -> dict[str, Any]:
|
|
return json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
|
|
|
|
def _current_writer_identity_hash(config: Mapping[str, Any]) -> str:
|
|
return build_writer_profile(config).execution_profile_sha256
|
|
|
|
|
|
def _make_stale_profile_hash(current_hash: str) -> str:
|
|
prefix, _, hex_part = current_hash.partition(":")
|
|
flipped_last = "0" if hex_part[-1] != "0" else "1"
|
|
return f"{prefix}:{hex_part[:-1]}{flipped_last}"
|
|
|
|
|
|
def _build_result(
|
|
profile: RoleExecutionProfile,
|
|
*,
|
|
candidate_body: str = "黎明前，守塔人划亮火柴，灯芯燃起一小团光。",
|
|
total_cost: str | None = "0.003000",
|
|
model_match: bool = True,
|
|
requested_model_id: str | None = None,
|
|
actual_model_id: str | None = None,
|
|
is_error: bool = False,
|
|
exit_code: int | None = None,
|
|
terminal_reason: str = "completed",
|
|
api_error_status: Any = None,
|
|
structured_output: Mapping[str, Any] | None = None,
|
|
structured_output_sha256: str | None = None,
|
|
execution_profile_sha256: str | None = None,
|
|
input_sha256: str | None = None,
|
|
json_schema_sha256: str | None = None,
|
|
) -> RoleInvocationResult:
|
|
if structured_output is None:
|
|
structured_output = {"candidateBody": candidate_body}
|
|
if structured_output_sha256 is None:
|
|
structured_output_sha256 = sha256_json(structured_output)
|
|
actual_model_id = actual_model_id or profile.resolved_model_id
|
|
receipt = RoleExecutionReceipt(
|
|
adapter_role=profile.adapter_role,
|
|
invocation_id="test-probe",
|
|
execution_profile_sha256=(
|
|
execution_profile_sha256 or profile.execution_profile_sha256
|
|
),
|
|
requested_model_id=requested_model_id or profile.model_alias,
|
|
actual_model_id=actual_model_id,
|
|
model_match=model_match,
|
|
effort="governed",
|
|
max_budget_usd_per_call=format(profile.max_budget_usd_per_call, "f"),
|
|
total_cost_usd=total_cost,
|
|
usage={"prompt_tokens": 1, "completion_tokens": 1},
|
|
model_usage=(
|
|
{actual_model_id: {"costUSD": total_cost}}
|
|
if total_cost is not None
|
|
else None
|
|
),
|
|
stop_reason="stop" if not is_error else None,
|
|
terminal_reason=terminal_reason,
|
|
is_error=is_error,
|
|
api_error_status=api_error_status,
|
|
exit_code=exit_code,
|
|
duration_ms=1,
|
|
input_sha256=input_sha256 or sha256_json(PROBE_BUSINESS_INPUT),
|
|
structured_output_sha256=structured_output_sha256,
|
|
json_schema_sha256=json_schema_sha256 or profile.json_schema_sha256,
|
|
)
|
|
return RoleInvocationResult(
|
|
structured_output=structured_output,
|
|
receipt=receipt,
|
|
)
|
|
|
|
|
|
def _invoker_factory(**overrides: Any):
|
|
def _invoker(
|
|
profile: RoleExecutionProfile,
|
|
_business_input: Mapping[str, Any],
|
|
) -> RoleInvocationResult:
|
|
return _build_result(profile, **overrides)
|
|
|
|
return _invoker
|
|
|
|
|
|
def _make_stale_config(config: Mapping[str, Any]) -> tuple[dict[str, Any], str]:
|
|
current_hash = _current_writer_identity_hash(config)
|
|
valid = refresh_runtime_probe(
|
|
config,
|
|
invoker=_invoker_factory(),
|
|
checked_at=CHECKED_AT,
|
|
).refreshed_config
|
|
stale_hash = _make_stale_profile_hash(current_hash)
|
|
probe = valid["executionAuthorization"]["runtimeProbe"]
|
|
probe["executionProfileSha256"] = stale_hash
|
|
probe["receiptSha256"] = canonical_sha256(
|
|
{key: value for key, value in probe.items() if key != "receiptSha256"}
|
|
)
|
|
return valid, stale_hash
|
|
|
|
|
|
def _adapters_from_config(config: Mapping[str, Any]) -> WriterReplayProductionAdapters:
|
|
profiles = config["executionProfiles"]
|
|
return WriterReplayProductionAdapters(
|
|
writer_runner=None,
|
|
writer_profile=profile_from_mapping(profiles["writer"], role="writer"),
|
|
semantic_model_runner=object(),
|
|
semantic_profile=profile_from_mapping(
|
|
profiles["semantic_detector"], role="semantic_detector"
|
|
),
|
|
judge_model_runner=object(),
|
|
judge_profile=profile_from_mapping(
|
|
profiles["blind_judge"], role="blind_judge"
|
|
),
|
|
oracle_truth_packs={},
|
|
)
|
|
|
|
|
|
class ProbeSuccessPathTest(unittest.TestCase):
|
|
def test_probe_and_refreshed_config_self_hash_pass_gate_verifier(self):
|
|
config = _load_base_config()
|
|
result = refresh_runtime_probe(
|
|
config,
|
|
invoker=_invoker_factory(),
|
|
checked_at=CHECKED_AT,
|
|
)
|
|
_verify_self_hash(result.probe, "receiptSha256", "runtimeProbe")
|
|
refreshed = result.refreshed_config["executionAuthorization"]["runtimeProbe"]
|
|
_verify_self_hash(refreshed, "receiptSha256", "runtimeProbe")
|
|
self.assertEqual(refreshed, result.probe)
|
|
|
|
def test_stale_probe_replaced_by_current_profile_identity(self):
|
|
config = _load_base_config()
|
|
current_hash = _current_writer_identity_hash(config)
|
|
stale_config, stale_hash = _make_stale_config(config)
|
|
result = refresh_runtime_probe(
|
|
stale_config,
|
|
invoker=_invoker_factory(),
|
|
checked_at=CHECKED_AT,
|
|
)
|
|
self.assertNotEqual(stale_hash, current_hash)
|
|
self.assertEqual(result.probe["executionProfileSha256"], current_hash)
|
|
self.assertEqual(
|
|
result.probe["executionProfileSha256"],
|
|
config["executionAuthorization"]["profileSha256"]["writer"],
|
|
)
|
|
|
|
def test_probe_field_set_and_runtime_bindings_are_exact(self):
|
|
result = refresh_runtime_probe(
|
|
_load_base_config(),
|
|
invoker=_invoker_factory(),
|
|
checked_at=CHECKED_AT,
|
|
)
|
|
probe = result.probe
|
|
self.assertEqual(set(probe), EXPECTED_PROBE_FIELDS)
|
|
self.assertEqual(probe["schemaVersion"], PROBE_SCHEMA_VERSION)
|
|
self.assertEqual(probe["runtimeAdapter"], RUNTIME_ADAPTER)
|
|
self.assertEqual(probe["runtimeAdapterVersion"], RUNTIME_ADAPTER_VERSION)
|
|
self.assertEqual(probe["modelPolicyVersion"], MODEL_POLICY_VERSION)
|
|
self.assertEqual(probe["modelAlias"], MODEL_POLICY_ALIAS)
|
|
self.assertTrue(HASH_PATTERN.fullmatch(probe["structuredOutputSha256"]))
|
|
|
|
def test_refresh_does_not_mutate_input_config(self):
|
|
config = _load_base_config()
|
|
before = json.dumps(config, ensure_ascii=False, sort_keys=True)
|
|
refresh_runtime_probe(
|
|
config,
|
|
invoker=_invoker_factory(),
|
|
checked_at=CHECKED_AT,
|
|
)
|
|
self.assertEqual(json.dumps(config, ensure_ascii=False, sort_keys=True), before)
|
|
|
|
def test_summary_excludes_prompt_response_and_paths(self):
|
|
result = refresh_runtime_probe(
|
|
_load_base_config(),
|
|
invoker=_invoker_factory(),
|
|
checked_at=CHECKED_AT,
|
|
)
|
|
rendered = json.dumps(result.summary, ensure_ascii=False)
|
|
for forbidden in ("candidateBody", "systemPrompt", "rawResponse", "rawPath"):
|
|
self.assertNotIn(forbidden, rendered)
|
|
|
|
|
|
class ProbeGateIntegrationTest(unittest.TestCase):
|
|
def test_refreshed_config_passes_execute_authorization_gate(self):
|
|
result = refresh_runtime_probe(
|
|
_load_base_config(),
|
|
invoker=_invoker_factory(),
|
|
checked_at=CHECKED_AT,
|
|
)
|
|
adapters = _adapters_from_config(result.refreshed_config)
|
|
gate_result, blocked_status, blocked_code = _validate_execute_authorization(
|
|
result.refreshed_config,
|
|
adapters,
|
|
)
|
|
self.assertIsNone(blocked_status)
|
|
self.assertIsNone(blocked_code)
|
|
self.assertIsNotNone(gate_result)
|
|
|
|
def test_stale_probe_is_blocked_at_contract_binding(self):
|
|
stale_config, _stale_hash = _make_stale_config(_load_base_config())
|
|
gate_result, blocked_status, blocked_code = _validate_execute_authorization(
|
|
stale_config,
|
|
_adapters_from_config(stale_config),
|
|
)
|
|
self.assertIsNone(gate_result)
|
|
self.assertEqual(blocked_status, "blocked_execute_probe_contract")
|
|
self.assertEqual(blocked_code, "EXECUTE_PROBE_CONTRACT_MISMATCH")
|
|
|
|
def test_budget_and_raw_authorizations_are_unchanged(self):
|
|
config = _load_base_config()
|
|
result = refresh_runtime_probe(
|
|
config,
|
|
invoker=_invoker_factory(),
|
|
checked_at=CHECKED_AT,
|
|
)
|
|
for name in ("budget", "rawRetention"):
|
|
self.assertEqual(
|
|
result.refreshed_config["executionAuthorization"][name],
|
|
config["executionAuthorization"][name],
|
|
)
|
|
|
|
|
|
class ProbeFailClosedTest(unittest.TestCase):
|
|
def _assert_fail_closed(self, **overrides: Any) -> ProbeRefreshError:
|
|
with self.assertRaises(ProbeRefreshError) as caught:
|
|
refresh_runtime_probe(
|
|
_load_base_config(),
|
|
invoker=_invoker_factory(**overrides),
|
|
checked_at=CHECKED_AT,
|
|
)
|
|
return caught.exception
|
|
|
|
def test_schema_invalid_empty_body_fails_closed(self):
|
|
self.assertEqual(
|
|
self._assert_fail_closed(structured_output={"candidateBody": ""}).code,
|
|
"PROBE_SCHEMA_INVALID",
|
|
)
|
|
|
|
def test_schema_invalid_extra_field_fails_closed(self):
|
|
self.assertEqual(
|
|
self._assert_fail_closed(
|
|
structured_output={"candidateBody": "正文", "runId": "leak"}
|
|
).code,
|
|
"PROBE_SCHEMA_INVALID",
|
|
)
|
|
|
|
def test_over_budget_and_missing_cost_fail_closed(self):
|
|
self.assertEqual(
|
|
self._assert_fail_closed(total_cost="9.000000").code,
|
|
"PROBE_BUDGET_EXCEEDED",
|
|
)
|
|
self.assertEqual(
|
|
self._assert_fail_closed(total_cost=None).code,
|
|
"PROBE_BUDGET_EXCEEDED",
|
|
)
|
|
|
|
def test_model_policy_mismatch_fails_closed(self):
|
|
self.assertEqual(
|
|
self._assert_fail_closed(model_match=False).code,
|
|
"PROBE_MODEL_MISMATCH",
|
|
)
|
|
self.assertEqual(
|
|
self._assert_fail_closed(actual_model_id="unapproved-model").code,
|
|
"PROBE_MODEL_MISMATCH",
|
|
)
|
|
self.assertEqual(
|
|
self._assert_fail_closed(requested_model_id="direct-model").code,
|
|
"PROBE_MODEL_MISMATCH",
|
|
)
|
|
|
|
def test_receipt_error_and_transport_fields_fail_closed(self):
|
|
self.assertEqual(
|
|
self._assert_fail_closed(is_error=True).code,
|
|
"PROBE_RECEIPT_ERROR",
|
|
)
|
|
self.assertEqual(
|
|
self._assert_fail_closed(exit_code=1).code,
|
|
"PROBE_TRANSPORT_INVALID",
|
|
)
|
|
self.assertEqual(
|
|
self._assert_fail_closed(api_error_status=529).code,
|
|
"PROBE_API_ERROR",
|
|
)
|
|
|
|
def test_terminal_reason_fails_closed(self):
|
|
self.assertEqual(
|
|
self._assert_fail_closed(terminal_reason="failed").code,
|
|
"PROBE_TERMINAL_REASON_INVALID",
|
|
)
|
|
|
|
def test_profile_schema_input_and_output_bindings_fail_closed(self):
|
|
current = _current_writer_identity_hash(_load_base_config())
|
|
self.assertEqual(
|
|
self._assert_fail_closed(
|
|
execution_profile_sha256=_make_stale_profile_hash(current)
|
|
).code,
|
|
"PROBE_PROFILE_BINDING_MISMATCH",
|
|
)
|
|
self.assertEqual(
|
|
self._assert_fail_closed(json_schema_sha256="sha256:" + "1" * 64).code,
|
|
"PROBE_SCHEMA_BINDING_MISMATCH",
|
|
)
|
|
self.assertEqual(
|
|
self._assert_fail_closed(input_sha256="sha256:" + "2" * 64).code,
|
|
"PROBE_INPUT_HASH_INVALID",
|
|
)
|
|
self.assertEqual(
|
|
self._assert_fail_closed(
|
|
structured_output_sha256="sha256:" + "3" * 64
|
|
).code,
|
|
"PROBE_OUTPUT_HASH_INVALID",
|
|
)
|
|
|
|
def test_role_runtime_error_propagates_to_cli_boundary(self):
|
|
def raising_invoker(_profile, _business_input):
|
|
raise RoleRuntimeError("WRITER_RUNTIME_FAILED", "受控失败")
|
|
|
|
with self.assertRaises(RoleRuntimeError):
|
|
refresh_runtime_probe(
|
|
_load_base_config(),
|
|
invoker=raising_invoker,
|
|
checked_at=CHECKED_AT,
|
|
)
|
|
|
|
|
|
class ProbeDryRunAndCliTest(unittest.TestCase):
|
|
def test_dry_run_invoker_produces_valid_probe(self):
|
|
result = refresh_runtime_probe(
|
|
_load_base_config(),
|
|
invoker=make_dry_run_invoker(),
|
|
checked_at=CHECKED_AT,
|
|
dry_run=True,
|
|
)
|
|
verify_probe_result(build_writer_profile(_load_base_config()), make_dry_run_invoker()(
|
|
build_writer_profile(_load_base_config()), PROBE_BUSINESS_INPUT
|
|
))
|
|
_verify_self_hash(result.probe, "receiptSha256", "runtimeProbe")
|
|
self.assertEqual(result.probe["status"], "dry_run")
|
|
self.assertTrue(result.summary["dryRun"])
|
|
|
|
def test_probe_input_is_synthetic_and_has_no_real_work_or_raw(self):
|
|
text = json.dumps(PROBE_BUSINESS_INPUT, ensure_ascii=False)
|
|
self.assertIn("运行探针合成任务", text)
|
|
for forbidden in ("林澈", "圣蒂曼", "深空", "raw", "/raw", "vault"):
|
|
self.assertNotIn(forbidden, text)
|
|
self.assertEqual(PROBE_BUSINESS_INPUT["proseExcerpts"], [])
|
|
self.assertEqual(PROBE_BUSINESS_INPUT["factConstraints"], [])
|
|
|
|
def test_cli_dry_run_writes_new_file_and_leaves_source_untouched(self):
|
|
source_before = CONFIG_PATH.read_text(encoding="utf-8")
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
output = pathlib.Path(directory) / "refreshed.json"
|
|
rc = refresh_module.main([
|
|
"--config", str(CONFIG_PATH),
|
|
"--output", str(output),
|
|
"--dry-run",
|
|
"--checked-at", CHECKED_AT,
|
|
])
|
|
self.assertEqual(rc, 0)
|
|
refreshed = json.loads(output.read_text(encoding="utf-8"))
|
|
self.assertEqual(
|
|
refreshed["executionAuthorization"]["runtimeProbe"]["status"],
|
|
"dry_run",
|
|
)
|
|
_verify_self_hash(
|
|
refreshed["executionAuthorization"]["runtimeProbe"],
|
|
"receiptSha256",
|
|
"runtimeProbe",
|
|
)
|
|
self.assertEqual(CONFIG_PATH.read_text(encoding="utf-8"), source_before)
|
|
|
|
def test_cli_refuses_in_place_output(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
config_path = pathlib.Path(directory) / "config.json"
|
|
config_path.write_text(CONFIG_PATH.read_text(encoding="utf-8"), encoding="utf-8")
|
|
self.assertEqual(
|
|
refresh_module.main([
|
|
"--config", str(config_path),
|
|
"--output", str(config_path),
|
|
"--dry-run",
|
|
]),
|
|
2,
|
|
)
|
|
|
|
def test_cli_runtime_failure_returns_nonzero_and_writes_nothing(self):
|
|
def raising_invoker(_profile, _business_input, **_kwargs):
|
|
raise RoleRuntimeError("WRITER_RUNTIME_FAILED", "受控失败")
|
|
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
output = pathlib.Path(directory) / "refreshed.json"
|
|
with mock.patch.object(refresh_module, "run_role", raising_invoker):
|
|
rc = refresh_module.main([
|
|
"--config", str(CONFIG_PATH),
|
|
"--output", str(output),
|
|
])
|
|
self.assertEqual(rc, 1)
|
|
self.assertFalse(output.exists())
|
|
|
|
def test_cli_schema_failure_returns_nonzero_and_writes_nothing(self):
|
|
profile = build_writer_profile(_load_base_config())
|
|
bad_result = _build_result(profile, structured_output={"candidateBody": ""})
|
|
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
output = pathlib.Path(directory) / "refreshed.json"
|
|
with mock.patch.object(
|
|
refresh_module,
|
|
"run_role",
|
|
lambda _profile, _business_input, **_kwargs: bad_result,
|
|
):
|
|
rc = refresh_module.main([
|
|
"--config", str(CONFIG_PATH),
|
|
"--output", str(output),
|
|
])
|
|
self.assertEqual(rc, 1)
|
|
self.assertFalse(output.exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|