d80df3ed7c
范围（不含 design-story-foundation、docs/design、docs/write-chapter、
craft/、humanization/README.md 等进行中改动）:
1. humanization 规则/样例运行时数据库权威
- db/ddl/111：example_ai_flavor_rule / example_ai_flavor_sample /
example_ai_flavor_rule_event（append-only 生命周期留痕），已应用到 muse-example
- deai/load_db.py：数据库装载器，激活门/重复检测/指纹与文件装载器同源；
数据库失败关闭，不静默回退 Git 文件资产
- humanization/tools/seed_rules_db.py：YAML 种子单事务同步，幂等、
变化留痕、--strict 对 db-only 行失败关闭；真实库已种入 26 规则/107 样例
- prevent/diagnose/revise 生产路径切到数据库读取（--offline 显式读文件），
落库前新鲜度检查与合同声明来源一致；四个 SKILL.md 数据库合同同步
- 真实库验证：规则库指纹与文件种子一致（v-609bc40e21d0b5db），
三个生产脚本端到端从库装载通过
2. PostgreSQL 集成：显式授权后 9/9 通过
- 此前被依赖门阻断的 6 个 _db/smoke 测试全部通过
- extract rollback 冒烟改为回滚事务内自给夹具（pending 窗/草稿缺失时自建），
不再依赖瞬时生产状态；夹具残留核验为 0
3. Skill 行为评测脚手架（真实执行数量仍为 0）
- harness/evals/skill_eval.py：场景合同、六类评测范畴、适配器和结构化裁决报告
- diagnose-ai-flavor 参考场景 4 条 + 管道自测 7 项通过
- 真实模型适配器未授权时以稳定码 EVAL_ADAPTER_UNAVAILABLE 失败关闭；
清单登记 skill_behavior_eval 条目，默认被依赖门阻断
4. evaluate-frozen-replay raw 存储边界冲突
- docs/2026-08-19 备忘录：平台 DB-first 合同（创始人批准）与回放链
仓外 vault 强制的冲突事实、两个选项和裁决前约束；运行时合同未单方面改写
5. harness 自身修复
- runner 对账语义：行为评测入口不参与测试资产双向等值，但登记文件必须存在；
manifest 保留 skill_behavior_eval 布尔字段并校验类型
- 新增 2 条对账回归用例
验证证据: harness 三组自测 15+15+7 通过；静态审计 32 Skill / 0 问题；
76 个非数据库条目通过；9 个 PostgreSQL 集成条目显式授权后通过；
行为评测条目默认阻断；py_compile 与 git diff --check 通过。
未调用真实模型、embedding 或额度；真实行为评测执行数量仍为 0。
1072 lines
36 KiB
Python
1072 lines
36 KiB
Python
#!/usr/bin/env python3
|
||
"""按 manifest 选择并执行离线测试条目的最小 harness。
|
||
|
||
该入口只在明确选择条目或传入 ``--all-offline`` 时运行，不承担全仓默认门禁。
|
||
执行器自身不连接数据库、网络或模型；危险依赖只有在 ``--allow-requires`` 明确
|
||
放行后才会启动对应子进程。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import ast
|
||
import json
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
from typing import Any, Iterable, Optional, Sequence
|
||
|
||
SCHEMA_VERSION = 1
|
||
DEFAULT_MANIFEST_PATH = Path("harness") / "manifests" / "test-inventory.json"
|
||
BLOCKED_REQUIRES = frozenset({"postgresql", "network", "model", "credentials"})
|
||
ENTRY_STATUSES = frozenset(
|
||
{"passed", "failed", "timeout", "blocked_dependency", "not_found"}
|
||
)
|
||
OUTPUT_SUMMARY_LIMIT = 2000
|
||
DEFAULT_TIMEOUT_SECONDS = 60.0
|
||
_TEST_ASSET_SCAN_ROOTS = (
|
||
Path(".claude") / "skills",
|
||
Path("tests"),
|
||
Path("humanization") / "tests",
|
||
Path("harness"),
|
||
Path("dashboard"),
|
||
)
|
||
_IGNORED_TEST_ASSET_PARTS = frozenset({".git", ".venv", "__pycache__"})
|
||
_NO_TESTS_OUTPUT_PATTERN = re.compile(
|
||
r"(?im)^\s*(?:"
|
||
r"no\s+tests?(?:\s+(?:were\s+)?(?:found|collected|run|ran|to\s+run))?"
|
||
r"|collected\s+0\s+items?"
|
||
r"|ran\s+0\s+tests?"
|
||
r"|0\s+tests?\s+ran"
|
||
r")\b[^\n]*$"
|
||
)
|
||
_SKIP_ONLY_LINE_PATTERN = re.compile(
|
||
r"(?i)^\s*(?:(?:\d+\s+)?skipped\b.*|"
|
||
r"skip(?:ped)?\s*(?:[:-]\s*.*)?)\s*$"
|
||
)
|
||
_UNITTEST_SKIP_SUMMARY_PATTERN = re.compile(
|
||
r"(?im)^\s*OK\s*\(\s*skipped\s*=\s*(\d+)\s*\)\s*$"
|
||
)
|
||
_UNITTEST_RUN_PATTERN = re.compile(r"(?i)\bRan\s+(\d+)\s+tests?\b")
|
||
_EXECUTION_RESULT_PATTERN = re.compile(
|
||
r"(?i)\b(?:\d+\s+(?:passed|failed|errors?|xfailed|xpassed)|"
|
||
r"(?:ran|collected)\s+[1-9]\d*)\b"
|
||
)
|
||
|
||
|
||
def _resolve_root(root: str | Path) -> Path:
|
||
path = Path(root).expanduser()
|
||
if not path.is_absolute():
|
||
path = Path.cwd() / path
|
||
return path.resolve()
|
||
|
||
|
||
def _resolve_manifest_path(root: Path, manifest: str | Path | None) -> Path:
|
||
path = DEFAULT_MANIFEST_PATH if manifest is None else Path(manifest).expanduser()
|
||
if not path.is_absolute():
|
||
path = root / path
|
||
return path.resolve()
|
||
|
||
|
||
def _relative_path(path: Path, root: Path) -> str:
|
||
try:
|
||
return path.resolve().relative_to(root).as_posix()
|
||
except ValueError:
|
||
return path.resolve().as_posix()
|
||
|
||
|
||
def _issue(code: str, message: str, **details: Any) -> dict[str, Any]:
|
||
issue: dict[str, Any] = {"code": code, "message": message}
|
||
issue.update(details)
|
||
return issue
|
||
|
||
|
||
def _normalise_manifest_entry_path(root: Path, value: str) -> tuple[Optional[str], Optional[str]]:
|
||
"""校验并规范 manifest 中的相对路径。"""
|
||
|
||
declared = Path(value)
|
||
if declared.is_absolute():
|
||
return None, "manifest_path_not_relative"
|
||
try:
|
||
candidate = (root / declared).resolve()
|
||
relative = candidate.relative_to(root).as_posix()
|
||
except (OSError, RuntimeError, ValueError):
|
||
return None, "manifest_path_outside_root"
|
||
if not relative or relative == ".":
|
||
return None, "manifest_path_empty"
|
||
return relative, None
|
||
|
||
|
||
def _normalise_selector_path(root: Path, value: str) -> str:
|
||
"""把路径选择器转换为与 manifest 相同的 POSIX 相对表示。"""
|
||
|
||
declared = Path(value).expanduser()
|
||
if declared.is_absolute():
|
||
try:
|
||
return declared.resolve().relative_to(root).as_posix()
|
||
except (OSError, RuntimeError, ValueError):
|
||
return declared.resolve().as_posix()
|
||
try:
|
||
return (root / declared).resolve().relative_to(root).as_posix()
|
||
except (OSError, RuntimeError, ValueError):
|
||
return declared.as_posix()
|
||
|
||
|
||
def _string_values(values: Iterable[str] | None) -> list[str]:
|
||
if values is None:
|
||
return []
|
||
return [str(value).strip() for value in values if str(value).strip()]
|
||
|
||
|
||
def _is_named_python_test(path: Path) -> bool:
|
||
return path.suffix.casefold() == ".py" and (
|
||
path.name.startswith("test_") or path.name.endswith("_test.py")
|
||
)
|
||
|
||
|
||
def _is_test_asset_path(relative: str) -> bool:
|
||
path = Path(relative)
|
||
parts = path.parts
|
||
if not parts or any(part in _IGNORED_TEST_ASSET_PARTS for part in parts):
|
||
return False
|
||
if path.name == "__init__.py" or path.suffix.casefold() == ".pyc":
|
||
return False
|
||
|
||
if parts[:2] == (".claude", "skills"):
|
||
return len(parts) >= 3 and path.suffix.casefold() == ".py" and (
|
||
path.name.startswith("test_") or path.name.endswith("_test.py")
|
||
)
|
||
if parts[0] == "tests":
|
||
return len(parts) > 1 and _is_named_python_test(path)
|
||
if parts[:2] == ("humanization", "tests"):
|
||
return len(parts) > 2 and _is_named_python_test(path)
|
||
if parts[0] in {"harness", "dashboard"}:
|
||
return len(parts) > 1 and path.suffix.casefold() == ".py" and path.name.startswith(
|
||
"test_"
|
||
)
|
||
return False
|
||
|
||
|
||
def _scan_test_assets(root: Path) -> tuple[set[str], list[dict[str, Any]]]:
|
||
"""扫描 generated_scope 约定的测试资产，失败时保留明确的扫描问题。"""
|
||
|
||
assets: set[str] = set()
|
||
issues: list[dict[str, Any]] = []
|
||
|
||
def onerror(error: OSError) -> None:
|
||
error_path = getattr(error, "filename", None)
|
||
path = Path(str(error_path)) if error_path else root
|
||
issues.append(
|
||
_issue(
|
||
"test_asset_scan_failed",
|
||
"无法完整扫描测试资产目录",
|
||
path=_relative_path(path, root),
|
||
error=str(error),
|
||
)
|
||
)
|
||
|
||
for relative_root in _TEST_ASSET_SCAN_ROOTS:
|
||
scan_root = root / relative_root
|
||
try:
|
||
if not scan_root.exists():
|
||
continue
|
||
if not scan_root.is_dir():
|
||
issues.append(
|
||
_issue(
|
||
"test_asset_scan_root_invalid",
|
||
"测试资产扫描根路径不是目录",
|
||
path=_relative_path(scan_root, root),
|
||
)
|
||
)
|
||
continue
|
||
except OSError as exc:
|
||
issues.append(
|
||
_issue(
|
||
"test_asset_scan_failed",
|
||
"无法检查测试资产扫描根目录",
|
||
path=_relative_path(scan_root, root),
|
||
error=str(exc),
|
||
)
|
||
)
|
||
continue
|
||
|
||
for current, directory_names, file_names in os.walk(
|
||
scan_root,
|
||
topdown=True,
|
||
followlinks=False,
|
||
onerror=onerror,
|
||
):
|
||
directory_names[:] = sorted(
|
||
name
|
||
for name in directory_names
|
||
if name not in _IGNORED_TEST_ASSET_PARTS
|
||
)
|
||
for file_name in sorted(file_names):
|
||
candidate = Path(current) / file_name
|
||
if candidate.is_symlink():
|
||
continue
|
||
try:
|
||
if not candidate.is_file():
|
||
continue
|
||
relative = candidate.relative_to(root).as_posix()
|
||
except (OSError, ValueError) as exc:
|
||
issues.append(
|
||
_issue(
|
||
"test_asset_scan_failed",
|
||
"无法读取测试资产路径",
|
||
path=_relative_path(candidate, root),
|
||
error=str(exc),
|
||
)
|
||
)
|
||
continue
|
||
if _is_test_asset_path(relative):
|
||
assets.add(relative)
|
||
|
||
return assets, issues
|
||
|
||
|
||
def _load_manifest(
|
||
root: Path, manifest_path: Path
|
||
) -> tuple[Optional[list[dict[str, Any]]], dict[str, Any], list[dict[str, Any]]]:
|
||
"""读取并校验执行所需的 manifest 字段，任何合同错误都关闭执行。"""
|
||
|
||
metadata: dict[str, Any] = {
|
||
"path": _relative_path(manifest_path, root),
|
||
"loaded": False,
|
||
"generated_scope_present": False,
|
||
}
|
||
issues: list[dict[str, Any]] = []
|
||
|
||
if not manifest_path.exists():
|
||
issues.append(
|
||
_issue("manifest_missing", "测试 manifest 不存在", path=metadata["path"])
|
||
)
|
||
return None, metadata, issues
|
||
if not manifest_path.is_file():
|
||
issues.append(
|
||
_issue("manifest_not_file", "测试 manifest 路径不是普通文件", path=metadata["path"])
|
||
)
|
||
return None, metadata, issues
|
||
|
||
try:
|
||
raw = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||
except UnicodeError as exc:
|
||
issues.append(
|
||
_issue(
|
||
"manifest_unreadable",
|
||
"测试 manifest 不是合法 UTF-8 文本",
|
||
path=metadata["path"],
|
||
error=str(exc),
|
||
)
|
||
)
|
||
return None, metadata, issues
|
||
except OSError as exc:
|
||
issues.append(
|
||
_issue(
|
||
"manifest_unreadable",
|
||
"无法读取测试 manifest",
|
||
path=metadata["path"],
|
||
error=str(exc),
|
||
)
|
||
)
|
||
return None, metadata, issues
|
||
except json.JSONDecodeError as exc:
|
||
issues.append(
|
||
_issue(
|
||
"manifest_invalid_json",
|
||
"测试 manifest 不是合法 JSON",
|
||
path=metadata["path"],
|
||
line=exc.lineno,
|
||
column=exc.colno,
|
||
error=exc.msg,
|
||
)
|
||
)
|
||
return None, metadata, issues
|
||
|
||
metadata["loaded"] = True
|
||
if not isinstance(raw, dict):
|
||
issues.append(
|
||
_issue(
|
||
"manifest_invalid_structure",
|
||
"测试 manifest 顶层必须是对象",
|
||
path=metadata["path"],
|
||
)
|
||
)
|
||
return None, metadata, issues
|
||
|
||
metadata["generated_scope_present"] = "generated_scope" in raw
|
||
if metadata["generated_scope_present"]:
|
||
generated_scope = raw["generated_scope"]
|
||
if not isinstance(generated_scope, str) or not generated_scope.strip():
|
||
issues.append(
|
||
_issue(
|
||
"manifest_generated_scope_invalid",
|
||
"测试 manifest 的 generated_scope 必须是非空字符串",
|
||
path=metadata["path"],
|
||
actual=generated_scope,
|
||
)
|
||
)
|
||
else:
|
||
metadata["generated_scope"] = generated_scope.strip()
|
||
|
||
if "schema_version" in raw and raw["schema_version"] != SCHEMA_VERSION:
|
||
issues.append(
|
||
_issue(
|
||
"manifest_schema_version_invalid",
|
||
"测试 manifest 的 schema_version 不受支持",
|
||
path=metadata["path"],
|
||
expected=SCHEMA_VERSION,
|
||
actual=raw["schema_version"],
|
||
)
|
||
)
|
||
|
||
raw_entries = raw.get("entries")
|
||
if not isinstance(raw_entries, list):
|
||
issues.append(
|
||
_issue(
|
||
"manifest_entries_invalid",
|
||
"测试 manifest.entries 必须是数组",
|
||
path=metadata["path"],
|
||
)
|
||
)
|
||
return None, metadata, issues
|
||
|
||
entries: list[dict[str, Any]] = []
|
||
seen_paths: dict[str, int] = {}
|
||
required_string_fields = ("scope", "owner_skill_or_domain", "kind")
|
||
|
||
for index, raw_entry in enumerate(raw_entries):
|
||
if not isinstance(raw_entry, dict):
|
||
issues.append(
|
||
_issue(
|
||
"manifest_entry_invalid",
|
||
"测试 manifest 条目必须是对象",
|
||
path=metadata["path"],
|
||
entry_index=index,
|
||
)
|
||
)
|
||
continue
|
||
|
||
declared_path = raw_entry.get("path")
|
||
if not isinstance(declared_path, str) or not declared_path.strip():
|
||
issues.append(
|
||
_issue(
|
||
"manifest_entry_path_invalid",
|
||
"测试 manifest 条目的 path 必须是非空字符串",
|
||
path=metadata["path"],
|
||
entry_index=index,
|
||
actual=declared_path,
|
||
)
|
||
)
|
||
continue
|
||
normalised_path, path_error = _normalise_manifest_entry_path(root, declared_path)
|
||
if path_error is not None or normalised_path is None:
|
||
issues.append(
|
||
_issue(
|
||
path_error or "manifest_entry_path_invalid",
|
||
"测试 manifest 条目的 path 必须位于项目根目录内且为相对路径",
|
||
path=metadata["path"],
|
||
entry_index=index,
|
||
actual=declared_path,
|
||
)
|
||
)
|
||
continue
|
||
|
||
if normalised_path in seen_paths:
|
||
issues.append(
|
||
_issue(
|
||
"manifest_duplicate_path",
|
||
"测试 manifest 重复登记同一个 path",
|
||
path=normalised_path,
|
||
entry_indexes=[seen_paths[normalised_path], index],
|
||
)
|
||
)
|
||
else:
|
||
seen_paths[normalised_path] = index
|
||
|
||
entry: dict[str, Any] = {"path": normalised_path}
|
||
for field in required_string_fields:
|
||
value = raw_entry.get(field)
|
||
if not isinstance(value, str) or not value.strip():
|
||
issues.append(
|
||
_issue(
|
||
f"manifest_entry_{field}_invalid",
|
||
f"测试 manifest 条目的 {field} 必须是非空字符串",
|
||
path=metadata["path"],
|
||
entry_index=index,
|
||
actual=value,
|
||
)
|
||
)
|
||
else:
|
||
entry[field] = value.strip()
|
||
|
||
requires = raw_entry.get("requires")
|
||
if not isinstance(requires, list) or any(
|
||
not isinstance(requirement, str) or not requirement.strip()
|
||
for requirement in requires
|
||
):
|
||
issues.append(
|
||
_issue(
|
||
"manifest_entry_requires_invalid",
|
||
"测试 manifest 条目的 requires 必须是字符串数组",
|
||
path=metadata["path"],
|
||
entry_index=index,
|
||
actual=requires,
|
||
)
|
||
)
|
||
else:
|
||
entry["requires"] = [requirement.strip() for requirement in requires]
|
||
|
||
behavior_eval = raw_entry.get("skill_behavior_eval")
|
||
if behavior_eval is not None and not isinstance(behavior_eval, bool):
|
||
issues.append(
|
||
_issue(
|
||
"manifest_entry_skill_behavior_eval_invalid",
|
||
"测试 manifest 条目的 skill_behavior_eval 必须是布尔值",
|
||
path=metadata["path"],
|
||
entry_index=index,
|
||
actual=behavior_eval,
|
||
)
|
||
)
|
||
elif behavior_eval:
|
||
entry["skill_behavior_eval"] = True
|
||
|
||
if all(field in entry for field in (*required_string_fields, "requires")):
|
||
entries.append(entry)
|
||
|
||
if issues:
|
||
return None, metadata, issues
|
||
return entries, metadata, []
|
||
|
||
|
||
def _reconcile_test_assets(
|
||
root: Path,
|
||
entries: Sequence[dict[str, Any]],
|
||
manifest_metadata: dict[str, Any],
|
||
) -> list[dict[str, Any]]:
|
||
"""对 generated_scope 清单做磁盘路径与登记路径的一一对账。
|
||
|
||
行为评测入口（skill_behavior_eval=true）不是测试资产：不参与双向等值，
|
||
但登记文件必须存在，避免清单指向空气。
|
||
"""
|
||
|
||
if not manifest_metadata.get("generated_scope_present"):
|
||
return []
|
||
|
||
disk_assets, issues = _scan_test_assets(root)
|
||
test_entries = [entry for entry in entries if not entry.get("skill_behavior_eval")]
|
||
declared_assets = {entry["path"] for entry in test_entries}
|
||
manifest_metadata["test_assets_scanned"] = len(disk_assets)
|
||
|
||
for entry in entries:
|
||
if not entry.get("skill_behavior_eval"):
|
||
continue
|
||
candidate = root / Path(entry["path"])
|
||
try:
|
||
exists = candidate.is_file()
|
||
except OSError:
|
||
exists = False
|
||
if not exists:
|
||
issues.append(
|
||
_issue(
|
||
"manifest_eval_entry_missing",
|
||
"行为评测入口登记文件不存在",
|
||
path=entry["path"],
|
||
)
|
||
)
|
||
|
||
for path in sorted(disk_assets - declared_assets):
|
||
issues.append(
|
||
_issue(
|
||
"manifest_test_asset_missing",
|
||
"磁盘测试资产未在 manifest 中登记",
|
||
path=path,
|
||
)
|
||
)
|
||
# 登记条目不要求都是 test_* 形状（如 skill_behavior_eval 入口），但必须真实存在；
|
||
# 磁盘侧反孤儿不变量仍由上面的 missing 检查承担。
|
||
for path in sorted(declared_assets - disk_assets):
|
||
candidate = root / path
|
||
try:
|
||
exists = candidate.is_file()
|
||
except OSError:
|
||
exists = False
|
||
if not exists:
|
||
issues.append(
|
||
_issue(
|
||
"manifest_test_asset_extra",
|
||
"manifest 登记了磁盘中不存在的测试资产",
|
||
path=path,
|
||
)
|
||
)
|
||
return issues
|
||
|
||
|
||
def _matches(
|
||
entry: dict[str, Any],
|
||
*,
|
||
paths: Sequence[str],
|
||
owners: Sequence[str],
|
||
kinds: Sequence[str],
|
||
scopes: Sequence[str],
|
||
) -> bool:
|
||
return (
|
||
(not paths or entry["path"] in paths)
|
||
and (not owners or entry["owner_skill_or_domain"] in owners)
|
||
and (not kinds or entry["kind"] in kinds)
|
||
and (not scopes or entry["scope"] in scopes)
|
||
)
|
||
|
||
|
||
def _blocked_requires(requires: Sequence[str], allow_requires: set[str]) -> list[str]:
|
||
return [
|
||
requirement
|
||
for requirement in requires
|
||
if requirement.casefold() in BLOCKED_REQUIRES
|
||
and requirement.casefold() not in allow_requires
|
||
]
|
||
|
||
|
||
def _coerce_output(value: Any) -> str:
|
||
if value is None:
|
||
return ""
|
||
if isinstance(value, bytes):
|
||
return value.decode("utf-8", errors="replace")
|
||
return str(value)
|
||
|
||
|
||
def _output_summary(value: Any, limit: int = OUTPUT_SUMMARY_LIMIT) -> str:
|
||
"""只保留有限摘要，避免把子进程 raw 输出写入报告。"""
|
||
|
||
text = _coerce_output(value).strip()
|
||
if len(text) <= limit:
|
||
return text
|
||
marker = "\n...[output summary truncated]...\n"
|
||
available = max(0, limit - len(marker))
|
||
head = available // 2
|
||
tail = available - head
|
||
return text[:head] + marker + text[-tail:]
|
||
|
||
|
||
def _is_no_execution_evidence(stdout: Any, stderr: Any) -> bool:
|
||
"""判断 0 返回码是否只有空输出或明确的跳过/无测试结果。"""
|
||
|
||
text = "\n".join(
|
||
value for value in (_coerce_output(stdout), _coerce_output(stderr)) if value
|
||
).strip()
|
||
if not text:
|
||
return True
|
||
|
||
if _NO_TESTS_OUTPUT_PATTERN.search(text):
|
||
return _EXECUTION_RESULT_PATTERN.search(text) is None
|
||
|
||
summary_match = _UNITTEST_SKIP_SUMMARY_PATTERN.search(text)
|
||
if summary_match:
|
||
run_match = _UNITTEST_RUN_PATTERN.search(text)
|
||
if run_match is None:
|
||
return True
|
||
return int(run_match.group(1)) == int(summary_match.group(1))
|
||
|
||
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
||
return bool(lines) and all(_SKIP_ONLY_LINE_PATTERN.match(line) for line in lines)
|
||
|
||
|
||
def _dotted_name(node: ast.AST) -> Optional[str]:
|
||
if isinstance(node, ast.Name):
|
||
return node.id
|
||
if isinstance(node, ast.Attribute):
|
||
parent = _dotted_name(node.value)
|
||
return f"{parent}.{node.attr}" if parent else None
|
||
return None
|
||
|
||
|
||
def _test_case_aliases(tree: ast.AST) -> tuple[set[str], set[str]]:
|
||
unittest_names = {"unittest"}
|
||
test_case_names = {"TestCase"}
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, ast.Import):
|
||
for alias in node.names:
|
||
if alias.name == "unittest":
|
||
unittest_names.add(alias.asname or alias.name)
|
||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||
if node.module == "unittest" or node.module.startswith("unittest."):
|
||
for alias in node.names:
|
||
if alias.name == "TestCase":
|
||
test_case_names.add(alias.asname or alias.name)
|
||
return unittest_names, test_case_names
|
||
|
||
|
||
def _is_test_case_class(
|
||
node: ast.ClassDef,
|
||
unittest_names: set[str],
|
||
test_case_names: set[str],
|
||
) -> bool:
|
||
for base in node.bases:
|
||
if isinstance(base, ast.Name) and base.id in test_case_names:
|
||
return True
|
||
dotted = _dotted_name(base)
|
||
if dotted and dotted.endswith(".TestCase"):
|
||
module_name = dotted[: -len(".TestCase")]
|
||
if module_name in unittest_names or any(
|
||
module_name.startswith(f"{name}.") for name in unittest_names
|
||
):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _is_main_guard(test: ast.AST) -> bool:
|
||
if not isinstance(test, ast.Compare):
|
||
return False
|
||
values = [test.left, *test.comparators]
|
||
for left, right in zip(values, values[1:]):
|
||
if (
|
||
isinstance(left, ast.Name)
|
||
and left.id == "__name__"
|
||
and isinstance(right, ast.Constant)
|
||
and right.value == "__main__"
|
||
) or (
|
||
isinstance(right, ast.Name)
|
||
and right.id == "__name__"
|
||
and isinstance(left, ast.Constant)
|
||
and left.value == "__main__"
|
||
):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _test_shape_error(candidate: Path, root: Path) -> Optional[dict[str, Any]]:
|
||
try:
|
||
source = candidate.read_text(encoding="utf-8")
|
||
except (OSError, UnicodeError) as exc:
|
||
return {
|
||
"code": "test_shape_unreadable",
|
||
"message": "无法读取 Python 测试文件进行形状检查",
|
||
"path": _relative_path(candidate, root),
|
||
"error": str(exc),
|
||
}
|
||
|
||
try:
|
||
tree = ast.parse(source, filename=str(candidate))
|
||
except SyntaxError as exc:
|
||
return {
|
||
"code": "test_shape_invalid",
|
||
"message": "Python 测试文件无法通过 AST 解析",
|
||
"path": _relative_path(candidate, root),
|
||
"line": exc.lineno,
|
||
"column": exc.offset,
|
||
"error": exc.msg,
|
||
}
|
||
|
||
has_test_function = any(
|
||
isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||
and node.name.startswith("test_")
|
||
for node in ast.walk(tree)
|
||
)
|
||
unittest_names, test_case_names = _test_case_aliases(tree)
|
||
has_test_case = any(
|
||
isinstance(node, ast.ClassDef)
|
||
and _is_test_case_class(node, unittest_names, test_case_names)
|
||
for node in ast.walk(tree)
|
||
)
|
||
has_main_entry = any(
|
||
isinstance(node, ast.If) and _is_main_guard(node.test)
|
||
for node in ast.walk(tree)
|
||
)
|
||
if has_test_function or has_test_case or has_main_entry:
|
||
return None
|
||
|
||
return {
|
||
"code": "test_shape_missing",
|
||
"message": (
|
||
"Python 测试文件必须包含 test_ 函数、unittest.TestCase 测试类"
|
||
"或 __main__ 入口"
|
||
),
|
||
"path": _relative_path(candidate, root),
|
||
}
|
||
|
||
|
||
def _entry_base(entry: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
"path": entry["path"],
|
||
"status": "failed",
|
||
"returncode": None,
|
||
"duration_ms": 0,
|
||
"requires": list(entry["requires"]),
|
||
"stdout": "",
|
||
"stderr": "",
|
||
}
|
||
|
||
|
||
def _run_entry(
|
||
root: Path,
|
||
python_executable: Path,
|
||
entry: dict[str, Any],
|
||
*,
|
||
timeout_seconds: float,
|
||
allow_requires: set[str],
|
||
enforce_test_shape: bool = False,
|
||
) -> dict[str, Any]:
|
||
result = _entry_base(entry)
|
||
candidate = root / Path(entry["path"])
|
||
|
||
try:
|
||
path_exists = candidate.is_file()
|
||
except OSError as exc:
|
||
path_exists = False
|
||
result["error"] = {"code": "path_unreadable", "message": str(exc)}
|
||
if not path_exists:
|
||
result["status"] = "not_found"
|
||
result.setdefault(
|
||
"error",
|
||
{"code": "path_not_found", "message": "测试路径不存在或不是普通文件"},
|
||
)
|
||
return result
|
||
|
||
if enforce_test_shape and candidate.suffix.casefold() == ".py":
|
||
shape_error = _test_shape_error(candidate, root)
|
||
if shape_error is not None:
|
||
result["error"] = shape_error
|
||
return result
|
||
|
||
blocked = _blocked_requires(entry["requires"], allow_requires)
|
||
if blocked:
|
||
result["status"] = "blocked_dependency"
|
||
result["blocked_requires"] = blocked
|
||
return result
|
||
|
||
if not python_executable.is_file():
|
||
result["error"] = {
|
||
"code": "python_executable_not_found",
|
||
"message": "项目 .venv/bin/python 不存在",
|
||
}
|
||
return result
|
||
if not os.access(python_executable, os.X_OK):
|
||
result["error"] = {
|
||
"code": "python_executable_not_executable",
|
||
"message": "项目 .venv/bin/python 不可执行",
|
||
}
|
||
return result
|
||
|
||
started = time.monotonic()
|
||
try:
|
||
completed = subprocess.run(
|
||
[str(python_executable), entry["path"]],
|
||
cwd=str(root),
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
timeout=timeout_seconds,
|
||
check=False,
|
||
)
|
||
except subprocess.TimeoutExpired as exc:
|
||
result["status"] = "timeout"
|
||
result["duration_ms"] = max(0, round((time.monotonic() - started) * 1000))
|
||
result["stdout"] = _output_summary(exc.stdout)
|
||
result["stderr"] = _output_summary(exc.stderr)
|
||
result["error"] = {
|
||
"code": "timeout",
|
||
"message": f"子进程超过 {timeout_seconds:g} 秒限制",
|
||
}
|
||
return result
|
||
except OSError as exc:
|
||
result["duration_ms"] = max(0, round((time.monotonic() - started) * 1000))
|
||
result["stdout"] = _output_summary(getattr(exc, "stdout", ""))
|
||
result["stderr"] = _output_summary(getattr(exc, "stderr", ""))
|
||
result["error"] = {"code": "subprocess_start_failed", "message": str(exc)}
|
||
return result
|
||
|
||
result["returncode"] = completed.returncode
|
||
result["duration_ms"] = max(0, round((time.monotonic() - started) * 1000))
|
||
result["stdout"] = _output_summary(completed.stdout)
|
||
result["stderr"] = _output_summary(completed.stderr)
|
||
if completed.returncode == 0 and _is_no_execution_evidence(
|
||
completed.stdout, completed.stderr
|
||
):
|
||
result["status"] = "failed"
|
||
result["error"] = {
|
||
"code": "no_execution_evidence",
|
||
"message": "子进程返回 0，但没有发现测试执行证据",
|
||
}
|
||
else:
|
||
result["status"] = "passed" if completed.returncode == 0 else "failed"
|
||
if result["status"] == "failed":
|
||
result["error"] = {
|
||
"code": "nonzero_returncode",
|
||
"message": f"子进程返回码为 {completed.returncode}",
|
||
}
|
||
return result
|
||
|
||
|
||
def _selectors_present(
|
||
paths: Sequence[str], owners: Sequence[str], kinds: Sequence[str], scopes: Sequence[str]
|
||
) -> bool:
|
||
return bool(paths or owners or kinds or scopes)
|
||
|
||
|
||
def _base_report(
|
||
root: Path,
|
||
manifest_path: Path,
|
||
*,
|
||
paths: Sequence[str],
|
||
owners: Sequence[str],
|
||
kinds: Sequence[str],
|
||
scopes: Sequence[str],
|
||
all_offline: bool,
|
||
offline_only: bool,
|
||
allow_requires: Sequence[str],
|
||
timeout_seconds: float,
|
||
) -> dict[str, Any]:
|
||
return {
|
||
"schema_version": SCHEMA_VERSION,
|
||
"status": "failed",
|
||
"root": str(root),
|
||
"manifest": {"path": _relative_path(manifest_path, root), "loaded": False},
|
||
"selectors": {
|
||
"path": list(paths),
|
||
"owner": list(owners),
|
||
"kind": list(kinds),
|
||
"scope": list(scopes),
|
||
"all_offline": all_offline,
|
||
},
|
||
"offline_only": offline_only,
|
||
"allow_requires": list(allow_requires),
|
||
"timeout_seconds": timeout_seconds,
|
||
"selected_count": 0,
|
||
"entries": [],
|
||
"issues": [],
|
||
"summary": {"selected": 0, "by_status": {}},
|
||
}
|
||
|
||
|
||
def run_selected(
|
||
root: str | Path = ".",
|
||
manifest: str | Path | None = None,
|
||
*,
|
||
paths: Sequence[str] | None = None,
|
||
owners: Sequence[str] | None = None,
|
||
skills: Sequence[str] | None = None,
|
||
kinds: Sequence[str] | None = None,
|
||
scopes: Sequence[str] | None = None,
|
||
all_offline: bool = False,
|
||
offline_only: bool = True,
|
||
allow_requires: Sequence[str] | None = None,
|
||
timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
|
||
) -> dict[str, Any]:
|
||
"""执行 manifest 中被选择的条目并返回 JSON 可序列化报告。"""
|
||
|
||
root_path = _resolve_root(root)
|
||
manifest_path = _resolve_manifest_path(root_path, manifest)
|
||
path_values = [_normalise_selector_path(root_path, value) for value in _string_values(paths)]
|
||
owner_values = _string_values(owners) + _string_values(skills)
|
||
kind_values = _string_values(kinds)
|
||
scope_values = _string_values(scopes)
|
||
allow_values = _string_values(allow_requires)
|
||
allow_set = {value.casefold() for value in allow_values}
|
||
report = _base_report(
|
||
root_path,
|
||
manifest_path,
|
||
paths=path_values,
|
||
owners=owner_values,
|
||
kinds=kind_values,
|
||
scopes=scope_values,
|
||
all_offline=all_offline,
|
||
offline_only=offline_only,
|
||
allow_requires=allow_values,
|
||
timeout_seconds=timeout_seconds,
|
||
)
|
||
|
||
if not _selectors_present(path_values, owner_values, kind_values, scope_values) and not all_offline:
|
||
report["status"] = "invalid_selector"
|
||
report["issues"] = [
|
||
_issue(
|
||
"selector_required",
|
||
"至少需要 --path、--owner/--skill、--kind 或 --scope，或显式传入 --all-offline",
|
||
)
|
||
]
|
||
return report
|
||
|
||
if not root_path.exists():
|
||
report["status"] = "root_not_found"
|
||
report["issues"] = [_issue("root_missing", "项目根目录不存在", path=".")]
|
||
return report
|
||
if not root_path.is_dir():
|
||
report["status"] = "root_invalid"
|
||
report["issues"] = [_issue("root_not_directory", "项目根路径不是目录", path=".")]
|
||
return report
|
||
|
||
entries, manifest_metadata, manifest_issues = _load_manifest(root_path, manifest_path)
|
||
report["manifest"] = manifest_metadata
|
||
if manifest_issues or entries is None:
|
||
report["status"] = "manifest_invalid"
|
||
report["issues"] = manifest_issues
|
||
return report
|
||
|
||
reconciliation_issues = _reconcile_test_assets(
|
||
root_path,
|
||
entries,
|
||
manifest_metadata,
|
||
)
|
||
if reconciliation_issues:
|
||
report["status"] = "manifest_invalid"
|
||
report["issues"] = reconciliation_issues
|
||
return report
|
||
|
||
has_selectors = _selectors_present(path_values, owner_values, kind_values, scope_values)
|
||
if all_offline and not has_selectors:
|
||
selected = list(entries)
|
||
else:
|
||
selected = [
|
||
entry
|
||
for entry in entries
|
||
if _matches(
|
||
entry,
|
||
paths=path_values,
|
||
owners=owner_values,
|
||
kinds=kind_values,
|
||
scopes=scope_values,
|
||
)
|
||
]
|
||
|
||
if not selected:
|
||
report["status"] = "no_matches"
|
||
report["issues"] = [
|
||
_issue(
|
||
"no_matches",
|
||
"没有条目匹配当前选择器",
|
||
selectors=report["selectors"],
|
||
)
|
||
]
|
||
return report
|
||
|
||
python_executable = root_path / ".venv" / "bin" / "python"
|
||
results = [
|
||
_run_entry(
|
||
root_path,
|
||
python_executable,
|
||
entry,
|
||
timeout_seconds=timeout_seconds,
|
||
allow_requires=allow_set,
|
||
enforce_test_shape=bool(manifest_metadata.get("generated_scope_present")),
|
||
)
|
||
for entry in selected
|
||
]
|
||
counts = Counter(result["status"] for result in results)
|
||
report["entries"] = results
|
||
report["selected_count"] = len(results)
|
||
report["summary"] = {
|
||
"selected": len(results),
|
||
"by_status": {status: counts[status] for status in sorted(counts)},
|
||
}
|
||
report["status"] = "passed" if all(result["status"] == "passed" for result in results) else "failed"
|
||
return report
|
||
|
||
|
||
def human_summary(report: dict[str, Any]) -> str:
|
||
"""输出不包含完整子进程 raw 的人读摘要。"""
|
||
|
||
lines = [
|
||
f"选择性测试：{report['status']}",
|
||
f"项目根：{report['root']}",
|
||
f"选择条目：{report['selected_count']}",
|
||
]
|
||
for entry in report["entries"]:
|
||
lines.append(
|
||
f"- {entry['path']} [{entry['status']}] "
|
||
f"returncode={entry['returncode']} duration_ms={entry['duration_ms']}"
|
||
)
|
||
for issue in report["issues"]:
|
||
lines.append(f"- issue [{issue['code']}] {issue['message']}")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def _positive_timeout(value: str) -> float:
|
||
try:
|
||
parsed = float(value)
|
||
except ValueError as exc:
|
||
raise argparse.ArgumentTypeError("--timeout-seconds 必须是数字") from exc
|
||
if parsed <= 0:
|
||
raise argparse.ArgumentTypeError("--timeout-seconds 必须大于 0")
|
||
return parsed
|
||
|
||
|
||
def _build_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(description="按 manifest 选择性执行项目测试。")
|
||
parser.add_argument("--root", default=".", help="项目根目录，默认为当前目录")
|
||
parser.add_argument(
|
||
"--manifest",
|
||
default=None,
|
||
help="manifest 路径；相对路径按项目根目录解析，默认 harness/manifests/test-inventory.json",
|
||
)
|
||
parser.add_argument("--path", dest="paths", action="append", default=[], help="按相对路径选择条目，可重复")
|
||
parser.add_argument(
|
||
"--owner",
|
||
"--skill",
|
||
dest="owners",
|
||
action="append",
|
||
default=[],
|
||
help="按 owner_skill_or_domain 选择条目，可使用 --skill 作为别名",
|
||
)
|
||
parser.add_argument("--kind", dest="kinds", action="append", default=[], help="按 kind 选择条目，可重复")
|
||
parser.add_argument("--scope", dest="scopes", action="append", default=[], help="按 scope 选择条目，可重复")
|
||
parser.add_argument(
|
||
"--all-offline",
|
||
action="store_true",
|
||
help="没有其它 selector 时选择 manifest 全部条目；危险依赖仍逐条报告为 blocked_dependency",
|
||
)
|
||
parser.add_argument(
|
||
"--offline-only",
|
||
action="store_true",
|
||
default=True,
|
||
help="启用离线依赖门；默认开启，危险依赖须配合 --allow-requires",
|
||
)
|
||
parser.add_argument(
|
||
"--allow-requires",
|
||
dest="allow_requires",
|
||
action="append",
|
||
default=[],
|
||
help="显式放行依赖名，可重复；例如 --allow-requires postgresql",
|
||
)
|
||
parser.add_argument(
|
||
"--timeout-seconds",
|
||
type=_positive_timeout,
|
||
default=DEFAULT_TIMEOUT_SECONDS,
|
||
help="每个子进程的超时秒数，默认 60",
|
||
)
|
||
parser.add_argument("--json", action="store_true", help="输出结构化 JSON 报告")
|
||
return parser
|
||
|
||
|
||
def _exit_code(report: dict[str, Any]) -> int:
|
||
return 0 if report["status"] == "passed" else 1
|
||
|
||
|
||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||
args = _build_parser().parse_args(argv)
|
||
report = run_selected(
|
||
args.root,
|
||
args.manifest,
|
||
paths=args.paths,
|
||
owners=args.owners,
|
||
kinds=args.kinds,
|
||
scopes=args.scopes,
|
||
all_offline=args.all_offline,
|
||
offline_only=args.offline_only,
|
||
allow_requires=args.allow_requires,
|
||
timeout_seconds=args.timeout_seconds,
|
||
)
|
||
if args.json:
|
||
print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
|
||
else:
|
||
print(human_summary(report))
|
||
return _exit_code(report)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|