#!/usr/bin/env python3 """skill_harness 的纯标准库离线回归测试。 所有夹具都在临时目录中构造，不读取当前仓库的 Skill，也不依赖数据库、网络或模型。 """ from __future__ import annotations import io import json import tempfile import unittest from contextlib import redirect_stdout from pathlib import Path from typing import Optional try: from .skill_harness import audit_skills, main except ImportError: # 允许直接执行 `.venv/bin/python harness/test_skill_harness.py` from skill_harness import audit_skills, main class SkillHarnessTests(unittest.TestCase): def make_skill( self, root: Path, directory: str, *, name: Optional[str] = None, body: str = "# 合同\n\n只描述运行时行为。\n", ) -> Path: skill_dir = root / ".claude" / "skills" / directory skill_dir.mkdir(parents=True, exist_ok=True) skill_path = skill_dir / "SKILL.md" frontmatter_name = directory if name is None else name skill_path.write_text( f"---\nname: {frontmatter_name}\ndescription: 离线夹具\n---\n{body}", encoding="utf-8", ) self.write_manifest(root) return skill_path def write_manifest( self, root: Path, entries: Optional[list[dict[str, object]]] = None, *, path: Optional[Path] = None, ) -> Path: if entries is None: entries = [] skills_root = root / ".claude" / "skills" if skills_root.exists(): for directory in sorted(skills_root.iterdir(), key=lambda item: item.name): if directory.is_dir(): entries.append( { "name": directory.name, "contract_owner": "测试夹具", "collaborates_with": [], "skill_path": ( Path(".claude") / "skills" / directory.name / "SKILL.md" ).as_posix(), } ) manifest_path = path or root / "harness" / "manifests" / "skills.json" manifest_path.parent.mkdir(parents=True, exist_ok=True) manifest_path.write_text( json.dumps( {"schema_version": 1, "skills": entries}, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) return manifest_path def test_passing_project_is_independent_of_current_repository(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill(root, "alpha") report = audit_skills(root) self.assertTrue(report["ok"]) self.assertEqual(report["status"], "passed") self.assertEqual(report["skills_scanned"], 1) self.assertEqual(report["issues"], []) def test_pollution_is_reported_for_each_obvious_category(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill( root, "polluted", body=( "## 自测\n" "pytest -q\n" "import unittest\n" "run test_sample.py and sample_test.py\n" "覆盖率达到 100%。\n" "测试通过。\n" ), ) report = audit_skills(root) codes = {issue["code"] for issue in report["issues"]} self.assertFalse(report["ok"]) self.assertTrue( { "development_test_heading", "pytest_reference", "unittest_reference", "test_file_reference", "coverage_reference", "test_pass_declaration", }.issubset(codes) ) def test_development_headings_support_multiple_levels(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill( root, "nested-development-sections", body=( "### 自测\n" "这里不能登记测试命令。\n" "## 离线验证\n" "这里也不能登记测试命令。\n" ), ) report = audit_skills(root) headings = [ issue for issue in report["issues"] if issue["code"] == "development_test_heading" ] self.assertFalse(report["ok"]) self.assertEqual(len(headings), 2) self.assertEqual([issue["line"] for issue in headings], [5, 7]) def test_check_contract_command_requires_development_section_context(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill( root, "contextual-contract-command", body=( "业务合同字段名可以写 check_contract.py。\n" "## 离线验证\n" "python check_contract.py\n" "## 业务说明\n" "普通业务 check_contract.py 不是测试入口。\n" ), ) report = audit_skills(root) command_issues = [ issue for issue in report["issues"] if issue["code"] == "development_test_command_reference" ] self.assertFalse(report["ok"]) self.assertEqual(len(command_issues), 1) self.assertEqual(command_issues[0]["line"], 7) def test_business_coverage_terms_are_not_development_pollution(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill( root, "business-coverage", body=( "细纲覆盖率达到 100%。\n" "事件覆盖率为 80%。\n" "硬约束覆盖率 100%。\n" ), ) report = audit_skills(root) self.assertTrue(report["ok"]) self.assertNotIn( "coverage_reference", {issue["code"] for issue in report["issues"]}, ) def test_development_coverage_terms_are_reported(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill( root, "development-coverage", body=( "coverage.py\n" "pytest-cov\n" "pytest --cov=harness\n" "coverage report\n" "测试覆盖率达到 90%。\n" "覆盖率达到 80%。\n" ), ) report = audit_skills(root) self.assertFalse(report["ok"]) self.assertIn( "coverage_reference", {issue["code"] for issue in report["issues"]}, ) def test_frontmatter_name_must_match_skill_directory(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill(root, "alpha", name="beta") report = audit_skills(root) self.assertFalse(report["ok"]) self.assertIn( "name_directory_mismatch", {issue["code"] for issue in report["issues"]}, ) def test_duplicate_frontmatter_names_are_reported(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill(root, "alpha", name="shared") self.make_skill(root, "beta", name="shared") report = audit_skills(root) duplicates = [ issue for issue in report["issues"] if issue["code"] == "duplicate_name" ] self.assertFalse(report["ok"]) self.assertEqual(len(duplicates), 1) self.assertEqual(duplicates[0]["name"], "shared") self.assertEqual(len(duplicates[0]["paths"]), 2) def test_clean_manifest_can_be_selected_explicitly(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill(root, "alpha") manifest_path = self.write_manifest( root, path=root / "clean-manifest.json", ) output = io.StringIO() with redirect_stdout(output): return_code = main( [ "--root", str(root), "--manifest", str(manifest_path), "--json", "--quiet", ] ) payload = json.loads(output.getvalue()) self.assertEqual(return_code, 0) self.assertTrue(payload["ok"]) self.assertTrue(payload["manifest"]["loaded"]) self.assertEqual(payload["manifest"]["path"], "clean-manifest.json") def test_error_manifest_fails_with_structured_issues(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill(root, "alpha") self.make_skill(root, "beta") broken_manifest = self.write_manifest( root, entries=[ { "name": "alpha", "contract_owner": "", "collaborates_with": ["上下文与知识检索", 3], "skill_path": ".claude/skills/beta/SKILL.md", }, { "name": "alpha", "contract_owner": "测试夹具", "collaborates_with": [], "skill_path": ".claude/skills/beta/SKILL.md", }, { "name": "extra", "contract_owner": "测试夹具", "collaborates_with": [], "skill_path": ".claude/skills/extra/SKILL.md", }, ], path=root / "broken-manifest.json", ) output = io.StringIO() with redirect_stdout(output): return_code = main( [ "--root", str(root), "--manifest", str(broken_manifest), "--json", ] ) payload = json.loads(output.getvalue()) codes = {issue["code"] for issue in payload["issues"]} self.assertEqual(return_code, 1) self.assertFalse(payload["ok"]) self.assertTrue(all(isinstance(issue, dict) for issue in payload["issues"])) self.assertTrue( { "manifest_contract_owner_invalid", "manifest_collaborates_with_invalid", "manifest_duplicate_skill_path", "manifest_duplicate_name", "manifest_skill_missing", "manifest_skill_extra", "manifest_skill_path_mismatch", "manifest_name_directory_mismatch", "manifest_skill_path_missing", }.issubset(codes) ) def test_missing_manifest_fails_closed(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill(root, "alpha") (root / "harness" / "manifests" / "skills.json").unlink() report = audit_skills(root) self.assertFalse(report["ok"]) self.assertIn( "manifest_missing", {issue["code"] for issue in report["issues"]}, ) def test_missing_skills_directory_fails_closed(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) report = audit_skills(root) self.assertFalse(report["ok"]) self.assertIn( "skills_directory_missing", {issue["code"] for issue in report["issues"]}, ) def test_skill_directory_without_skill_file_fails(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) (root / ".claude" / "skills" / "missing").mkdir(parents=True) report = audit_skills(root) self.assertFalse(report["ok"]) self.assertIn( "missing_skill_file", {issue["code"] for issue in report["issues"]}, ) def test_empty_skill_file_is_reported(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) skill_path = self.make_skill(root, "empty") skill_path.write_text("", encoding="utf-8") report = audit_skills(root) self.assertFalse(report["ok"]) self.assertIn( "empty_skill_file", {issue["code"] for issue in report["issues"]}, ) def test_json_cli_and_exit_code(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_skill(root, "alpha") output = io.StringIO() with redirect_stdout(output): return_code = main(["--root", str(root), "--json", "--quiet"]) payload = json.loads(output.getvalue()) self.assertEqual(return_code, 0) self.assertTrue(payload["ok"]) self.assertEqual(payload["skills_scanned"], 1) (root / ".claude" / "skills" / "alpha" / "SKILL.md").write_text( "---\nname: alpha\n---\n## 测试\n", encoding="utf-8", ) output = io.StringIO() with redirect_stdout(output): return_code = main(["--root", str(root), "--json"]) failed_payload = json.loads(output.getvalue()) self.assertEqual(return_code, 1) self.assertFalse(failed_payload["ok"]) if __name__ == "__main__": unittest.main(verbosity=2)