diff --git a/Makefile b/Makefile index 21f3191e..df395bcb 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install validate validate-structure validate-collection-schema validate-collection-compliance validate-compass-manifests validate-skill-design validate-skill-design-changed validate-mcp-tools validate-spelling package clean check-uv +.PHONY: help install validate validate-structure validate-collection-schema validate-collection-compliance validate-compass-manifests validate-lifecycle-ceiling validate-skill-design validate-skill-design-changed validate-mcp-tools validate-spelling package clean check-uv help: @echo "agentic-plugins" @@ -10,6 +10,7 @@ help: @echo " validate-collection-schema - Schema + roster + banners (subset of compliance)" @echo " validate-collection-compliance - Full .catalog compliance (includes collection.json drift)" @echo " validate-compass-manifests - Compass manifests, roster, refs, and skill references/ layout" + @echo " validate-lifecycle-ceiling - Compass lifecycle ceiling (skill <= plugin lifecycle) + unit tests" @echo " validate-skill-design - Validate all skills (use PACK=rh-sre for a specific pack)" @echo " validate-skill-design-changed - Validate only changed skills (staged + unstaged, for local dev)" @echo " validate-mcp-tools - Validate allowed-tools against live MCP servers (requires podman)" @@ -62,6 +63,10 @@ validate: check-uv uv run python scripts/validate_collection_compliance.py || EXIT=1; \ echo "=== Validating Compass manifests..."; \ uv run python scripts/validate_compass_manifests.py || EXIT=1; \ + echo "=== Validating Compass lifecycle ceiling (skill <= plugin lifecycle)..."; \ + uv run python scripts/validate_lifecycle_ceiling.py || EXIT=1; \ + echo "=== Running lifecycle ceiling unit tests..."; \ + uv run python scripts/test_validate_lifecycle_ceiling.py || EXIT=1; \ echo "=== Validating MCP tool references (skips gracefully without podman)..."; \ uv run python scripts/validate_mcp_tools.py --summary-only --log-file .validate/mcp-tools.log || EXIT=1; \ echo "=== Validating skill design principles..."; \ @@ -88,6 +93,10 @@ validate-structure: check-uv uv run python scripts/validate_collection_compliance.py || EXIT=1; \ echo "=== Validating Compass manifests..."; \ uv run python scripts/validate_compass_manifests.py || EXIT=1; \ + echo "=== Validating Compass lifecycle ceiling (skill <= plugin lifecycle)..."; \ + uv run python scripts/validate_lifecycle_ceiling.py || EXIT=1; \ + echo "=== Running lifecycle ceiling unit tests..."; \ + uv run python scripts/test_validate_lifecycle_ceiling.py || EXIT=1; \ echo "=== Validating MCP tool references (skips gracefully without podman)..."; \ uv run python scripts/validate_mcp_tools.py --summary-only --log-file .validate/mcp-tools.log || EXIT=1; \ echo "=== Validation complete!"; \ @@ -102,6 +111,10 @@ validate-collection-compliance: check-uv validate-compass-manifests: check-uv @uv run python scripts/validate_compass_manifests.py +validate-lifecycle-ceiling: check-uv + @uv run python scripts/validate_lifecycle_ceiling.py + @uv run python scripts/test_validate_lifecycle_ceiling.py + validate-skill-design: check-uv @uv run python scripts/validate_skills_tier2.py $(if $(PACK),$(PACK)) diff --git a/scripts/test_validate_lifecycle_ceiling.py b/scripts/test_validate_lifecycle_ceiling.py new file mode 100644 index 00000000..56b40aa6 --- /dev/null +++ b/scripts/test_validate_lifecycle_ceiling.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Unit tests for the Compass lifecycle ceiling validator.""" + +from __future__ import annotations + +import importlib.util +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +import yaml + +_SCRIPTS = Path(__file__).resolve().parent + + +def _load_module(name: str, filename: str): + spec = importlib.util.spec_from_file_location(name, _SCRIPTS / filename) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +lifecycle_ceiling = _load_module("validate_lifecycle_ceiling", "validate_lifecycle_ceiling.py") + + +def _write_manifest(path: Path, *, name: str, kind: str = "AiResource", lifecycle: str | None = "__unset__") -> None: + """Write a minimal Compass manifest. lifecycle='__unset__' omits the field entirely.""" + data: dict = { + "apiVersion": "backstage.io/v1alpha1", + "kind": kind, + "metadata": {"name": name}, + "spec": {}, + } + if lifecycle != "__unset__": + data["spec"]["lifecycle"] = lifecycle + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + + +def _write_root_catalog(root: Path, packs: list[str]) -> None: + data = { + "apiVersion": "backstage.io/v1alpha1", + "kind": "Location", + "metadata": {"name": "agentic-plugins"}, + "spec": {"targets": [f"./{pack}/catalog-info.yaml" for pack in packs] + ["./mcps/catalog-info.yaml"]}, + } + (root / "catalog-info.yaml").write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + (root / "mcps").mkdir(parents=True, exist_ok=True) + (root / "mcps" / "catalog-info.yaml").write_text( + yaml.safe_dump({"apiVersion": "backstage.io/v1alpha1", "kind": "Location", "spec": {"targets": []}}), + encoding="utf-8", + ) + + +def _write_pack( + root: Path, + pack: str, + *, + plugin_lifecycle: str | None = "__unset__", + skills: dict[str, str | None] | None = None, +) -> None: + """Create /-plugin.yaml and /skills//catalog-info.yaml files.""" + pack_dir = root / pack + _write_manifest(pack_dir / f"{pack}-plugin.yaml", name=pack, lifecycle=plugin_lifecycle) + for skill_name, skill_lifecycle in (skills or {}).items(): + _write_manifest( + pack_dir / "skills" / skill_name / "catalog-info.yaml", + name=skill_name, + lifecycle=skill_lifecycle, + ) + (root / pack / "catalog-info.yaml").write_text( + yaml.safe_dump({"apiVersion": "backstage.io/v1alpha1", "kind": "Location", "spec": {"targets": []}}), + encoding="utf-8", + ) + + +class _TempRepoTestCase(unittest.TestCase): + repo_root: Path + + def setUp(self) -> None: + self.repo_root = Path(tempfile.mkdtemp()) + + def tearDown(self) -> None: + shutil.rmtree(self.repo_root, ignore_errors=True) + + +class TestLifecycleRank(unittest.TestCase): + def test_known_lifecycles_ordered(self) -> None: + self.assertEqual(lifecycle_ceiling.lifecycle_rank("development"), 0) + self.assertEqual(lifecycle_ceiling.lifecycle_rank("beta"), 1) + self.assertEqual(lifecycle_ceiling.lifecycle_rank("production"), 2) + + def test_missing_lifecycle_defaults_to_development(self) -> None: + self.assertEqual( + lifecycle_ceiling.lifecycle_rank(None), + lifecycle_ceiling.lifecycle_rank("development"), + ) + + def test_unknown_lifecycle_raises(self) -> None: + with self.assertRaises(ValueError): + lifecycle_ceiling.lifecycle_rank("ga") + + def test_is_deprecated(self) -> None: + self.assertTrue(lifecycle_ceiling.is_deprecated("deprecated")) + self.assertTrue(lifecycle_ceiling.is_deprecated("Deprecated")) + self.assertFalse(lifecycle_ceiling.is_deprecated("beta")) + self.assertFalse(lifecycle_ceiling.is_deprecated(None)) + + +class TestPassingCases(_TempRepoTestCase): + def test_skill_equal_to_plugin_passes(self) -> None: + _write_pack(self.repo_root, "rh-demo", plugin_lifecycle="beta", skills={"demo-skill": "beta"}) + + errors: list[str] = [] + lifecycle_ceiling.check_pack(self.repo_root, "rh-demo", errors) + + self.assertEqual(errors, []) + + def test_skill_less_mature_than_plugin_passes(self) -> None: + _write_pack( + self.repo_root, "rh-demo", plugin_lifecycle="production", skills={"demo-skill": "development"} + ) + + errors: list[str] = [] + lifecycle_ceiling.check_pack(self.repo_root, "rh-demo", errors) + + self.assertEqual(errors, []) + + def test_missing_lifecycles_default_to_development_and_pass(self) -> None: + _write_pack( + self.repo_root, + "rh-demo", + plugin_lifecycle="__unset__", + skills={"demo-skill": "__unset__"}, + ) + + errors: list[str] = [] + lifecycle_ceiling.check_pack(self.repo_root, "rh-demo", errors) + + self.assertEqual(errors, []) + + def test_multiple_skills_all_within_ceiling(self) -> None: + _write_pack( + self.repo_root, + "rh-demo", + plugin_lifecycle="beta", + skills={"skill-a": "development", "skill-b": "beta"}, + ) + + errors: list[str] = [] + lifecycle_ceiling.check_pack(self.repo_root, "rh-demo", errors) + + self.assertEqual(errors, []) + + +class TestFailCase(_TempRepoTestCase): + def test_skill_more_mature_than_plugin_fails(self) -> None: + _write_pack(self.repo_root, "rh-demo", plugin_lifecycle="development", skills={"demo-skill": "beta"}) + + errors: list[str] = [] + lifecycle_ceiling.check_pack(self.repo_root, "rh-demo", errors) + + self.assertEqual(len(errors), 1) + self.assertIn("demo-skill", errors[0]) + self.assertIn("'beta'", errors[0]) + self.assertIn("'development'", errors[0]) + + def test_production_skill_under_beta_plugin_fails(self) -> None: + _write_pack(self.repo_root, "rh-demo", plugin_lifecycle="beta", skills={"demo-skill": "production"}) + + errors: list[str] = [] + lifecycle_ceiling.check_pack(self.repo_root, "rh-demo", errors) + + self.assertEqual(len(errors), 1) + + def test_only_offending_skill_is_reported(self) -> None: + _write_pack( + self.repo_root, + "rh-demo", + plugin_lifecycle="development", + skills={"ok-skill": "development", "bad-skill": "production"}, + ) + + errors: list[str] = [] + lifecycle_ceiling.check_pack(self.repo_root, "rh-demo", errors) + + self.assertEqual(len(errors), 1) + self.assertIn("bad-skill", errors[0]) + self.assertNotIn("ok-skill", errors[0]) + + +class TestDeprecatedSkipLogic(_TempRepoTestCase): + def test_deprecated_skill_is_skipped_even_if_more_mature(self) -> None: + _write_pack(self.repo_root, "rh-demo", plugin_lifecycle="development", skills={"demo-skill": "deprecated"}) + + errors: list[str] = [] + lifecycle_ceiling.check_pack(self.repo_root, "rh-demo", errors) + + self.assertEqual(errors, []) + + def test_deprecated_plugin_skips_all_skills(self) -> None: + _write_pack( + self.repo_root, + "rh-demo", + plugin_lifecycle="deprecated", + skills={"demo-skill": "production"}, + ) + + errors: list[str] = [] + lifecycle_ceiling.check_pack(self.repo_root, "rh-demo", errors) + + self.assertEqual(errors, []) + + def test_deprecated_skill_among_others_only_skips_itself(self) -> None: + _write_pack( + self.repo_root, + "rh-demo", + plugin_lifecycle="development", + skills={"deprecated-skill": "deprecated", "bad-skill": "beta"}, + ) + + errors: list[str] = [] + lifecycle_ceiling.check_pack(self.repo_root, "rh-demo", errors) + + self.assertEqual(len(errors), 1) + self.assertIn("bad-skill", errors[0]) + self.assertNotIn("deprecated-skill", errors[0]) + + +class TestValidateAll(_TempRepoTestCase): + def test_validate_all_discovers_registered_packs_from_root_catalog(self) -> None: + _write_root_catalog(self.repo_root, ["rh-good", "rh-bad"]) + _write_pack(self.repo_root, "rh-good", plugin_lifecycle="beta", skills={"good-skill": "beta"}) + _write_pack(self.repo_root, "rh-bad", plugin_lifecycle="development", skills={"bad-skill": "production"}) + + errors = lifecycle_ceiling.validate_all(self.repo_root) + + self.assertEqual(len(errors), 1) + self.assertIn("bad-skill", errors[0]) + + def test_validate_all_ignores_unregistered_packs(self) -> None: + _write_root_catalog(self.repo_root, ["rh-good"]) + _write_pack(self.repo_root, "rh-good", plugin_lifecycle="beta", skills={"good-skill": "beta"}) + _write_pack(self.repo_root, "rh-bad", plugin_lifecycle="development", skills={"bad-skill": "production"}) + + errors = lifecycle_ceiling.validate_all(self.repo_root) + + self.assertEqual(errors, []) + + def test_validate_all_missing_root_catalog_reports_error(self) -> None: + errors = lifecycle_ceiling.validate_all(self.repo_root) + + self.assertEqual(len(errors), 1) + self.assertIn("catalog-info.yaml", errors[0]) + + def test_pack_missing_plugin_manifest_reports_error(self) -> None: + _write_root_catalog(self.repo_root, ["rh-orphan"]) + (self.repo_root / "rh-orphan").mkdir(parents=True) + + errors = lifecycle_ceiling.validate_all(self.repo_root) + + self.assertEqual(len(errors), 1) + self.assertIn("rh-orphan", errors[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate_lifecycle_ceiling.py b/scripts/validate_lifecycle_ceiling.py new file mode 100644 index 00000000..0ee3210f --- /dev/null +++ b/scripts/validate_lifecycle_ceiling.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +""" +Validate the Compass "lifecycle ceiling" rule. + +By design, a child skill cannot have a more mature ``spec.lifecycle`` than +its parent plugin (pack). This script enforces that rule in CI: + + - The allowed lifecycle order is: development (0) < beta (1) < production (2). + - Missing lifecycles default to "development". + - Entities with lifecycle "deprecated" (skills or plugins) are skipped — + they are not compared against the ceiling. + - Packs are discovered from the root ``catalog-info.yaml`` ``spec.targets`` + (the same set Compass ingests), mirroring ``validate_compass_manifests.py``. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import yaml + +_REPO_ROOT = Path(__file__).resolve().parent.parent + +# Allowed lifecycle maturity order: development < beta < production. +LIFECYCLE_RANK = {"development": 0, "beta": 1, "production": 2} +DEFAULT_LIFECYCLE = "development" +DEPRECATED_LIFECYCLE = "deprecated" + + +def _load_yaml(path: Path) -> dict: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError(f"{path}: expected mapping at top level") + return data + + +def normalize_lifecycle(lifecycle: str | None) -> str: + """Return the effective lifecycle string, defaulting missing values to development.""" + if lifecycle is None: + return DEFAULT_LIFECYCLE + value = str(lifecycle).strip().lower() + return value or DEFAULT_LIFECYCLE + + +def is_deprecated(lifecycle: str | None) -> bool: + """Return True when the (normalized) lifecycle is 'deprecated'.""" + return normalize_lifecycle(lifecycle) == DEPRECATED_LIFECYCLE + + +def lifecycle_rank(lifecycle: str | None) -> int: + """Map a lifecycle string to its maturity rank (missing -> development).""" + value = normalize_lifecycle(lifecycle) + if value not in LIFECYCLE_RANK: + raise ValueError( + f"unknown lifecycle '{lifecycle}'; expected one of " + f"{sorted(LIFECYCLE_RANK)} or '{DEPRECATED_LIFECYCLE}'" + ) + return LIFECYCLE_RANK[value] + + +def registered_packs(root: Path) -> list[str]: + """Return pack directory names referenced from the root catalog-info.yaml.""" + root_catalog = root / "catalog-info.yaml" + data = _load_yaml(root_catalog) + packs: list[str] = [] + for target in data.get("spec", {}).get("targets", []): + if not isinstance(target, str): + continue + if target.startswith("./mcps/"): + continue + if not target.endswith("/catalog-info.yaml"): + continue + parts = Path(target).parts + if len(parts) != 2: + continue + packs.append(parts[0]) + return sorted(set(packs)) + + +def _skill_manifests(pack_dir: Path) -> list[Path]: + skills_dir = pack_dir / "skills" + if not skills_dir.is_dir(): + return [] + return sorted(skills_dir.glob("*/catalog-info.yaml")) + + +def check_pack(root: Path, pack: str, errors: list[str]) -> None: + """Validate the lifecycle ceiling for a single pack (skills vs. their plugin).""" + pack_dir = root / pack + plugin_path = pack_dir / f"{pack}-plugin.yaml" + if not plugin_path.is_file(): + errors.append(f"{pack}: missing plugin manifest {plugin_path.relative_to(root)}") + return + + try: + plugin_data = _load_yaml(plugin_path) + except (OSError, ValueError, yaml.YAMLError) as exc: + errors.append(f"{plugin_path.relative_to(root)}: failed to load ({exc})") + return + + plugin_lifecycle = normalize_lifecycle(plugin_data.get("spec", {}).get("lifecycle")) + if is_deprecated(plugin_lifecycle): + # Deprecated plugins are exempt — none of their skills are enforced either. + return + + try: + plugin_rank = lifecycle_rank(plugin_lifecycle) + except ValueError as exc: + errors.append(f"{plugin_path.relative_to(root)}: {exc}") + return + + for manifest in _skill_manifests(pack_dir): + try: + skill_data = _load_yaml(manifest) + except (OSError, ValueError, yaml.YAMLError) as exc: + errors.append(f"{manifest.relative_to(root)}: failed to load ({exc})") + continue + + skill_name = skill_data.get("metadata", {}).get("name", manifest.parent.name) + skill_lifecycle = normalize_lifecycle(skill_data.get("spec", {}).get("lifecycle")) + + if is_deprecated(skill_lifecycle): + continue # deprecated skills are exempt from the ceiling check + + try: + skill_rank = lifecycle_rank(skill_lifecycle) + except ValueError as exc: + errors.append(f"{manifest.relative_to(root)}: {exc}") + continue + + if skill_rank > plugin_rank: + errors.append( + f"{pack}/{skill_name}: lifecycle '{skill_lifecycle}' exceeds parent " + f"plugin '{pack}' lifecycle '{plugin_lifecycle}' " + f"({manifest.relative_to(root)})" + ) + + +def validate_all(root: Path) -> list[str]: + """Run the lifecycle ceiling check for every pack registered in catalog-info.yaml.""" + errors: list[str] = [] + root_catalog = root / "catalog-info.yaml" + if not root_catalog.is_file(): + errors.append(f"missing root catalog Location: {root_catalog}") + return errors + + for pack in registered_packs(root): + check_pack(root, pack, errors) + return errors + + +def main() -> int: + errors = validate_all(_REPO_ROOT) + + if errors: + print("Lifecycle ceiling validation failed:", file=sys.stderr) + for err in errors: + print(f" • {err}", file=sys.stderr) + return 1 + + print("✓ Lifecycle ceiling validation passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main())