"""Tests for KIN-OBS-006: permissions field convention and prompt JSON integrity. Covers: 1. (convention #957) Every agent in specialists.yaml has an explicit 'permissions' field. 2. (convention #957) Parametrized: all analysis-only agents have permissions == 'read_only'. 3. (convention #957) Parametrized: write agents (full/read_bash) do NOT have permissions == 'read_only'. 4. (gotcha #958) Parametrized: JSON code blocks in ## Return Format sections of all prompt files must be parseable — catches accidental corruption during editing. 5. Registration gap: smoke_tester and analyst are not yet in specialists.yaml (known audit finding). """ import json import re from pathlib import Path import pytest import yaml SPECIALISTS_YAML = Path(__file__).parent.parent / "agents" / "specialists.yaml" PROMPTS_DIR = Path(__file__).parent.parent / "agents" / "prompts" # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @pytest.fixture(scope="module") def spec(): return yaml.safe_load(SPECIALISTS_YAML.read_text()) @pytest.fixture(scope="module") def specialists(spec): return spec["specialists"] # --------------------------------------------------------------------------- # Agent permission categories (derived from specialists.yaml current state) # --------------------------------------------------------------------------- # All agents registered in specialists.yaml with expected permissions = read_only READ_ONLY_AGENTS = [ "pm", "architect", "reviewer", "tech_researcher", "repo_researcher", "constitution", "spec", "constitutional_validator", "task_decomposer", "backend_head", "frontend_head", "qa_head", "security_head", "infra_head", "prompt_engineer", "cto_advisor", "knowledge_synthesizer", "research_head", "error_coordinator", "return_analyst", "marketing_head", ] # Agents that have write or bash-execute access — must NOT have permissions == 'read_only' WRITE_AGENTS = [ ("debugger", "read_bash"), ("frontend_dev", "full"), ("backend_dev", "full"), ("tester", "full"), ("security", "read_bash"), ("sysadmin", "read_bash"), ] # --------------------------------------------------------------------------- # Convention #957: every agent has the 'permissions' field # --------------------------------------------------------------------------- class TestPermissionsFieldPresence: def test_every_registered_agent_has_permissions_field(self, specialists): """Every specialist entry must have an explicit 'permissions' field (convention #957).""" missing = [ name for name, entry in specialists.items() if "permissions" not in entry ] assert not missing, ( f"Agents missing 'permissions' field (convention #957): {missing}" ) # --------------------------------------------------------------------------- # Convention #957: read_only agents — parametrized # --------------------------------------------------------------------------- class TestReadOnlyAgentPermissions: @pytest.mark.parametrize("agent_name", READ_ONLY_AGENTS) def test_read_only_agent_is_registered_in_specialists_yaml(self, specialists, agent_name): """Every analysis-only agent must be registered in specialists.yaml.""" assert agent_name in specialists, ( f"Agent '{agent_name}' is not registered in specialists.yaml" ) @pytest.mark.parametrize("agent_name", READ_ONLY_AGENTS) def test_read_only_agent_has_permissions_read_only(self, specialists, agent_name): """Analysis-only agents must have permissions == 'read_only' (convention #957).""" if agent_name not in specialists: pytest.skip(f"Agent '{agent_name}' not registered in specialists.yaml") actual = specialists[agent_name]["permissions"] assert actual == "read_only", ( f"Agent '{agent_name}' expected permissions='read_only', got: {actual!r}" ) # --------------------------------------------------------------------------- # Convention #957: write agents — parametrized # --------------------------------------------------------------------------- class TestWriteAgentPermissions: @pytest.mark.parametrize("agent_name,expected_permissions", WRITE_AGENTS) def test_write_agent_is_registered_in_specialists_yaml(self, specialists, agent_name, expected_permissions): """Write agents must be registered in specialists.yaml.""" assert agent_name in specialists, ( f"Write agent '{agent_name}' is not registered in specialists.yaml" ) @pytest.mark.parametrize("agent_name,expected_permissions", WRITE_AGENTS) def test_write_agent_does_not_have_read_only_permissions(self, specialists, agent_name, expected_permissions): """Write agents must NOT have permissions == 'read_only' (convention #957).""" if agent_name not in specialists: pytest.skip(f"Agent '{agent_name}' not registered in specialists.yaml") actual = specialists[agent_name]["permissions"] assert actual != "read_only", ( f"Agent '{agent_name}' is a write agent but has permissions='read_only'" ) @pytest.mark.parametrize("agent_name,expected_permissions", WRITE_AGENTS) def test_write_agent_has_expected_permissions_value(self, specialists, agent_name, expected_permissions): """Write agent has the exact expected permissions value (full or read_bash).""" if agent_name not in specialists: pytest.skip(f"Agent '{agent_name}' not registered in specialists.yaml") actual = specialists[agent_name]["permissions"] assert actual == expected_permissions, ( f"Agent '{agent_name}' expected permissions='{expected_permissions}', got '{actual}'" ) # --------------------------------------------------------------------------- # Registration gap: known unregistered agents (audit finding from KIN-OBS-006) # --------------------------------------------------------------------------- class TestUnregisteredAgentGaps: def test_smoke_tester_prompt_file_exists_on_disk(self): """smoke_tester.md exists on disk.""" assert (PROMPTS_DIR / "smoke_tester.md").exists() def test_smoke_tester_not_yet_in_specialists_yaml(self, specialists): """smoke_tester is NOT registered in specialists.yaml (audit finding). When added, it must have permissions: read_bash (uses SSH and curl against prod). Update this test when the gap is fixed. """ assert "smoke_tester" not in specialists, ( "smoke_tester is now registered. " "Verify permissions: read_bash is set, then convert this to a positive test." ) def test_analyst_prompt_file_exists_on_disk(self): """analyst.md exists on disk.""" assert (PROMPTS_DIR / "analyst.md").exists() def test_analyst_not_yet_in_specialists_yaml(self, specialists): """analyst is NOT registered in specialists.yaml (audit finding). When added, it must have permissions: read_only. Update this test when the gap is fixed. """ assert "analyst" not in specialists, ( "analyst is now registered. " "Verify permissions: read_only is set, then convert this to a positive test." ) # --------------------------------------------------------------------------- # Gotcha #958: JSON examples in ## Return Format must not be damaged # --------------------------------------------------------------------------- def _extract_return_format_json_blocks(prompt_path: Path) -> list[str]: """Extract all ```json...``` blocks from the ## Return Format section of a prompt file.""" content = prompt_path.read_text() match = re.search(r"## Return Format\b", content) if match is None: return [] start = match.end() next_heading = re.search(r"\n## ", content[start:]) section = content[start: start + next_heading.start()] if next_heading else content[start:] blocks = re.findall(r"```json\s*(.*?)```", section, re.DOTALL) return [b.strip() for b in blocks] def _collect_prompt_json_cases() -> list[tuple[str, int, str]]: """Build parametrize input: (prompt_stem, block_index, json_text) for all prompt files.""" cases = [] for prompt_file in sorted(PROMPTS_DIR.glob("*.md")): for idx, block in enumerate(_extract_return_format_json_blocks(prompt_file)): if block: cases.append((prompt_file.stem, idx, block)) return cases _PROMPT_JSON_CASES = _collect_prompt_json_cases() class TestPromptJsonIntegrity: @pytest.mark.parametrize( "prompt_name,block_index,json_text", _PROMPT_JSON_CASES, ids=[f"{name}_block{i}" for name, i, _ in _PROMPT_JSON_CASES], ) def test_return_format_json_block_is_valid_json(self, prompt_name, block_index, json_text): """Every JSON code block in ## Return Format must parse without error (gotcha #958). Catches accidental corruption of JSON examples when editing prompt files. """ try: json.loads(json_text) except json.JSONDecodeError as e: pytest.fail( f"agents/prompts/{prompt_name}.md: JSON block #{block_index} " f"in ## Return Format section is not valid JSON.\n" f"Parse error: {e}\n" f"Block content (first 500 chars):\n{json_text[:500]}" ) def test_all_prompt_files_have_return_format_section(self): """All prompt files should have a ## Return Format section. Verifies no prompt was accidentally stripped of its output contract. """ prompt_files = sorted(PROMPTS_DIR.glob("*.md")) missing = [] for pf in prompt_files: content = pf.read_text() if "## Return Format" not in content: missing.append(pf.name) assert not missing, ( f"Prompt files missing ## Return Format section: {missing}" )