diff --git a/agents/specialists.yaml b/agents/specialists.yaml index 61c7845..2b95677 100644 --- a/agents/specialists.yaml +++ b/agents/specialists.yaml @@ -166,6 +166,7 @@ specialists: description: "Gate agent: validates mission alignment, stack alignment, and complexity appropriateness before implementation begins" permissions: read_only gate: true + output_format: verdict_text_plus_json context_rules: decisions: all modules: all diff --git a/tests/test_kin_obs_006_permissions.py b/tests/test_kin_obs_006_permissions.py new file mode 100644 index 0000000..8654752 --- /dev/null +++ b/tests/test_kin_obs_006_permissions.py @@ -0,0 +1,257 @@ +"""Tests for KIN-OBS-006: permissions field convention and prompt JSON integrity. + +Covers: +1. (convention #957) Every agent in specialists.yaml has an explicit 'permissions' field. +2. (convention #957) Parametrized: all analysis-only agents have permissions == 'read_only'. +3. (convention #957) Parametrized: write agents (full/read_bash) do NOT have permissions == 'read_only'. +4. (gotcha #958) Parametrized: JSON code blocks in ## Return Format sections of all prompt files + must be parseable — catches accidental corruption during editing. +5. Registration gap: smoke_tester and analyst are not yet in specialists.yaml (known audit finding). +""" + +import json +import re +from pathlib import Path + +import pytest +import yaml + +SPECIALISTS_YAML = Path(__file__).parent.parent / "agents" / "specialists.yaml" +PROMPTS_DIR = Path(__file__).parent.parent / "agents" / "prompts" + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def spec(): + return yaml.safe_load(SPECIALISTS_YAML.read_text()) + + +@pytest.fixture(scope="module") +def specialists(spec): + return spec["specialists"] + + +# --------------------------------------------------------------------------- +# Agent permission categories (derived from specialists.yaml current state) +# --------------------------------------------------------------------------- + +# All agents registered in specialists.yaml with expected permissions = read_only +READ_ONLY_AGENTS = [ + "pm", + "architect", + "reviewer", + "tech_researcher", + "repo_researcher", + "constitution", + "spec", + "constitutional_validator", + "task_decomposer", + "backend_head", + "frontend_head", + "qa_head", + "security_head", + "infra_head", + "prompt_engineer", + "cto_advisor", + "knowledge_synthesizer", + "research_head", + "error_coordinator", + "return_analyst", + "marketing_head", +] + +# Agents that have write or bash-execute access — must NOT have permissions == 'read_only' +WRITE_AGENTS = [ + ("debugger", "read_bash"), + ("frontend_dev", "full"), + ("backend_dev", "full"), + ("tester", "full"), + ("security", "read_bash"), + ("sysadmin", "read_bash"), +] + + +# --------------------------------------------------------------------------- +# Convention #957: every agent has the 'permissions' field +# --------------------------------------------------------------------------- + +class TestPermissionsFieldPresence: + + def test_every_registered_agent_has_permissions_field(self, specialists): + """Every specialist entry must have an explicit 'permissions' field (convention #957).""" + missing = [ + name for name, entry in specialists.items() + if "permissions" not in entry + ] + assert not missing, ( + f"Agents missing 'permissions' field (convention #957): {missing}" + ) + + +# --------------------------------------------------------------------------- +# Convention #957: read_only agents — parametrized +# --------------------------------------------------------------------------- + +class TestReadOnlyAgentPermissions: + + @pytest.mark.parametrize("agent_name", READ_ONLY_AGENTS) + def test_read_only_agent_is_registered_in_specialists_yaml(self, specialists, agent_name): + """Every analysis-only agent must be registered in specialists.yaml.""" + assert agent_name in specialists, ( + f"Agent '{agent_name}' is not registered in specialists.yaml" + ) + + @pytest.mark.parametrize("agent_name", READ_ONLY_AGENTS) + def test_read_only_agent_has_permissions_read_only(self, specialists, agent_name): + """Analysis-only agents must have permissions == 'read_only' (convention #957).""" + if agent_name not in specialists: + pytest.skip(f"Agent '{agent_name}' not registered in specialists.yaml") + actual = specialists[agent_name]["permissions"] + assert actual == "read_only", ( + f"Agent '{agent_name}' expected permissions='read_only', got: {actual!r}" + ) + + +# --------------------------------------------------------------------------- +# Convention #957: write agents — parametrized +# --------------------------------------------------------------------------- + +class TestWriteAgentPermissions: + + @pytest.mark.parametrize("agent_name,expected_permissions", WRITE_AGENTS) + def test_write_agent_is_registered_in_specialists_yaml(self, specialists, agent_name, expected_permissions): + """Write agents must be registered in specialists.yaml.""" + assert agent_name in specialists, ( + f"Write agent '{agent_name}' is not registered in specialists.yaml" + ) + + @pytest.mark.parametrize("agent_name,expected_permissions", WRITE_AGENTS) + def test_write_agent_does_not_have_read_only_permissions(self, specialists, agent_name, expected_permissions): + """Write agents must NOT have permissions == 'read_only' (convention #957).""" + if agent_name not in specialists: + pytest.skip(f"Agent '{agent_name}' not registered in specialists.yaml") + actual = specialists[agent_name]["permissions"] + assert actual != "read_only", ( + f"Agent '{agent_name}' is a write agent but has permissions='read_only'" + ) + + @pytest.mark.parametrize("agent_name,expected_permissions", WRITE_AGENTS) + def test_write_agent_has_expected_permissions_value(self, specialists, agent_name, expected_permissions): + """Write agent has the exact expected permissions value (full or read_bash).""" + if agent_name not in specialists: + pytest.skip(f"Agent '{agent_name}' not registered in specialists.yaml") + actual = specialists[agent_name]["permissions"] + assert actual == expected_permissions, ( + f"Agent '{agent_name}' expected permissions='{expected_permissions}', got '{actual}'" + ) + + +# --------------------------------------------------------------------------- +# Registration gap: known unregistered agents (audit finding from KIN-OBS-006) +# --------------------------------------------------------------------------- + +class TestUnregisteredAgentGaps: + + def test_smoke_tester_prompt_file_exists_on_disk(self): + """smoke_tester.md exists on disk.""" + assert (PROMPTS_DIR / "smoke_tester.md").exists() + + def test_smoke_tester_not_yet_in_specialists_yaml(self, specialists): + """smoke_tester is NOT registered in specialists.yaml (audit finding). + + When added, it must have permissions: read_bash (uses SSH and curl against prod). + Update this test when the gap is fixed. + """ + assert "smoke_tester" not in specialists, ( + "smoke_tester is now registered. " + "Verify permissions: read_bash is set, then convert this to a positive test." + ) + + def test_analyst_prompt_file_exists_on_disk(self): + """analyst.md exists on disk.""" + assert (PROMPTS_DIR / "analyst.md").exists() + + def test_analyst_not_yet_in_specialists_yaml(self, specialists): + """analyst is NOT registered in specialists.yaml (audit finding). + + When added, it must have permissions: read_only. + Update this test when the gap is fixed. + """ + assert "analyst" not in specialists, ( + "analyst is now registered. " + "Verify permissions: read_only is set, then convert this to a positive test." + ) + + +# --------------------------------------------------------------------------- +# Gotcha #958: JSON examples in ## Return Format must not be damaged +# --------------------------------------------------------------------------- + +def _extract_return_format_json_blocks(prompt_path: Path) -> list[str]: + """Extract all ```json...``` blocks from the ## Return Format section of a prompt file.""" + content = prompt_path.read_text() + + match = re.search(r"## Return Format\b", content) + if match is None: + return [] + + start = match.end() + next_heading = re.search(r"\n## ", content[start:]) + section = content[start: start + next_heading.start()] if next_heading else content[start:] + + blocks = re.findall(r"```json\s*(.*?)```", section, re.DOTALL) + return [b.strip() for b in blocks] + + +def _collect_prompt_json_cases() -> list[tuple[str, int, str]]: + """Build parametrize input: (prompt_stem, block_index, json_text) for all prompt files.""" + cases = [] + for prompt_file in sorted(PROMPTS_DIR.glob("*.md")): + for idx, block in enumerate(_extract_return_format_json_blocks(prompt_file)): + if block: + cases.append((prompt_file.stem, idx, block)) + return cases + + +_PROMPT_JSON_CASES = _collect_prompt_json_cases() + + +class TestPromptJsonIntegrity: + + @pytest.mark.parametrize( + "prompt_name,block_index,json_text", + _PROMPT_JSON_CASES, + ids=[f"{name}_block{i}" for name, i, _ in _PROMPT_JSON_CASES], + ) + def test_return_format_json_block_is_valid_json(self, prompt_name, block_index, json_text): + """Every JSON code block in ## Return Format must parse without error (gotcha #958). + + Catches accidental corruption of JSON examples when editing prompt files. + """ + try: + json.loads(json_text) + except json.JSONDecodeError as e: + pytest.fail( + f"agents/prompts/{prompt_name}.md: JSON block #{block_index} " + f"in ## Return Format section is not valid JSON.\n" + f"Parse error: {e}\n" + f"Block content (first 500 chars):\n{json_text[:500]}" + ) + + def test_all_prompt_files_have_return_format_section(self): + """All prompt files should have a ## Return Format section. + + Verifies no prompt was accidentally stripped of its output contract. + """ + prompt_files = sorted(PROMPTS_DIR.glob("*.md")) + missing = [] + for pf in prompt_files: + content = pf.read_text() + if "## Return Format" not in content: + missing.append(pf.name) + assert not missing, ( + f"Prompt files missing ## Return Format section: {missing}" + )