kin: auto-commit after pipeline
This commit is contained in:
Gros Frumos
parent
23c7157274
commit
d5b4b1a4a6
2 changed files with 258 additions and 0 deletions
257
tests/test_kin_obs_006_permissions.py
Normal file
257
tests/test_kin_obs_006_permissions.py
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
"""Tests for KIN-OBS-006: permissions field convention and prompt JSON integrity.
|
||||
|
||||
Covers:
|
||||
1. (convention #957) Every agent in specialists.yaml has an explicit 'permissions' field.
|
||||
2. (convention #957) Parametrized: all analysis-only agents have permissions == 'read_only'.
|
||||
3. (convention #957) Parametrized: write agents (full/read_bash) do NOT have permissions == 'read_only'.
|
||||
4. (gotcha #958) Parametrized: JSON code blocks in ## Return Format sections of all prompt files
|
||||
must be parseable — catches accidental corruption during editing.
|
||||
5. Registration gap: smoke_tester and analyst are not yet in specialists.yaml (known audit finding).
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
SPECIALISTS_YAML = Path(__file__).parent.parent / "agents" / "specialists.yaml"
|
||||
PROMPTS_DIR = Path(__file__).parent.parent / "agents" / "prompts"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def spec():
|
||||
return yaml.safe_load(SPECIALISTS_YAML.read_text())
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def specialists(spec):
|
||||
return spec["specialists"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Agent permission categories (derived from specialists.yaml current state)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# All agents registered in specialists.yaml with expected permissions = read_only
|
||||
READ_ONLY_AGENTS = [
|
||||
"pm",
|
||||
"architect",
|
||||
"reviewer",
|
||||
"tech_researcher",
|
||||
"repo_researcher",
|
||||
"constitution",
|
||||
"spec",
|
||||
"constitutional_validator",
|
||||
"task_decomposer",
|
||||
"backend_head",
|
||||
"frontend_head",
|
||||
"qa_head",
|
||||
"security_head",
|
||||
"infra_head",
|
||||
"prompt_engineer",
|
||||
"cto_advisor",
|
||||
"knowledge_synthesizer",
|
||||
"research_head",
|
||||
"error_coordinator",
|
||||
"return_analyst",
|
||||
"marketing_head",
|
||||
]
|
||||
|
||||
# Agents that have write or bash-execute access — must NOT have permissions == 'read_only'
|
||||
WRITE_AGENTS = [
|
||||
("debugger", "read_bash"),
|
||||
("frontend_dev", "full"),
|
||||
("backend_dev", "full"),
|
||||
("tester", "full"),
|
||||
("security", "read_bash"),
|
||||
("sysadmin", "read_bash"),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Convention #957: every agent has the 'permissions' field
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPermissionsFieldPresence:
|
||||
|
||||
def test_every_registered_agent_has_permissions_field(self, specialists):
|
||||
"""Every specialist entry must have an explicit 'permissions' field (convention #957)."""
|
||||
missing = [
|
||||
name for name, entry in specialists.items()
|
||||
if "permissions" not in entry
|
||||
]
|
||||
assert not missing, (
|
||||
f"Agents missing 'permissions' field (convention #957): {missing}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Convention #957: read_only agents — parametrized
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestReadOnlyAgentPermissions:
|
||||
|
||||
@pytest.mark.parametrize("agent_name", READ_ONLY_AGENTS)
|
||||
def test_read_only_agent_is_registered_in_specialists_yaml(self, specialists, agent_name):
|
||||
"""Every analysis-only agent must be registered in specialists.yaml."""
|
||||
assert agent_name in specialists, (
|
||||
f"Agent '{agent_name}' is not registered in specialists.yaml"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("agent_name", READ_ONLY_AGENTS)
|
||||
def test_read_only_agent_has_permissions_read_only(self, specialists, agent_name):
|
||||
"""Analysis-only agents must have permissions == 'read_only' (convention #957)."""
|
||||
if agent_name not in specialists:
|
||||
pytest.skip(f"Agent '{agent_name}' not registered in specialists.yaml")
|
||||
actual = specialists[agent_name]["permissions"]
|
||||
assert actual == "read_only", (
|
||||
f"Agent '{agent_name}' expected permissions='read_only', got: {actual!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Convention #957: write agents — parametrized
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestWriteAgentPermissions:
|
||||
|
||||
@pytest.mark.parametrize("agent_name,expected_permissions", WRITE_AGENTS)
|
||||
def test_write_agent_is_registered_in_specialists_yaml(self, specialists, agent_name, expected_permissions):
|
||||
"""Write agents must be registered in specialists.yaml."""
|
||||
assert agent_name in specialists, (
|
||||
f"Write agent '{agent_name}' is not registered in specialists.yaml"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("agent_name,expected_permissions", WRITE_AGENTS)
|
||||
def test_write_agent_does_not_have_read_only_permissions(self, specialists, agent_name, expected_permissions):
|
||||
"""Write agents must NOT have permissions == 'read_only' (convention #957)."""
|
||||
if agent_name not in specialists:
|
||||
pytest.skip(f"Agent '{agent_name}' not registered in specialists.yaml")
|
||||
actual = specialists[agent_name]["permissions"]
|
||||
assert actual != "read_only", (
|
||||
f"Agent '{agent_name}' is a write agent but has permissions='read_only'"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("agent_name,expected_permissions", WRITE_AGENTS)
|
||||
def test_write_agent_has_expected_permissions_value(self, specialists, agent_name, expected_permissions):
|
||||
"""Write agent has the exact expected permissions value (full or read_bash)."""
|
||||
if agent_name not in specialists:
|
||||
pytest.skip(f"Agent '{agent_name}' not registered in specialists.yaml")
|
||||
actual = specialists[agent_name]["permissions"]
|
||||
assert actual == expected_permissions, (
|
||||
f"Agent '{agent_name}' expected permissions='{expected_permissions}', got '{actual}'"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registration gap: known unregistered agents (audit finding from KIN-OBS-006)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestUnregisteredAgentGaps:
|
||||
|
||||
def test_smoke_tester_prompt_file_exists_on_disk(self):
|
||||
"""smoke_tester.md exists on disk."""
|
||||
assert (PROMPTS_DIR / "smoke_tester.md").exists()
|
||||
|
||||
def test_smoke_tester_not_yet_in_specialists_yaml(self, specialists):
|
||||
"""smoke_tester is NOT registered in specialists.yaml (audit finding).
|
||||
|
||||
When added, it must have permissions: read_bash (uses SSH and curl against prod).
|
||||
Update this test when the gap is fixed.
|
||||
"""
|
||||
assert "smoke_tester" not in specialists, (
|
||||
"smoke_tester is now registered. "
|
||||
"Verify permissions: read_bash is set, then convert this to a positive test."
|
||||
)
|
||||
|
||||
def test_analyst_prompt_file_exists_on_disk(self):
|
||||
"""analyst.md exists on disk."""
|
||||
assert (PROMPTS_DIR / "analyst.md").exists()
|
||||
|
||||
def test_analyst_not_yet_in_specialists_yaml(self, specialists):
|
||||
"""analyst is NOT registered in specialists.yaml (audit finding).
|
||||
|
||||
When added, it must have permissions: read_only.
|
||||
Update this test when the gap is fixed.
|
||||
"""
|
||||
assert "analyst" not in specialists, (
|
||||
"analyst is now registered. "
|
||||
"Verify permissions: read_only is set, then convert this to a positive test."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gotcha #958: JSON examples in ## Return Format must not be damaged
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_return_format_json_blocks(prompt_path: Path) -> list[str]:
|
||||
"""Extract all ```json...``` blocks from the ## Return Format section of a prompt file."""
|
||||
content = prompt_path.read_text()
|
||||
|
||||
match = re.search(r"## Return Format\b", content)
|
||||
if match is None:
|
||||
return []
|
||||
|
||||
start = match.end()
|
||||
next_heading = re.search(r"\n## ", content[start:])
|
||||
section = content[start: start + next_heading.start()] if next_heading else content[start:]
|
||||
|
||||
blocks = re.findall(r"```json\s*(.*?)```", section, re.DOTALL)
|
||||
return [b.strip() for b in blocks]
|
||||
|
||||
|
||||
def _collect_prompt_json_cases() -> list[tuple[str, int, str]]:
|
||||
"""Build parametrize input: (prompt_stem, block_index, json_text) for all prompt files."""
|
||||
cases = []
|
||||
for prompt_file in sorted(PROMPTS_DIR.glob("*.md")):
|
||||
for idx, block in enumerate(_extract_return_format_json_blocks(prompt_file)):
|
||||
if block:
|
||||
cases.append((prompt_file.stem, idx, block))
|
||||
return cases
|
||||
|
||||
|
||||
_PROMPT_JSON_CASES = _collect_prompt_json_cases()
|
||||
|
||||
|
||||
class TestPromptJsonIntegrity:
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prompt_name,block_index,json_text",
|
||||
_PROMPT_JSON_CASES,
|
||||
ids=[f"{name}_block{i}" for name, i, _ in _PROMPT_JSON_CASES],
|
||||
)
|
||||
def test_return_format_json_block_is_valid_json(self, prompt_name, block_index, json_text):
|
||||
"""Every JSON code block in ## Return Format must parse without error (gotcha #958).
|
||||
|
||||
Catches accidental corruption of JSON examples when editing prompt files.
|
||||
"""
|
||||
try:
|
||||
json.loads(json_text)
|
||||
except json.JSONDecodeError as e:
|
||||
pytest.fail(
|
||||
f"agents/prompts/{prompt_name}.md: JSON block #{block_index} "
|
||||
f"in ## Return Format section is not valid JSON.\n"
|
||||
f"Parse error: {e}\n"
|
||||
f"Block content (first 500 chars):\n{json_text[:500]}"
|
||||
)
|
||||
|
||||
def test_all_prompt_files_have_return_format_section(self):
|
||||
"""All prompt files should have a ## Return Format section.
|
||||
|
||||
Verifies no prompt was accidentally stripped of its output contract.
|
||||
"""
|
||||
prompt_files = sorted(PROMPTS_DIR.glob("*.md"))
|
||||
missing = []
|
||||
for pf in prompt_files:
|
||||
content = pf.read_text()
|
||||
if "## Return Format" not in content:
|
||||
missing.append(pf.name)
|
||||
assert not missing, (
|
||||
f"Prompt files missing ## Return Format section: {missing}"
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue