195 lines
7.5 KiB
Python
195 lines
7.5 KiB
Python
"""Tests for KIN-037: tech_researcher specialist — YAML validation and prompt structure."""
|
|
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
import pytest
|
|
|
|
SPECIALISTS_YAML = Path(__file__).parent.parent / "agents" / "specialists.yaml"
|
|
PROMPTS_DIR = Path(__file__).parent.parent / "agents" / "prompts"
|
|
TECH_RESEARCHER_PROMPT = PROMPTS_DIR / "tech_researcher.md"
|
|
|
|
REQUIRED_SPECIALIST_FIELDS = {"name", "model", "tools", "description", "permissions"}
|
|
REQUIRED_OUTPUT_SCHEMA_FIELDS = {
|
|
"status", "api_overview", "endpoints", "rate_limits", "auth_method",
|
|
"data_schemas", "limitations", "gotchas", "codebase_diff", "notes",
|
|
}
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def spec():
|
|
"""Load and parse specialists.yaml once for all tests."""
|
|
return yaml.safe_load(SPECIALISTS_YAML.read_text())
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def tech_researcher(spec):
|
|
return spec["specialists"]["tech_researcher"]
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def prompt_text():
|
|
return TECH_RESEARCHER_PROMPT.read_text()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# YAML validity
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestSpecialistsYaml:
|
|
def test_yaml_parses_without_error(self):
|
|
content = SPECIALISTS_YAML.read_text()
|
|
parsed = yaml.safe_load(content)
|
|
assert parsed is not None
|
|
|
|
def test_yaml_has_specialists_key(self, spec):
|
|
assert "specialists" in spec
|
|
|
|
def test_yaml_has_routes_key(self, spec):
|
|
assert "routes" in spec
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# tech_researcher entry structure
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestTechResearcherEntry:
|
|
def test_tech_researcher_exists_in_specialists(self, spec):
|
|
assert "tech_researcher" in spec["specialists"]
|
|
|
|
def test_tech_researcher_has_required_fields(self, tech_researcher):
|
|
missing = REQUIRED_SPECIALIST_FIELDS - set(tech_researcher.keys())
|
|
assert not missing, f"Missing fields: {missing}"
|
|
|
|
def test_tech_researcher_name_is_string(self, tech_researcher):
|
|
assert isinstance(tech_researcher["name"], str)
|
|
assert tech_researcher["name"].strip()
|
|
|
|
def test_tech_researcher_model_is_sonnet(self, tech_researcher):
|
|
assert tech_researcher["model"] == "sonnet"
|
|
|
|
def test_tech_researcher_tools_is_list(self, tech_researcher):
|
|
assert isinstance(tech_researcher["tools"], list)
|
|
assert len(tech_researcher["tools"]) > 0
|
|
|
|
def test_tech_researcher_tools_include_webfetch(self, tech_researcher):
|
|
assert "WebFetch" in tech_researcher["tools"]
|
|
|
|
def test_tech_researcher_tools_include_read_grep_glob(self, tech_researcher):
|
|
for tool in ("Read", "Grep", "Glob"):
|
|
assert tool in tech_researcher["tools"], f"Missing tool: {tool}"
|
|
|
|
def test_tech_researcher_permissions_is_read_only(self, tech_researcher):
|
|
assert tech_researcher["permissions"] == "read_only"
|
|
|
|
def test_tech_researcher_description_is_non_empty_string(self, tech_researcher):
|
|
assert isinstance(tech_researcher["description"], str)
|
|
assert len(tech_researcher["description"]) > 10
|
|
|
|
def test_tech_researcher_has_output_schema(self, tech_researcher):
|
|
assert "output_schema" in tech_researcher
|
|
|
|
def test_tech_researcher_output_schema_has_required_fields(self, tech_researcher):
|
|
schema = tech_researcher["output_schema"]
|
|
missing = REQUIRED_OUTPUT_SCHEMA_FIELDS - set(schema.keys())
|
|
assert not missing, f"Missing output_schema fields: {missing}"
|
|
|
|
def test_tech_researcher_context_rules_decisions_is_list(self, tech_researcher):
|
|
decisions = tech_researcher.get("context_rules", {}).get("decisions")
|
|
assert isinstance(decisions, list)
|
|
|
|
def test_tech_researcher_context_rules_includes_gotcha(self, tech_researcher):
|
|
decisions = tech_researcher.get("context_rules", {}).get("decisions", [])
|
|
assert "gotcha" in decisions
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# api_research route
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestApiResearchRoute:
|
|
def test_api_research_route_exists(self, spec):
|
|
assert "api_research" in spec["routes"]
|
|
|
|
def test_api_research_route_has_steps(self, spec):
|
|
route = spec["routes"]["api_research"]
|
|
assert "steps" in route
|
|
assert isinstance(route["steps"], list)
|
|
assert len(route["steps"]) >= 1
|
|
|
|
def test_api_research_route_starts_with_tech_researcher(self, spec):
|
|
steps = spec["routes"]["api_research"]["steps"]
|
|
assert steps[0] == "tech_researcher"
|
|
|
|
def test_api_research_route_includes_architect(self, spec):
|
|
steps = spec["routes"]["api_research"]["steps"]
|
|
assert "architect" in steps
|
|
|
|
def test_api_research_route_has_description(self, spec):
|
|
route = spec["routes"]["api_research"]
|
|
assert "description" in route
|
|
assert isinstance(route["description"], str)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Prompt file existence
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestTechResearcherPromptFile:
|
|
def test_prompt_file_exists(self):
|
|
assert TECH_RESEARCHER_PROMPT.exists(), (
|
|
f"Prompt file not found: {TECH_RESEARCHER_PROMPT}"
|
|
)
|
|
|
|
def test_prompt_file_is_not_empty(self, prompt_text):
|
|
assert len(prompt_text.strip()) > 100
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Prompt content — structured review instructions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestTechResearcherPromptContent:
|
|
def test_prompt_contains_json_output_instruction(self, prompt_text):
|
|
assert "JSON" in prompt_text or "json" in prompt_text
|
|
|
|
def test_prompt_defines_status_field(self, prompt_text):
|
|
assert '"status"' in prompt_text
|
|
|
|
def test_prompt_defines_done_partial_blocked_statuses(self, prompt_text):
|
|
assert "done" in prompt_text
|
|
assert "partial" in prompt_text
|
|
assert "blocked" in prompt_text
|
|
|
|
def test_prompt_defines_api_overview_field(self, prompt_text):
|
|
assert "api_overview" in prompt_text
|
|
|
|
def test_prompt_defines_endpoints_field(self, prompt_text):
|
|
assert "endpoints" in prompt_text
|
|
|
|
def test_prompt_defines_rate_limits_field(self, prompt_text):
|
|
assert "rate_limits" in prompt_text
|
|
|
|
def test_prompt_defines_codebase_diff_field(self, prompt_text):
|
|
assert "codebase_diff" in prompt_text
|
|
|
|
def test_prompt_defines_gotchas_field(self, prompt_text):
|
|
assert "gotchas" in prompt_text
|
|
|
|
def test_prompt_contains_webfetch_instruction(self, prompt_text):
|
|
assert "WebFetch" in prompt_text
|
|
|
|
def test_prompt_mentions_no_secrets_logging(self, prompt_text):
|
|
"""Prompt must instruct agent not to log secret values."""
|
|
lower = prompt_text.lower()
|
|
assert "secret" in lower or "credential" in lower or "token" in lower
|
|
|
|
def test_prompt_specifies_readonly_bash(self, prompt_text):
|
|
"""Bash must be restricted to read-only operations per rules."""
|
|
assert "read-only" in prompt_text or "read only" in prompt_text or "GET" in prompt_text
|
|
|
|
def test_prompt_defines_partial_reason_for_partial_status(self, prompt_text):
|
|
assert "partial_reason" in prompt_text
|
|
|
|
def test_prompt_defines_blocked_reason_for_blocked_status(self, prompt_text):
|
|
assert "blocked_reason" in prompt_text
|