298 lines
15 KiB
Python
298 lines
15 KiB
Python
|
"""Regression tests for KIN-DOCS-006 — repo_researcher: no API fields in output schema.
|
|||
|
|
|
|||
|
|
Acceptance criteria:
|
|||
|
|
1. repo_researcher зарегистрирован в specialists.yaml с output_schema без API-полей
|
|||
|
|
2. output_schema repo_researcher НЕ содержит полей: endpoints, rate_limits, auth_method
|
|||
|
|
3. output_schema repo_researcher содержит поля основного отчёта: repo_overview, tech_stack,
|
|||
|
|
architecture_summary, key_components, strengths, weaknesses, integration_points, gotchas, notes
|
|||
|
|
4. agents/prompts/repo_researcher.md существует и содержит все 5 стандартных секций
|
|||
|
|
5. Промпт repo_researcher НЕ содержит API-полей в Return Format
|
|||
|
|
6. repo_researcher доступен в departments.research.workers
|
|||
|
|
7. Регрессия: tech_researcher по-прежнему содержит свои API-поля (не сломан)
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
import pytest
|
|||
|
|
import yaml
|
|||
|
|
|
|||
|
|
|
|||
|
|
SPECIALISTS_YAML = Path(__file__).parent.parent / "agents" / "specialists.yaml"
|
|||
|
|
PROMPTS_DIR = Path(__file__).parent.parent / "agents" / "prompts"
|
|||
|
|
REPO_RESEARCHER_PROMPT = PROMPTS_DIR / "repo_researcher.md"
|
|||
|
|
|
|||
|
|
REQUIRED_SECTIONS = [
|
|||
|
|
"## Working Mode",
|
|||
|
|
"## Focus On",
|
|||
|
|
"## Quality Checks",
|
|||
|
|
"## Return Format",
|
|||
|
|
"## Constraints",
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
# Поля, которые НЕ должны присутствовать в repo_researcher (API-специфичные поля)
|
|||
|
|
API_FIELDS_FORBIDDEN = {"endpoints", "rate_limits", "auth_method"}
|
|||
|
|
|
|||
|
|
# Поля, которые ОБЯЗАНЫ присутствовать в repo_researcher output_schema
|
|||
|
|
REPO_RESEARCHER_REQUIRED_SCHEMA_FIELDS = {
|
|||
|
|
"status",
|
|||
|
|
"repo_overview",
|
|||
|
|
"tech_stack",
|
|||
|
|
"architecture_summary",
|
|||
|
|
"key_components",
|
|||
|
|
"strengths",
|
|||
|
|
"weaknesses",
|
|||
|
|
"integration_points",
|
|||
|
|
"gotchas",
|
|||
|
|
"notes",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# API-поля, которые обязаны остаться в tech_researcher (регрессия)
|
|||
|
|
TECH_RESEARCHER_API_FIELDS = {"endpoints", "rate_limits", "auth_method"}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _load_yaml():
|
|||
|
|
return yaml.safe_load(SPECIALISTS_YAML.read_text(encoding="utf-8"))
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ===========================================================================
|
|||
|
|
# 1. repo_researcher — YAML schema: отсутствие API-полей
|
|||
|
|
# ===========================================================================
|
|||
|
|
|
|||
|
|
class TestRepoResearcherOutputSchemaNoApiFields:
|
|||
|
|
"""output_schema repo_researcher НЕ должна содержать API-специфичные поля."""
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize("forbidden_field", sorted(API_FIELDS_FORBIDDEN))
|
|||
|
|
def test_output_schema_does_not_contain_api_field(self, forbidden_field):
|
|||
|
|
"""output_schema repo_researcher не содержит API-поле."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
schema = data["specialists"]["repo_researcher"]["output_schema"]
|
|||
|
|
assert forbidden_field not in schema, (
|
|||
|
|
f"output_schema repo_researcher не должна содержать поле {forbidden_field!r} — "
|
|||
|
|
"это API-специфичное поле, принадлежащее tech_researcher"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ===========================================================================
|
|||
|
|
# 2. repo_researcher — YAML schema: наличие полей отчёта
|
|||
|
|
# ===========================================================================
|
|||
|
|
|
|||
|
|
class TestRepoResearcherOutputSchemaRequiredFields:
|
|||
|
|
"""output_schema repo_researcher ДОЛЖНА содержать все поля основного отчёта."""
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize("required_field", sorted(REPO_RESEARCHER_REQUIRED_SCHEMA_FIELDS))
|
|||
|
|
def test_output_schema_contains_required_field(self, required_field):
|
|||
|
|
"""output_schema repo_researcher содержит обязательное поле отчёта."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
schema = data["specialists"]["repo_researcher"]["output_schema"]
|
|||
|
|
assert required_field in schema, (
|
|||
|
|
f"output_schema repo_researcher обязана содержать поле {required_field!r}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ===========================================================================
|
|||
|
|
# 3. repo_researcher — регистрация и базовая структура в specialists.yaml
|
|||
|
|
# ===========================================================================
|
|||
|
|
|
|||
|
|
class TestRepoResearcherSpecialistsEntry:
|
|||
|
|
"""repo_researcher зарегистрирован в specialists.yaml с корректной базовой структурой."""
|
|||
|
|
|
|||
|
|
def test_repo_researcher_exists_in_specialists(self):
|
|||
|
|
"""repo_researcher присутствует в секции specialists."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
assert "repo_researcher" in data.get("specialists", {}), (
|
|||
|
|
"repo_researcher отсутствует в specialists.yaml"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_repo_researcher_model_is_sonnet(self):
|
|||
|
|
"""repo_researcher использует модель sonnet."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
role = data["specialists"]["repo_researcher"]
|
|||
|
|
assert role.get("model") == "sonnet", (
|
|||
|
|
f"Ожидался model=sonnet, получили: {role.get('model')}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_repo_researcher_permissions_is_read_only(self):
|
|||
|
|
"""repo_researcher имеет permissions=read_only."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
role = data["specialists"]["repo_researcher"]
|
|||
|
|
assert role.get("permissions") == "read_only", (
|
|||
|
|
f"Ожидался permissions=read_only, получили: {role.get('permissions')}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_repo_researcher_tools_include_read_grep_glob(self):
|
|||
|
|
"""repo_researcher имеет инструменты Read, Grep, Glob."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
tools = data["specialists"]["repo_researcher"].get("tools", [])
|
|||
|
|
for tool in ("Read", "Grep", "Glob"):
|
|||
|
|
assert tool in tools, f"repo_researcher должен иметь инструмент {tool!r}"
|
|||
|
|
|
|||
|
|
def test_repo_researcher_has_output_schema(self):
|
|||
|
|
"""repo_researcher имеет поле output_schema."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
role = data["specialists"]["repo_researcher"]
|
|||
|
|
assert "output_schema" in role, "repo_researcher должен иметь output_schema"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ===========================================================================
|
|||
|
|
# 4. repo_researcher промпт — существование и структура
|
|||
|
|
# ===========================================================================
|
|||
|
|
|
|||
|
|
class TestRepoResearcherPromptStructure:
|
|||
|
|
"""agents/prompts/repo_researcher.md существует и содержит все 5 стандартных секций."""
|
|||
|
|
|
|||
|
|
def test_prompt_file_exists(self):
|
|||
|
|
"""Файл agents/prompts/repo_researcher.md существует."""
|
|||
|
|
assert REPO_RESEARCHER_PROMPT.exists(), (
|
|||
|
|
f"Промпт repo_researcher не найден: {REPO_RESEARCHER_PROMPT}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_file_is_not_empty(self):
|
|||
|
|
"""Файл repo_researcher.md не пустой."""
|
|||
|
|
content = REPO_RESEARCHER_PROMPT.read_text(encoding="utf-8")
|
|||
|
|
assert len(content.strip()) > 100
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize("section", REQUIRED_SECTIONS)
|
|||
|
|
def test_prompt_has_required_section(self, section):
|
|||
|
|
"""Промпт repo_researcher.md содержит каждую из 5 стандартных секций."""
|
|||
|
|
content = REPO_RESEARCHER_PROMPT.read_text(encoding="utf-8")
|
|||
|
|
assert section in content, (
|
|||
|
|
f"repo_researcher.md не содержит обязательную секцию {section!r}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_sections_in_correct_order(self):
|
|||
|
|
"""5 обязательных секций расположены в правильном порядке."""
|
|||
|
|
content = REPO_RESEARCHER_PROMPT.read_text(encoding="utf-8")
|
|||
|
|
positions = [content.find(sec) for sec in REQUIRED_SECTIONS]
|
|||
|
|
assert all(p != -1 for p in positions), "Не все 5 секций найдены в repo_researcher.md"
|
|||
|
|
assert positions == sorted(positions), (
|
|||
|
|
f"Секции в repo_researcher.md расположены не по порядку. "
|
|||
|
|
f"Позиции: {dict(zip(REQUIRED_SECTIONS, positions))}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_has_input_section(self):
|
|||
|
|
"""Промпт repo_researcher.md содержит секцию ## Input."""
|
|||
|
|
content = REPO_RESEARCHER_PROMPT.read_text(encoding="utf-8")
|
|||
|
|
assert "## Input" in content, "repo_researcher.md не содержит секцию '## Input'"
|
|||
|
|
|
|||
|
|
def test_prompt_contains_blocked_protocol(self):
|
|||
|
|
"""Промпт repo_researcher.md содержит Blocked Protocol."""
|
|||
|
|
content = REPO_RESEARCHER_PROMPT.read_text(encoding="utf-8")
|
|||
|
|
assert "blocked_reason" in content, (
|
|||
|
|
"repo_researcher.md не содержит 'blocked_reason' — Blocked Protocol обязателен"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ===========================================================================
|
|||
|
|
# 5. repo_researcher промпт — отсутствие API-полей
|
|||
|
|
# ===========================================================================
|
|||
|
|
|
|||
|
|
class TestRepoResearcherPromptNoApiFields:
|
|||
|
|
"""Промпт repo_researcher.md не упоминает API-специфичные поля в Return Format."""
|
|||
|
|
|
|||
|
|
def test_prompt_does_not_define_endpoints_field(self):
|
|||
|
|
"""Промпт repo_researcher.md не содержит поля 'endpoints' в выходной схеме."""
|
|||
|
|
content = REPO_RESEARCHER_PROMPT.read_text(encoding="utf-8")
|
|||
|
|
# endpoints может встречаться только в integration_points контексте,
|
|||
|
|
# но не как самостоятельное JSON-поле вида "endpoints":
|
|||
|
|
assert '"endpoints"' not in content, (
|
|||
|
|
"repo_researcher.md не должен определять JSON-поле 'endpoints' — "
|
|||
|
|
"это API-специфичное поле tech_researcher"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_does_not_define_rate_limits_field(self):
|
|||
|
|
"""Промпт repo_researcher.md не содержит поля 'rate_limits' в выходной схеме."""
|
|||
|
|
content = REPO_RESEARCHER_PROMPT.read_text(encoding="utf-8")
|
|||
|
|
assert '"rate_limits"' not in content, (
|
|||
|
|
"repo_researcher.md не должен определять JSON-поле 'rate_limits' — "
|
|||
|
|
"это API-специфичное поле tech_researcher"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_does_not_define_auth_method_field(self):
|
|||
|
|
"""Промпт repo_researcher.md не содержит поля 'auth_method' в выходной схеме."""
|
|||
|
|
content = REPO_RESEARCHER_PROMPT.read_text(encoding="utf-8")
|
|||
|
|
assert '"auth_method"' not in content, (
|
|||
|
|
"repo_researcher.md не должен определять JSON-поле 'auth_method' — "
|
|||
|
|
"это API-специфичное поле tech_researcher"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_defines_repo_overview_field(self):
|
|||
|
|
"""Промпт repo_researcher.md определяет поле 'repo_overview'."""
|
|||
|
|
content = REPO_RESEARCHER_PROMPT.read_text(encoding="utf-8")
|
|||
|
|
assert "repo_overview" in content, (
|
|||
|
|
"repo_researcher.md должен определять поле 'repo_overview'"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_defines_architecture_summary_field(self):
|
|||
|
|
"""Промпт repo_researcher.md определяет поле 'architecture_summary'."""
|
|||
|
|
content = REPO_RESEARCHER_PROMPT.read_text(encoding="utf-8")
|
|||
|
|
assert "architecture_summary" in content, (
|
|||
|
|
"repo_researcher.md должен определять поле 'architecture_summary'"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ===========================================================================
|
|||
|
|
# 6. repo_researcher в departments.research
|
|||
|
|
# ===========================================================================
|
|||
|
|
|
|||
|
|
class TestRepoResearcherInResearchDepartment:
|
|||
|
|
"""repo_researcher доступен в departments.research.workers."""
|
|||
|
|
|
|||
|
|
def test_repo_researcher_in_research_workers(self):
|
|||
|
|
"""repo_researcher присутствует в departments.research.workers."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
workers = data["departments"]["research"].get("workers", [])
|
|||
|
|
assert "repo_researcher" in workers, (
|
|||
|
|
f"repo_researcher должен быть в departments.research.workers. "
|
|||
|
|
f"Текущие workers: {workers}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_research_head_describes_repo_researcher(self):
|
|||
|
|
"""research_head description упоминает repo_researcher."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
description = data["specialists"]["research_head"].get("description", "")
|
|||
|
|
assert "repo_researcher" in description, (
|
|||
|
|
"research_head description должен упоминать repo_researcher"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_tech_researcher_still_in_research_workers(self):
|
|||
|
|
"""Регрессия: tech_researcher по-прежнему в departments.research.workers."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
workers = data["departments"]["research"].get("workers", [])
|
|||
|
|
assert "tech_researcher" in workers, (
|
|||
|
|
"Регрессия: tech_researcher пропал из departments.research.workers"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ===========================================================================
|
|||
|
|
# 7. Регрессия: tech_researcher не сломан (API-поля на месте)
|
|||
|
|
# ===========================================================================
|
|||
|
|
|
|||
|
|
class TestTechResearcherApiFieldsRegression:
|
|||
|
|
"""Регрессия: tech_researcher по-прежнему содержит API-специфичные поля."""
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize("api_field", sorted(TECH_RESEARCHER_API_FIELDS))
|
|||
|
|
def test_tech_researcher_output_schema_still_has_api_field(self, api_field):
|
|||
|
|
"""tech_researcher output_schema по-прежнему содержит API-поле (регрессия)."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
schema = data["specialists"]["tech_researcher"]["output_schema"]
|
|||
|
|
assert api_field in schema, (
|
|||
|
|
f"Регрессия: tech_researcher output_schema потеряла поле {api_field!r}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_tech_researcher_description_mentions_external_api(self):
|
|||
|
|
"""tech_researcher description явно указывает назначение: внешние API."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
description = data["specialists"]["tech_researcher"].get("description", "").lower()
|
|||
|
|
assert "external" in description or "api" in description, (
|
|||
|
|
"tech_researcher description должен упоминать внешние API"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_tech_researcher_description_mentions_repo_researcher(self):
|
|||
|
|
"""tech_researcher description упоминает repo_researcher для кодовых баз."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
description = data["specialists"]["tech_researcher"].get("description", "")
|
|||
|
|
assert "repo_researcher" in description, (
|
|||
|
|
"tech_researcher description должен упоминать repo_researcher "
|
|||
|
|
"как альтернативу для анализа репозиториев"
|
|||
|
|
)
|